12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383638463856386638763886389639063916392639363946395639663976398639964006401640264036404640564066407640864096410641164126413641464156416641764186419642064216422642364246425642664276428642964306431643264336434643564366437643864396440644164426443644464456446644764486449645064516452645364546455645664576458645964606461646264636464646564666467646864696470647164726473647464756476647764786479648064816482648364846485648664876488648964906491649264936494649564966497649864996500650165026503650465056506650765086509651065116512651365146515651665176518651965206521652265236524652565266527652865296530653165326533653465356536653765386539654065416542654365446545654665476548654965506551655265536554655565566557655865596560656165626563656465656566656765686569657065716572657365746575657665776578657965806581658265836584658565866587658865896590659165926593659465956596659765986599660066016602660366046605660666076608660966106611661266136614661566166617661866196620662166226623662466256626662766286629663066316632663366346635663666376638663966406641664266436644664566466647664866496650665166526653665466556656665766586659666066616662666366646665666666676668666966706671667266736674667566766677667866796680668166826683668466856686668766886689669066916692669366946695669666976698669967006701670267036704670567066707670867096710671167126713671467156716671767186719672067216722672367246725672667276728672967306731673267336734673567366737673867396740674167426743674467456746674767486749675067516752675367546755675667576758675967606761676267636764676567666767676867696770677167726773677467756776677767786779678067816782678367846785678667876788678967906791679267936794679567966797679867996800680168026803680468056806680768086809681068116812681368146815681668176818681968206821682268236824682568266827682868296830683168326833683468356836683768386839684068416842684368446845684668476848684968506851685268536854685568566857685868596860686168626863686468656866686768686869687068716872687368746875687668776878687968806881688268836884688568866887688868896890689168926893689468956896689768986899690069016902690369046905690669076908690969106911691269136914691569166917691869196920692169226923692469256926692769286929693069316932693369346935693669376938693969406941694269436944694569466947694869496950695169526953695469556956695769586959696069616962696369646965696669676968696969706971697269736974697569766977697869796980698169826983698469856986698769886989699069916992699369946995699669976998699970007001700270037004700570067007700870097010701170127013701470157016701770187019702070217022702370247025702670277028702970307031703270337034703570367037703870397040704170427043704470457046704770487049705070517052705370547055705670577058705970607061706270637064706570667067706870697070707170727073707470757076707770787079708070817082708370847085708670877088708970907091709270937094709570967097709870997100710171027103710471057106710771087109711071117112711371147115711671177118711971207121712271237124712571267127712871297130713171327133713471357136713771387139714071417142714371447145714671477148714971507151715271537154715571567157715871597160716171627163716471657166716771687169717071717172717371747175717671777178717971807181718271837184718571867187718871897190719171927193719471957196719771987199720072017202720372047205720672077208720972107211721272137214721572167217721872197220722172227223722472257226722772287229723072317232723372347235723672377238723972407241724272437244724572467247724872497250725172527253725472557256725772587259726072617262726372647265726672677268726972707271727272737274727572767277727872797280728172827283 |
- /*##############################################################################
- HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- ############################################################################## */
- #include "jliball.hpp"
- #include "hql.hpp"
- #include "platform.h"
- #include "jlib.hpp"
- #include "jmisc.hpp"
- #include "jstream.ipp"
- #include "jdebug.hpp"
- #include "eclrtl_imp.hpp"
- #include "rtlkey.hpp"
- #include "hql.hpp"
- #include "hqlattr.hpp"
- #include "hqlmeta.hpp"
- #include "hqlthql.hpp"
- #include "hqlhtcpp.ipp"
- #include "hqlttcpp.ipp"
- #include "hqlutil.hpp"
- #include "hqlthql.hpp"
- #include "hqlwcpp.hpp"
- #include "hqlcpputil.hpp"
- #include "hqltcppc.ipp"
- #include "hqlopt.hpp"
- #include "hqlfold.hpp"
- #include "hqlcerrors.hpp"
- #include "hqlcatom.hpp"
- #include "hqltrans.ipp"
- #include "hqlpmap.hpp"
- #include "hqlttcpp.ipp"
- #include "hqlsource.ipp"
- #include "hqlcse.ipp"
- #include "hqliter.ipp"
- #include "thorcommon.hpp"
- #include "hqlinline.hpp"
- //#define FLATTEN_DATASETS
- //#define HACK_TO_IGNORE_TABLE
- //#define TraceExprPrintLog(x, expr) PrintLog(x ": %s", expr->toString(StringBuffer()).str());
- #define TraceExprPrintLog(x, expr)
- //#define TraceTableFields
- inline bool needToSerializeRecord(node_operator mode)
- {
- return (mode == no_thor || mode == no_flat);
- }
- inline bool needToSerializeRecord(IHqlExpression * mode)
- {
- return needToSerializeRecord(mode->getOperator());
- }
- //---------------------------------------------------------------------------
- void HqlCppTranslator::addGlobalOnWarning(IHqlExpression * setMetaExpr)
- {
- globalOnWarnings->addOnWarning(setMetaExpr);
- }
- unsigned HqlCppTranslator::getSourceAggregateOptimizeFlags() const
- {
- const bool insideChildQuery = false; // value does not currently matter
- return getOptimizeFlags(insideChildQuery)|HOOfold|HOOinsidecompound;
- }
- void HqlCppTranslator::doBuildExprFilepos(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt)
- {
- if (buildExprInCorrectContext(ctx, expr, tgt, false))
- return;
- throwError(HQLERR_CouldNotResolveFileposition); // internal error: fileposition should have been available.
- }
- void HqlCppTranslator::doBuildExprFileLogicalName(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt)
- {
- if (buildExprInCorrectContext(ctx, expr, tgt, false))
- return;
- throwError(HQLERR_CouldNotResolveFileposition); // internal error: fileposition should have been available.
- }
- //---------------------------------------------------------------------------
- node_operator getDatasetKind(IHqlExpression * dataset)
- {
- dataset = queryPhysicalRootTable(dataset);
- if (dataset)
- {
- IHqlExpression * mode = dataset->queryChild(2);
- if (mode)
- return mode->getOperator();
- }
- return no_none;
- }
- unsigned getProjectCount(IHqlExpression * expr)
- {
- unsigned projectCount = 0;
- while (expr->getOperator() != no_table)
- {
- switch (expr->getOperator())
- {
- case no_hqlproject:
- case no_usertable:
- case no_newusertable:
- case no_selectfields:
- projectCount++;
- break;
- }
- expr = expr->queryChild(0);
- }
- return projectCount;
- }
- IHqlExpression * queryFetch(IHqlExpression * expr)
- {
- for (;;)
- {
- switch (expr->getOperator())
- {
- case no_fetch:
- return expr;
- case no_filter:
- case no_compound_fetch:
- case no_limit:
- case no_keyedlimit:
- case no_choosen:
- break;
- default:
- UNIMPLEMENTED;
- }
- expr = expr->queryChild(0);
- }
- }
- bool isSimpleSource(IHqlExpression * expr)
- {
- for (;;)
- {
- switch (expr->getOperator())
- {
- case no_keyindex:
- case no_newkeyindex:
- case no_table:
- case no_temptable:
- case no_inlinetable:
- case no_workunit_dataset:
- case no_xmlproject:
- case no_null:
- case no_datasetfromrow:
- case no_datasetfromdictionary:
- case no_getgraphresult:
- case no_getgraphloopresult:
- case no_rows:
- return true;
- case no_choosen:
- case no_limit:
- case no_keyedlimit:
- case no_sorted:
- case no_stepped:
- case no_distributed:
- case no_preservemeta:
- case no_unordered:
- case no_grouped:
- case no_compound_diskread:
- case no_compound_disknormalize:
- case no_compound_diskaggregate:
- case no_compound_diskcount:
- case no_compound_diskgroupaggregate:
- case no_compound_indexread:
- case no_compound_indexnormalize:
- case no_compound_indexaggregate:
- case no_compound_indexcount:
- case no_compound_indexgroupaggregate:
- case no_compound_childread:
- case no_compound_childnormalize:
- case no_compound_childaggregate:
- case no_compound_childcount:
- case no_compound_childgroupaggregate:
- case no_compound_selectnew:
- case no_compound_inline:
- case no_section:
- case no_sectioninput:
- case no_nofold:
- case no_nohoist:
- case no_nocombine:
- case no_dataset_alias:
- break;
- default:
- return false;
- }
- expr = expr->queryChild(0);
- }
- }
- IHqlExpression * getVirtualSelector(IHqlExpression * dataset)
- {
- IHqlExpression * table = queryPhysicalRootTable(dataset);
- if (!table)
- table = dataset;
- return LINK(table->queryNormalizedSelector());
- }
- IHqlExpression * getFilepos(IHqlExpression * dataset, bool isLocal)
- {
- IHqlExpression * attr = isLocal ? createLocalAttribute() : NULL;
- return createValue(no_filepos, LINK(fposType), getVirtualSelector(dataset), attr);
- }
- IHqlExpression * getFileLogicalName(IHqlExpression * dataset)
- {
- return createValue(no_file_logicalname, makeVarStringType(UNKNOWN_LENGTH, NULL, NULL), getVirtualSelector(dataset));
- }
- IHqlExpression * getVirtualReplacement(IHqlExpression * field, IHqlExpression * virtualDef, IHqlExpression * dataset)
- {
- IAtom * virtualKind = virtualDef->queryName();
- if (virtualKind == filepositionAtom)
- return getFilepos(dataset, false);
- else if (virtualKind == localFilePositionAtom)
- return getFilepos(dataset, true);
- else if (virtualKind == sizeofAtom)
- return createValue(no_sizeof, LINK(sizetType), getVirtualSelector(dataset));
- else if (virtualKind == logicalFilenameAtom)
- return createValue(no_implicitcast, field->getType(), getFileLogicalName(dataset));
- throwError1(HQLERR_UnknownVirtualAttr, str(virtualKind));
- return NULL;
- }
- static IHqlExpression * getExplicitlyPromotedCompare(IHqlExpression * filter)
- {
- switch (filter->getOperator())
- {
- case no_in:
- case no_notin:
- return LINK(filter);
- }
- IHqlExpression * l = filter->queryChild(0);
- IHqlExpression * r = filter->queryChild(1);
- ITypeInfo * lType = queryUnqualifiedType(l->queryType());
- ITypeInfo * rType = queryUnqualifiedType(r->queryType());
- if (lType == rType)
- return LINK(filter);
- //Add explicit casts to the type. ensureExprType won't add a (string) to a string2 field.
- Owned<ITypeInfo> promotedType = getPromotedECLType(lType, rType);
- HqlExprArray args;
- if (lType == promotedType)
- args.append(*LINK(l));
- else
- args.append(*createValue(no_implicitcast, LINK(promotedType), LINK(l)));
- args.append(*ensureExprType(r, promotedType));
- return filter->clone(args);
- }
- static IHqlExpression * createFileposCall(HqlCppTranslator & translator, IIdAtom * name, const char * provider, const char * rowname)
- {
- HqlExprArray args;
- args.append(*createVariable(provider, makeBoolType()));
- args.append(*createVariable(rowname, makeBoolType())); // really a row
- return translator.bindFunctionCall(name, args);
- }
- //---------------------------------------------------------------------------
- void VirtualFieldsInfo::gatherVirtualFields(IHqlExpression * _record, bool ignoreVirtuals, bool ensureSerialized)
- {
- OwnedHqlExpr record = ensureSerialized ? getSerializedForm(_record, diskAtom) : LINK(_record);
- if (record != _record)
- requiresDeserialize = true;
- //MORE: This should really recurse through records to check for nested virtual fields.
- // e.g., inside ifblocks, or even records....
- ForEachChild(idx, record)
- {
- IHqlExpression * cur = record->queryChild(idx);
- IHqlExpression * virtualAttr = NULL;
- if (!ignoreVirtuals)
- virtualAttr = cur->queryAttribute(virtualAtom);
- if (virtualAttr)
- {
- selects.append(*LINK(cur));
- if (isUnknownSize(cur->queryType()))
- simpleVirtualsAtEnd = false;
- if (virtuals.find(*virtualAttr) == NotFound)
- virtuals.append(*LINK(virtualAttr));
- }
- else
- {
- //Also adds attributes...
- physicalFields.append(*LINK(cur));
- if (virtuals.ordinality())
- simpleVirtualsAtEnd = false;
- }
- }
- }
- IHqlExpression * VirtualFieldsInfo::createPhysicalRecord()
- {
- if (physicalFields.ordinality() == 1)
- if (physicalFields.item(0).getOperator() == no_record)
- return LINK(&physicalFields.item(0));
- return createRecord(physicalFields);
- }
- //---------------------------------------------------------------------------
- class VirtualRecordTransformCreator : public RecordTransformCreator
- {
- public:
- VirtualRecordTransformCreator(IHqlExpression * _dataset) { dataset = _dataset; }
- virtual IHqlExpression * getMissingAssignValue(IHqlExpression * expr)
- {
- IHqlExpression * virtualAttr = expr->queryAttribute(virtualAtom);
- assertex(virtualAttr);
- return getVirtualReplacement(expr, virtualAttr->queryChild(0), dataset);
- }
- protected:
- IHqlExpression * dataset;
- };
- IHqlExpression * createTableWithoutVirtuals(VirtualFieldsInfo & info, IHqlExpression * tableExpr)
- {
- IHqlExpression * record = tableExpr->queryChild(1);
- OwnedHqlExpr diskRecord = info.createPhysicalRecord();
- //Clone the annotations to improve the regenerated text in the graph
- OwnedHqlExpr diskRecordWithMeta = record->cloneAllAnnotations(diskRecord);
- HqlExprArray args;
- unwindChildren(args, tableExpr);
- args.replace(*LINK(diskRecordWithMeta), 1);
- IHqlExpression * newDataset = tableExpr->queryBody()->clone(args);
- VirtualRecordTransformCreator mapper(newDataset);
- IHqlExpression * newTransform = mapper.createMappingTransform(no_newtransform, record, newDataset);
- OwnedHqlExpr projected = createDatasetF(no_newusertable, newDataset, LINK(record), newTransform, createAttribute(_internal_Atom), NULL);
- return tableExpr->cloneAllAnnotations(projected);
- }
- IHqlExpression * buildTableWithoutVirtuals(VirtualFieldsInfo & info, IHqlExpression * expr)
- {
- IHqlExpression * tableExpr = queryPhysicalRootTable(expr);
- OwnedHqlExpr projected = createTableWithoutVirtuals(info, tableExpr);
- return replaceExpression(expr, tableExpr, projected);
- }
- static IHqlExpression * nextDiskField(IHqlExpression * diskRecord, unsigned & diskIndex)
- {
- for (;;)
- {
- IHqlExpression * cur = diskRecord->queryChild(diskIndex++);
- if (!cur || !cur->isAttribute())
- return cur;
- }
- }
- static void createPhysicalLogicalAssigns(HqlExprArray & assigns, IHqlExpression * self, IHqlExpression * diskRecord, IHqlExpression * record, IHqlExpression * diskDataset, bool allowTranslate, unsigned fileposIndex)
- {
- unsigned numFields = record->numChildren();
- unsigned diskIndex = 0;
- for (unsigned idx2=0; idx2 < numFields; idx2++)
- {
- IHqlExpression * cur = record->queryChild(idx2);
- switch (cur->getOperator())
- {
- case no_ifblock:
- {
- IHqlExpression * ifblock = nextDiskField(diskRecord, diskIndex);
- assertex(ifblock && ifblock->getOperator() == no_ifblock);
- createPhysicalLogicalAssigns(assigns, self, ifblock->queryChild(1), cur->queryChild(1), diskDataset, false, NotFound);
- break;
- }
- case no_record:
- {
- IHqlExpression * srcRecord = nextDiskField(diskRecord, diskIndex);
- assertex(srcRecord && srcRecord->getOperator() == no_record);
- createPhysicalLogicalAssigns(assigns, self, srcRecord, cur, diskDataset, allowTranslate, NotFound);
- break;
- }
- case no_field:
- {
- OwnedHqlExpr target = createSelectExpr(LINK(self), LINK(cur));
- OwnedHqlExpr newValue;
- IHqlExpression * curPhysical = nextDiskField(diskRecord, diskIndex);
- OwnedHqlExpr physicalSelect = createSelectExpr(LINK(diskDataset), LINK(curPhysical));
- if (cur->isDatarow() && !cur->hasAttribute(blobAtom) && (!isInPayload() || (physicalSelect->queryType() != target->queryType())))
- {
- HqlExprArray subassigns;
- OwnedHqlExpr childSelf = createSelector(no_self, cur, NULL);
- createPhysicalLogicalAssigns(subassigns, childSelf, curPhysical->queryRecord(), cur->queryRecord(), physicalSelect, false, NotFound);
- OwnedHqlExpr transform = createValue(no_transform, makeTransformType(cur->queryRecord()->getType()), subassigns);
- newValue.setown(createRow(no_createrow, transform.getClear()));
- }
- else
- newValue.setown(convertIndexPhysical2LogicalValue(cur, physicalSelect, allowTranslate && (idx2 != fileposIndex)));
- if (newValue)
- assigns.append(*createAssign(target.getClear(), newValue.getClear()));
- break;
- }
- }
- }
- }
- static void createPhysicalLogicalAssigns(HqlExprArray & assigns, IHqlExpression * diskDataset, IHqlExpression * tableExpr, bool hasFilePosition)
- {
- IHqlExpression * record = tableExpr->queryRecord();
- unsigned fileposIndex = (hasFilePosition ? record->numChildren() - 1 : NotFound);
- OwnedHqlExpr self = getSelf(record);
- createPhysicalLogicalAssigns(assigns, self, diskDataset->queryRecord(), record, diskDataset, true, fileposIndex);
- }
- static IHqlExpression * mapIfBlock(HqlMapTransformer & mapper, IHqlExpression * cur);
- static IHqlExpression * mapIfBlockRecord(HqlMapTransformer & mapper, IHqlExpression * record)
- {
- HqlExprArray mapped;
- ForEachChild(i, record)
- {
- IHqlExpression * cur = record->queryChild(i);
- if (cur->getOperator() == no_ifblock)
- mapped.append(*mapIfBlock(mapper, cur));
- else
- mapped.append(*LINK(cur));
- }
- return record->clone(mapped);
- }
- static IHqlExpression * mapIfBlock(HqlMapTransformer & mapper, IHqlExpression * cur)
- {
- HqlExprArray args;
- unwindChildren(args, cur);
- args.replace(*mapper.transformRoot(&args.item(0)), 0);
- args.replace(*mapIfBlockRecord(mapper, &args.item(1)), 1);
- return cur->clone(args);
- }
- static IHqlExpression * createPhysicalIndexRecord(HqlMapTransformer & mapper, IHqlExpression * record, bool hasInternalFileposition, bool createKeyedTypes)
- {
- HqlExprArray physicalFields;
- unsigned max = record->numChildren() - (hasInternalFileposition ? 1 : 0);
- for (unsigned idx=0; idx < max; idx++)
- {
- IHqlExpression * cur = record->queryChild(idx);
- IHqlExpression * newField = NULL;
- if (cur->isAttribute())
- physicalFields.append(*LINK(cur));
- else if (cur->getOperator() == no_ifblock)
- physicalFields.append(*mapIfBlock(mapper, cur));
- else if (cur->getOperator() == no_record)
- physicalFields.append(*createPhysicalIndexRecord(mapper, cur, false, createKeyedTypes));
- else if (cur->hasAttribute(blobAtom))
- {
- newField = createField(cur->queryId(), makeIntType(8, false), nullptr, createKeyedTypes ? createAttribute(_payload_Atom) : nullptr);
- }
- else
- {
- //This should support other non serialized formats. E.g., link counted strings.
- //Simplest would be to move getSerializedForm code + call that first.
- if (cur->hasAttribute(_linkCounted_Atom) || cur->isDatarow())
- {
- newField = getSerializedForm(cur, diskAtom);
- assertex(newField != cur || cur->isDatarow());
- }
- else
- {
- Owned<ITypeInfo> hozedType = getHozedKeyType(cur);
- if (hozedType == cur->queryType())
- newField = LINK(cur);
- else if (createKeyedTypes)
- newField = createField(cur->queryId(), makeKeyedType(cur->getType()), nullptr, extractFieldAttrs(cur));
- else
- newField = createField(cur->queryId(), hozedType.getClear(), nullptr, extractFieldAttrs(cur));
- }
- }
- if (newField)
- {
- physicalFields.append(*newField);
- if (cur != newField)
- {
- IHqlExpression * self = querySelfReference();
- OwnedHqlExpr select = createSelectExpr(LINK(self), LINK(cur));
- OwnedHqlExpr physicalSelect = createSelectExpr(LINK(self), LINK(newField));
- OwnedHqlExpr newValue = convertIndexPhysical2LogicalValue(cur, physicalSelect, true);
- mapper.setMapping(select, newValue);
- }
- }
- }
- if (hasInternalFileposition)
- {
- if (createKeyedTypes)
- {
- IHqlExpression * cur = record->queryChild(record->numChildren()-1);
- IHqlExpression *fposField = createField(cur->queryId(), makeFilePosType(cur->getType()), nullptr, extractFieldAttrs(cur));
- physicalFields.append(*fposField);
- }
- else
- {
- IHqlExpression * cur = record->queryChild(record->numChildren()-1);
- IHqlExpression *fposField = createField(cur->queryId(), makeSwapIntType(8, false), nullptr, extractFieldAttrs(cur));
- physicalFields.append(*fposField);
- }
- }
- return createRecord(physicalFields);
- }
- IHqlExpression * createMetadataIndexRecord(IHqlExpression * record, bool hasInternalFilePosition)
- {
- HqlMapTransformer mapper;
- return createPhysicalIndexRecord(mapper, record, hasInternalFilePosition, true);
- }
- IHqlExpression * HqlCppTranslator::convertToPhysicalIndex(IHqlExpression * tableExpr)
- {
- LinkedHqlExpr * match = physicalIndexCache.getValue(tableExpr);
- if (match)
- return LINK(*match);
- if (tableExpr->hasAttribute(_original_Atom))
- return LINK(tableExpr);
- assertex(tableExpr->getOperator() == no_newkeyindex);
- IHqlExpression * record = tableExpr->queryRecord();
- HqlMapTransformer mapper;
- bool hasFileposition = getBoolAttribute(tableExpr, filepositionAtom, true);
- IHqlExpression * diskRecord = createPhysicalIndexRecord(mapper, record, hasFileposition, false);
- unsigned payload = numPayloadFields(tableExpr);
- assertex(payload || !hasFileposition);
- HqlExprArray args;
- unwindChildren(args, tableExpr);
- args.replace(*diskRecord, 1);
- removeAttribute(args, _payload_Atom);
- args.append(*createAttribute(_payload_Atom, getSizetConstant(payload)));
- args.append(*createAttribute(_original_Atom, LINK(tableExpr)));
- //remove the preload attribute and replace with correct value
- IHqlExpression * newDataset = createDataset(tableExpr->getOperator(), args);
- HqlExprArray assigns;
- createPhysicalLogicalAssigns(assigns, newDataset, tableExpr, hasFileposition);
- OwnedHqlExpr projectedTable = createDataset(no_newusertable, newDataset, createComma(LINK(record), createValue(no_newtransform, makeTransformType(record->getType()), assigns)));
- physicalIndexCache.setValue(tableExpr, projectedTable);
- return projectedTable.getClear();
- }
- IHqlExpression * convertToPhysicalTable(IHqlExpression * tableExpr, bool ensureSerialized)
- {
- VirtualFieldsInfo fieldInfo;
- fieldInfo.gatherVirtualFields(tableExpr->queryRecord(), tableExpr->hasAttribute(_noVirtual_Atom), ensureSerialized);
- if (fieldInfo.hasVirtualsOrDeserialize())
- return createTableWithoutVirtuals(fieldInfo, tableExpr);
- return LINK(tableExpr);
- }
- IHqlExpression * HqlCppTranslator::buildIndexFromPhysical(IHqlExpression * expr)
- {
- IHqlExpression * tableExpr = queryPhysicalRootTable(expr);
- OwnedHqlExpr newProject = convertToPhysicalIndex(tableExpr);
- return replaceExpression(expr, tableExpr, newProject);
- }
- //---------------------------------------------------------------------------
- class SourceSteppingInfo
- {
- public:
- inline bool exists() { return rawStepping.exists(); }
- IHqlExpression * firstStepped()
- {
- if (rawStepping.exists())
- return rawStepping.fields->queryChild(0);
- return NULL;
- }
- void extractRaw()
- {
- rawSteppingProject.extractFields(rawStepping);
- }
- void checkKeyable(MonitorExtractor & monitors)
- {
- if (!rawStepping.exists())
- return;
- unsigned prev = NotFound;
- ForEachChild(i, rawStepping.fields)
- {
- IHqlExpression * cur = rawStepping.fields->queryChild(i);
- unsigned thisMatch = monitors.queryKeySelectIndex(cur);
- if (thisMatch == NotFound)
- throwError1(HQLERR_StepFieldNotKeyed, str(cur->queryChild(1)->queryName()));
- if ((prev != NotFound) && (thisMatch != prev+1))
- throwError1(HQLERR_StepFieldNotContiguous, str(cur->queryChild(1)->queryName()));
- prev = thisMatch;
- }
- }
- void generateMembers(HqlCppTranslator & translator, BuildCtx & ctx)
- {
- rawStepping.generateSteppingMetaMember(translator, ctx, "RawSteppingMeta");
- if (outputStepping.exists())
- {
- if (outputStepping.exists() && outputStepping.ds != rawStepping.ds)
- {
- outputStepping.generateSteppingMetaMember(translator, ctx, "ProjectedSteppingMeta");
- MemberFunction func(translator, ctx, "virtual void mapOutputToInput(ARowBuilder & crSelf, const void * _projected, unsigned numFields) override");
- translator.ensureRowAllocated(func.ctx, "crSelf");
- func.ctx.addQuotedLiteral("const byte * pr = (const byte *)_projected;");
- translator.bindTableCursor(func.ctx, rawStepping.ds, "crSelf.row()");
- translator.bindTableCursor(func.ctx, outputStepping.ds, "pr");
- StringBuffer s;
- ForEachChild(i, outputStepping.fields)
- {
- IHqlExpression * curOutput = outputStepping.fields->queryChild(i);
- IHqlExpression * curRawExpr = rawSteppingProject.fields->queryChild(i);
- IHqlExpression * curRawSelect = rawStepping.fields->queryChild(i);
- OwnedHqlExpr original = outputStepping.invertTransform(curRawExpr, curOutput);
- func.ctx.addQuoted(s.clear().append("if (numFields < ").append(i+1).append(") return;"));
- translator.buildAssign(func.ctx, curRawSelect, original);
- }
- }
- }
- else
- {
- OwnedHqlExpr fail = createValue(no_fail, makeVoidType(), createConstant("Cannot step output of index read"));
- MemberFunction func(translator, ctx, "virtual void mapOutputToInput(ARowBuilder & crSelf, const void * _projected, unsigned numFields) override");
- translator.buildStmt(func.ctx, fail);
- }
- }
- public:
- SteppingFieldSelection outputStepping;
- SteppingFieldSelection rawSteppingProject;
- SteppingFieldSelection rawStepping;
- };
- //---------------------------------------------------------------------------
- class SourceBuilder
- {
- public:
- SourceBuilder(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr)
- : tableExpr(_tableExpr), translator(_translator)
- {
- nameExpr.setown(foldHqlExpression(_nameExpr));
- needDefaultTransform = true;
- needToCallTransform = false;
- isPreloaded = false;
- isCompoundCount = false;
- transformCanFilter = false;
- IHqlExpression * preload = tableExpr ? tableExpr->queryAttribute(preloadAtom) : NULL;
- if (preload)
- {
- isPreloaded = true;
- preloadSize.set(preload->queryChild(0));
- }
- failedFilterValue.set(queryZero());
- isNormalize = false;
- aggregation = false;
- instance = NULL;
- returnIfFilterFails = true;
- useFilterMappings = true;
- generateUnfilteredTransform = false;
- allowDynamicFormatChange = tableExpr && !tableExpr->hasAttribute(fixedAtom);
- onlyExistsAggreate = false;
- monitorsForGrouping = false;
- useImplementationClass = false;
- isUnfilteredCount = false;
- isVirtualLogicalFilenameUsed = false;
- requiresOrderedMerge = false;
- rootSelfRow = NULL;
- activityKind = TAKnone;
- }
- virtual ~SourceBuilder() {}
- virtual void buildMembers(IHqlExpression * expr) = 0;
- virtual void buildTransformFpos(BuildCtx & transformCtx) = 0;
- virtual void extractMonitors(IHqlExpression * ds, SharedHqlExpr & unkeyedFilter, HqlExprArray & conds);
- virtual void buildTransformElements(BuildCtx & ctx, IHqlExpression * expr, bool ignoreFilters);
- virtual void buildTransform(IHqlExpression * expr) = 0;
- virtual void analyse(IHqlExpression * expr);
- void buildFilenameMember();
- void appendFilter(SharedHqlExpr & unkeyedFilter, IHqlExpression * expr);
- void buildKeyedLimitHelper(IHqlExpression * expr);
- void buildLimits(BuildCtx & classctx, IHqlExpression * expr, unique_id_t id);
- void buildReadMembers( IHqlExpression * expr);
- void buildSteppingMeta(IHqlExpression * expr, MonitorExtractor * monitors);
- void buildTransformBody(BuildCtx & transformCtx, IHqlExpression * expr, bool returnSize, bool ignoreFilters, bool bindInputRow);
- void checkDependencies(BuildCtx & ctx, IHqlExpression * expr);
- bool containsStepping(IHqlExpression * expr);
- ABoundActivity * buildActivity(BuildCtx & ctx, IHqlExpression * expr, ThorActivityKind activityKind, const char *kind, ABoundActivity *input);
- void gatherVirtualFields(bool ignoreVirtuals, bool ensureSerialized);
- bool recordHasVirtuals() { return fieldInfo.hasVirtuals(); }
- bool recordHasVirtualsOrDeserialize() { return fieldInfo.hasVirtualsOrDeserialize(); }
- bool isSourceInvariant(IHqlExpression * dataset, IHqlExpression * expr);
- bool hasExistChoosenLimit() { return (choosenValue && getIntValue(choosenValue) == 1); }
- protected:
- void assignLocalExtract(BuildCtx & ctx, ParentExtract * extractBuilder, IHqlExpression * dataset, const char * argName);
- void associateFilePositions(BuildCtx & ctx, const char * provider, const char * rowname);
- void buildSteppedHelpers();
- void doBuildAggregateSelectIterator(BuildCtx & ctx, IHqlExpression * expr);
- void doBuildNormalizeIterators(BuildCtx & ctx, IHqlExpression * expr, bool isChildIterator);
- void buildAggregateHelpers(IHqlExpression * expr);
- void buildCountHelpers(IHqlExpression * expr, bool allowMultiple);
- virtual void buildFlagsMember(IHqlExpression * expr) {}
- void buildGlobalGroupAggregateHelpers(IHqlExpression * expr);
- void buildGroupAggregateHelpers(ParentExtract * extractBuilder, IHqlExpression * aggregate);
- void buildGroupAggregateCompareHelper(ParentExtract * extractBuilder, IHqlExpression * aggregate, HqlExprArray & recordFields, HqlExprArray & agrgegateFields);
- void buildGroupAggregateHashHelper(ParentExtract * extractBuilder, IHqlExpression * dataset, IHqlExpression * fields);
- void buildGroupAggregateProcessHelper(ParentExtract * extractBuilder, IHqlExpression * aggregate, const char * name, bool doneAny);
- void buildGroupingMonitors(IHqlExpression * expr, MonitorExtractor & monitors);
- void buildGroupAggregateTransformBody(BuildCtx & transformctx, IHqlExpression * expr, bool useExtract, bool bindInputRow);
- void buildNormalizeHelpers(IHqlExpression * expr);
- void buildTargetCursor(Shared<BoundRow> & tempRow, Shared<BoundRow> & rowBuilder, BuildCtx & ctx, IHqlExpression * expr);
- void associateTargetCursor(BuildCtx & subctx, BuildCtx & ctx, BoundRow * tempRow, BoundRow * rowBuilder, IHqlExpression * expr);
- IHqlExpression * ensureAggregateGroupingAliased(IHqlExpression * aggregate);
- void gatherSteppingMeta(IHqlExpression * expr, SourceSteppingInfo & info);
- void gatherSteppingMeta(IHqlExpression * expr, SteppingFieldSelection & outputStepping, SteppingFieldSelection & rawStepping);
- void gatherFieldUsage(SourceFieldUsage * fieldUsage, IHqlExpression * expr);
- void rebindFilepositons(BuildCtx & ctx, IHqlExpression * dataset, node_operator side, IHqlExpression * selSeq, bool isLocal);
- void rebindFilepositons(BuildCtx & ctx, IHqlExpression * dataset, node_operator side, IHqlExpression * selSeq);
- virtual void processTransformSelect(BuildCtx & ctx, IHqlExpression * expr);
- virtual void analyseGraph(IHqlExpression * expr);
- virtual bool isExplicitExists() { return false; }
- public:
- VirtualFieldsInfo fieldInfo;
- ActivityInstance * instance;
- BoundRow * rootSelfRow;
- HqlExprAttr tableExpr;
- HqlExprAttr nameExpr;
- HqlExprAttr fpos;
- HqlExprAttr lfpos;
- HqlExprAttr limitExpr;
- HqlExprAttr keyedLimitExpr;
- HqlExprAttr choosenValue;
- HqlExprAttr preloadSize;
- HqlExprAttr lastTransformer;
- HqlExprAttr alreadyDoneFlag;
- HqlExprArray originalFilters;
- HqlExprArray mappedFilters;
- HqlExprArray removedFilters;
- HqlExprAttr failedFilterValue;
- HqlExprAttr compoundCountVar;
- HqlExprAttr physicalRecord;
- HqlExprAttr steppedExpr;
- Linked<BuildCtx> globaliterctx;
- HqlExprCopyArray parentCursors;
- HqlExprAttr logicalFilenameMarker;
- ThorActivityKind activityKind;
- bool useFilterMappings;
- bool needDefaultTransform;
- bool needToCallTransform;
- bool isPreloaded;
- bool transformCanFilter;
- bool isCompoundCount;
- bool isNormalize;
- bool aggregation;
- bool returnIfFilterFails;
- bool allowDynamicFormatChange;
- bool onlyExistsAggreate;
- bool monitorsForGrouping;
- bool generateUnfilteredTransform;
- bool useImplementationClass;
- bool isUnfilteredCount;
- bool isVirtualLogicalFilenameUsed;
- bool requiresOrderedMerge;
- protected:
- HqlCppTranslator & translator;
- };
- struct HQLCPP_API HqlFilePositionDefinedValue : public HqlSimpleDefinedValue
- {
- public:
- HqlFilePositionDefinedValue(SourceBuilder & _builder, IHqlExpression * _original, IHqlExpression * _expr)
- : HqlSimpleDefinedValue(_original, _expr), builder(_builder)
- { }
- virtual IHqlExpression * queryExpr() const
- {
- builder.isVirtualLogicalFilenameUsed = true;
- return HqlSimpleDefinedValue::queryExpr();
- }
- public:
- SourceBuilder & builder;
- };
- bool SourceBuilder::isSourceInvariant(IHqlExpression * dataset, IHqlExpression * expr)
- {
- if (containsAssertKeyed(expr))
- return false;
- if (!containsActiveDataset(expr))
- return true;
- HqlExprCopyArray inScope;
- expr->gatherTablesUsed(NULL, &inScope);
- if (!canEvaluateInScope(parentCursors, inScope))
- return false;
- //Carefull.... It looks ok, but it is possible that the same dataset occurs at multiple levels (sqfilt.hql)
- //So need to be careful that the datasets being referenced aren't newly in scope for this activity...
- for (;;)
- {
- if (inScope.contains(*dataset->queryNormalizedSelector()))
- return false;
- //To be strictly correct we need to walk no_select chain of root datasets!
- dataset = queryNextMultiLevelDataset(dataset, true);
- if (!dataset)
- return true;
- }
- }
- void SourceBuilder::analyse(IHqlExpression * expr)
- {
- IHqlExpression * body = expr->queryBody(true);
- if (expr != body)
- {
- switch (expr->getAnnotationKind())
- {
- case annotate_meta:
- //the onwarning state will be restored by the scope held in HqlCppTranslator::buildActivity
- translator.localOnWarnings->processMetaAnnotation(expr);
- break;
- case annotate_location:
- instance->addLocationAttribute(expr);
- break;
- case annotate_symbol:
- //don't clear onWarnings when we hit a symbol because the warnings within a compound activity aren't generated at the correct point
- instance->addNameAttribute(expr);
- break;
- }
- analyse(body);
- return;
- }
- node_operator op = expr->getOperator();
- switch (op)
- {
- case no_cachealias:
- analyse(expr->queryChild(1));
- return;
- case no_newkeyindex:
- case no_table:
- case no_fetch:
- case no_select: // handled below
- case no_null:
- case no_anon:
- case no_pseudods:
- case no_workunit_dataset:
- case no_call:
- case no_externalcall:
- case no_rows:
- case no_libraryinput:
- break;
- default:
- analyse(expr->queryChild(0));
- break;
- }
- switch (op)
- {
- case no_table:
- case no_newkeyindex:
- if (fieldInfo.hasVirtuals())
- {
- assertex(fieldInfo.simpleVirtualsAtEnd);
- needToCallTransform = true;
- needDefaultTransform = false;
- }
- break;
- case no_null:
- case no_anon:
- case no_pseudods:
- case no_workunit_dataset:
- case no_getgraphresult:
- case no_call:
- case no_externalcall:
- case no_rows:
- case no_libraryinput:
- break;
- case no_filter:
- {
- OwnedHqlExpr unkeyedFilter;
- HqlExprArray conds;
- unwindFilterConditions(conds, expr);
- extractMonitors(expr, unkeyedFilter, conds);
- if (unkeyedFilter)
- {
- transformCanFilter = true;
- originalFilters.append(*LINK(expr));
- mappedFilters.append(*unkeyedFilter.getClear());
- }
- else
- removedFilters.append(*LINK(expr));
- break;
- }
- case no_select:
- {
- bool isNew;
- IHqlExpression * ds = querySelectorDataset(expr, isNew);
- if (isNew && isMultiLevelDatasetSelector(expr, false))
- {
- if (!translator.resolveSelectorDataset(instance->startctx, ds))
- {
- analyse(ds);
- isNormalize = true;
- }
- }
- break;
- }
- case no_stepped:
- if (steppedExpr)
- throwError(HQLERR_MultipleStepped);
- steppedExpr.set(expr);
- break;
- case no_sorted:
- case no_distributed:
- case no_preservemeta:
- case no_unordered:
- case no_grouped:
- case no_alias_scope:
- case no_section:
- case no_sectioninput:
- case no_nofold:
- case no_nohoist:
- case no_nocombine:
- case no_dataset_alias:
- break;
- case no_preload:
- isPreloaded = true;
- preloadSize.set(queryRealChild(expr, 1));
- break;
- case no_limit:
- limitExpr.set(expr);
- break;
- case no_keyedlimit:
- keyedLimitExpr.set(expr);
- break;
- case no_choosen:
- {
- choosenValue.set(expr->queryChild(1));
- IHqlExpression * first = queryRealChild(expr, 2);
- if (first)
- {
- Owned<ITypeInfo> type = makeIntType(8, true);
- choosenValue.setown(createValue(no_sub, LINK(type), createValue(no_add, LINK(type), ensureExprType(choosenValue, type), ensureExprType(first, type)), createConstant(I64C(1))));
- }
- choosenValue.setown(foldHqlExpression(choosenValue));
- }
- break;
- case no_hqlproject:
- needToCallTransform = true;
- needDefaultTransform = false;
- lastTransformer.set(expr);
- break;
- case no_newusertable:
- needToCallTransform = true;
- needDefaultTransform = false;
- lastTransformer.set(expr);
- break;
- case no_aggregate:
- {
- needToCallTransform = true;
- needDefaultTransform = false;
- aggregation = true;
- lastTransformer.set(expr);
- break;
- }
- case no_newaggregate:
- {
- needToCallTransform = true;
- needDefaultTransform = false;
- aggregation = true;
- lastTransformer.set(expr);
- IHqlExpression * transform = expr->queryChild(2);
- node_operator aggOp = queryTransformSingleAggregate(transform);
- onlyExistsAggreate = ((aggOp == no_existsgroup) || (aggOp == no_none)); // The implicit project code can remove all the aggregate() operators....
- if (isCompoundCount)
- {
- IHqlExpression * rhs = transform->queryChild(0)->queryChild(1);
- IHqlExpression * filter = queryRealChild(rhs, 0);
- if (filter)
- transformCanFilter = true;
- }
- break;
- }
- case no_usertable:
- case no_selectfields:
- UNIMPLEMENTED;
- break;
- case no_fetch:
- needToCallTransform = true;
- needDefaultTransform = false;
- lastTransformer.set(expr);
- break;
- case no_compound_diskread:
- case no_compound_disknormalize:
- case no_compound_diskaggregate:
- case no_compound_diskcount:
- case no_compound_diskgroupaggregate:
- case no_compound_indexread:
- case no_compound_indexnormalize:
- case no_compound_indexaggregate:
- case no_compound_indexcount:
- case no_compound_indexgroupaggregate:
- case no_compound_childread:
- case no_compound_childnormalize:
- case no_compound_childaggregate:
- case no_compound_childcount:
- case no_compound_childgroupaggregate:
- case no_compound_fetch:
- case no_compound_selectnew:
- break;
- default:
- throwUnexpectedOp(op);
- }
- }
- void SourceBuilder::appendFilter(SharedHqlExpr & unkeyedFilter, IHqlExpression * expr)
- {
- if (expr)
- {
- if (expr->queryValue())
- {
- if (!expr->queryValue()->getBoolValue())
- unkeyedFilter.set(expr);
- }
- else
- {
- if (unkeyedFilter)
- unkeyedFilter.setown(createValue(no_and, unkeyedFilter.getClear(), LINK(expr)));
- else
- unkeyedFilter.set(expr);
- }
- }
- }
- void SourceBuilder::associateFilePositions(BuildCtx & ctx, const char * provider, const char * rowname)
- {
- if (fpos)
- {
- Owned<IHqlExpression> fposExpr = createFileposCall(translator, getFilePositionId, provider, rowname);
- ctx.associateExpr(fpos, fposExpr);
- }
- if (lfpos)
- {
- Owned<IHqlExpression> fposExpr = createFileposCall(translator, getLocalFilePositionId, provider, rowname);
- ctx.associateExpr(lfpos, fposExpr);
- }
- if (logicalFilenameMarker)
- {
- Owned<IHqlExpression> nameExpr = createFileposCall(translator, queryLogicalFilenameId, provider, rowname);
- ctx.associateOwn(*new HqlFilePositionDefinedValue(*this, logicalFilenameMarker, nameExpr));
- }
- }
- void SourceBuilder::rebindFilepositons(BuildCtx & ctx, IHqlExpression * dataset, node_operator side, IHqlExpression * selSeq, bool isLocal)
- {
- if (!tableExpr)
- return;
- OwnedHqlExpr searchPos = getFilepos(tableExpr, isLocal);
- HqlExprAssociation * match = ctx.queryMatchExpr(searchPos);
- if (match)
- {
- OwnedHqlExpr selector = createSelector(side, dataset, selSeq);
- OwnedHqlExpr selectorFpos = getFilepos(selector, isLocal);
- ctx.associateExpr(selectorFpos, match->queryExpr());
- }
- }
- void SourceBuilder::rebindFilepositons(BuildCtx & ctx, IHqlExpression * dataset, node_operator side, IHqlExpression * selSeq)
- {
- bool savedIsVirtualLogicalFilenameUsed = isVirtualLogicalFilenameUsed; // don't allow the rebinding to set the flag.
- rebindFilepositons(ctx, dataset, side, selSeq, true);
- rebindFilepositons(ctx, dataset, side, selSeq, false);
- OwnedHqlExpr searchLogicalFilename = getFileLogicalName(dataset);
- HqlExprAssociation * match = ctx.queryMatchExpr(searchLogicalFilename);
- if (match)
- {
- OwnedHqlExpr selector = createSelector(side, dataset, selSeq);
- OwnedHqlExpr selectorLogicalFilename = getFileLogicalName(dataset);
- ctx.associateOwn(*new HqlFilePositionDefinedValue(*this, selectorLogicalFilename, match->queryExpr()));
- }
- isVirtualLogicalFilenameUsed = savedIsVirtualLogicalFilenameUsed;
- }
- void SourceBuilder::buildFilenameMember()
- {
- //---- virtual const char * getFileName() { return "x.d00"; } ----
- translator.buildFilenameFunction(*instance, instance->startctx, "getFileName", nameExpr, translator.hasDynamicFilename(tableExpr));
- }
- void SourceBuilder::buildReadMembers(IHqlExpression * expr)
- {
- buildFilenameMember();
- //---- virtual bool needTransform() { return <bool>; } ----
- if (needToCallTransform || transformCanFilter)
- translator.doBuildBoolFunction(instance->classctx, "needTransform", true);
- //---- virtual bool transformMayFilter() { return <bool>; } ----
- if (transformCanFilter)
- translator.doBuildBoolFunction(instance->classctx, "transformMayFilter", true);
- }
- void SourceBuilder::buildLimits(BuildCtx & classctx, IHqlExpression * expr, unique_id_t id)
- {
- if (limitExpr)
- translator.buildLimitHelpers(classctx, limitExpr, nameExpr, id);
- if (choosenValue)
- {
- MemberFunction func(translator, classctx, "virtual unsigned __int64 getChooseNLimit() override");
- OwnedHqlExpr newLimit = ensurePositiveOrZeroInt64(choosenValue);
- translator.buildReturn(func.ctx, newLimit);
- }
- }
- void SourceBuilder::buildTransformBody(BuildCtx & transformCtx, IHqlExpression * expr, bool returnSize, bool ignoreFilters, bool bindInputRow)
- {
- if (tableExpr && bindInputRow)
- {
- IHqlExpression * mode = (tableExpr->getOperator() == no_table) ? tableExpr->queryChild(2) : NULL;
- if (mode && mode->getOperator() == no_csv)
- {
- translator.bindCsvTableCursor(transformCtx, tableExpr, "Src", no_none, NULL, true, queryCsvEncoding(mode));
- }
- else
- {
- //NOTE: The source is not link counted - it comes from a prefetched row, and does not include any virtual file position field.
- OwnedHqlExpr boundSrc = createVariable("left", makeRowReferenceType(physicalRecord));
- IHqlExpression * accessor = NULL;
- transformCtx.associateOwn(*new BoundRow(tableExpr->queryNormalizedSelector(), boundSrc, accessor, translator.queryRecordOffsetMap(physicalRecord, (accessor != NULL)), no_none, NULL));
- }
- transformCtx.addGroup();
- }
- rootSelfRow = translator.bindSelf(transformCtx, expr, "crSelf");
- buildTransformFpos(transformCtx);
- buildTransformElements(transformCtx, expr, ignoreFilters);
- if (returnSize)
- {
- CHqlBoundExpr boundTargetSize;
- if (needDefaultTransform)
- {
- IHqlExpression * left = expr->queryNormalizedSelector();
- OwnedHqlExpr source = ensureActiveRow(left);
- translator.buildAssign(transformCtx, rootSelfRow->querySelector(), source);
- }
- translator.getRecordSize(transformCtx, rootSelfRow->querySelector(), boundTargetSize);
- transformCtx.setNextDestructor();
- transformCtx.addReturn(boundTargetSize.expr);
- }
- rootSelfRow = NULL;
- }
- void SourceBuilder::buildTargetCursor(Shared<BoundRow> & tempRow, Shared<BoundRow> & rowBuilder, BuildCtx & ctx, IHqlExpression * expr)
- {
- assertex(lastTransformer != NULL);
- if (expr == lastTransformer)
- {
- rowBuilder.set(rootSelfRow);
- }
- else
- {
- tempRow.setown(translator.declareTempAnonRow(ctx, ctx, expr));
- ctx.addGroup();
- // group is important, otherwise sizeof(self.x) gets cached incorrectly
- // not so sure, but references to LEFT(x) may be misresolved
- rowBuilder.setown(translator.createRowBuilder(ctx, tempRow));
- }
- }
- void SourceBuilder::associateTargetCursor(BuildCtx & subctx, BuildCtx & ctx, BoundRow * tempRow, BoundRow * rowBuilder, IHqlExpression * expr)
- {
- //First remove the old active dataset
- //This is not strictly necessary, but it avoids the redundant row being serialized to any child queries
- BoundRow * oldCursor = translator.resolveSelectorDataset(ctx, expr->queryChild(0));
- ctx.removeAssociation(oldCursor);
- //And add an association for expr
- if (tempRow)
- {
- translator.finalizeTempRow(subctx, tempRow, rowBuilder);
- translator.bindTableCursor(ctx, expr, tempRow->queryBound());
- }
- else
- translator.bindTableCursor(ctx, expr, rowBuilder->queryBound());
- }
- void SourceBuilder::buildTransformElements(BuildCtx & ctx, IHqlExpression * expr, bool ignoreFilters)
- {
- //This function can be called again for the unfiltered tranform. Don't process annotations again.
- if ((expr != instance->activityExpr) && !ignoreFilters)
- instance->processAnnotations(expr);
- expr = expr->queryBody();
- node_operator op = expr->getOperator();
- switch (op)
- {
- case no_cachealias:
- buildTransformElements(ctx, expr->queryChild(1), ignoreFilters);
- return;
- case no_newkeyindex:
- case no_table:
- case no_fetch:
- case no_select: // handled below
- case no_null:
- case no_anon:
- case no_pseudods:
- case no_workunit_dataset:
- case no_getgraphresult:
- case no_call:
- case no_externalcall:
- case no_rows:
- case no_libraryinput:
- break;
- default:
- buildTransformElements(ctx, expr->queryChild(0), ignoreFilters);
- break;
- }
- switch (op)
- {
- case no_newkeyindex:
- case no_table:
- if (fieldInfo.hasVirtuals())
- {
- IHqlExpression * record = expr->queryRecord();
- assertex(fieldInfo.simpleVirtualsAtEnd);
- assertex(!recordRequiresSerialization(record, diskAtom));
- CHqlBoundExpr bound;
- StringBuffer s;
- translator.getRecordSize(ctx, expr, bound);
- size32_t virtualSize = 0;
- ForEachChild(idx, record)
- {
- IHqlExpression * field = record->queryChild(idx);
- IHqlExpression * virtualAttr = field->queryAttribute(virtualAtom);
- if (virtualAttr)
- {
- size32_t fieldSize = field->queryType()->getSize();
- assertex(fieldSize != UNKNOWN_LENGTH);
- virtualSize += fieldSize;
- }
- }
- if (!isFixedSizeRecord(record))
- {
- OwnedHqlExpr ensureSize = adjustValue(bound.expr, virtualSize);
- s.clear().append("crSelf.ensureCapacity(");
- translator.generateExprCpp(s, ensureSize).append(", NULL);");
- ctx.addQuoted(s);
- }
- s.clear().append("memcpy(crSelf.row(), left, ");
- translator.generateExprCpp(s, bound.expr).append(");");
- ctx.addQuoted(s);
- ForEachChild(idx2, record)
- {
- IHqlExpression * field = record->queryChild(idx2);
- IHqlExpression * virtualAttr = field->queryAttribute(virtualAtom);
- if (virtualAttr)
- {
- IHqlExpression * self = rootSelfRow->querySelector();
- OwnedHqlExpr target = createSelectExpr(LINK(self), LINK(field));
- OwnedHqlExpr value = getVirtualReplacement(field, virtualAttr->queryChild(0), expr);
- translator.buildAssign(ctx, target, value);
- }
- }
- }
- break;
- case no_null:
- case no_anon:
- case no_pseudods:
- break;
- case no_workunit_dataset:
- case no_getgraphresult:
- case no_call:
- case no_externalcall:
- case no_rows:
- case no_libraryinput:
- throwUnexpectedOp(op);
- break;
- case no_select:
- processTransformSelect(ctx, expr);
- break;
- case no_sorted:
- case no_stepped:
- case no_distributed:
- case no_preservemeta:
- case no_unordered:
- case no_grouped:
- case no_preload:
- case no_limit:
- case no_keyedlimit:
- case no_choosen:
- case no_alias_scope:
- case no_section:
- case no_sectioninput:
- case no_nofold:
- case no_nohoist:
- case no_nocombine:
- break;
- case no_filter:
- {
- if (!ignoreFilters)
- {
- LinkedHqlExpr cond;
- if (useFilterMappings)
- {
- unsigned match = originalFilters.find(*expr);
- if (match != NotFound)
- cond.set(&mappedFilters.item(match));
- }
- else
- {
- HqlExprArray args;
- unwindRealChildren(args, expr, 1);
- cond.setown(createBalanced(no_and, queryBoolType(), args));
- }
- if (cond)
- {
- IHqlExpression * ds = expr->queryChild(0);
- OwnedHqlExpr test = returnIfFilterFails ? getInverse(cond) : LINK(cond);
- if (translator.queryOptions().foldFilter)
- test.setown(foldScopedHqlExpression(translator.queryErrorProcessor(), ds->queryNormalizedSelector(), test));
- if (translator.options.spotCSE)
- test.setown(spotScalarCSE(test, ds, translator.queryOptions().spotCseInIfDatasetConditions));
- if (!returnIfFilterFails)
- translator.buildFilter(ctx, test);
- else
- {
- LinkedHqlExpr mismatchReturnValue = failedFilterValue;
- //If the output row has already been generated, then returning at this point will leak any
- //child datasets. To avoid that we explicitly call the destructor on the output row.
- if (recordRequiresDestructor(expr->queryRecord()))
- {
- if (lastTransformer && lastTransformer->queryNormalizedSelector() == expr->queryNormalizedSelector())
- {
- StringBuffer s;
- translator.buildMetaForRecord(s, expr->queryRecord());
- s.append(".destruct(crSelf.row())");
- OwnedHqlExpr cleanupAction = createQuoted(s.str(), makeVoidType());
- //Create a compound expression (destroy-old, return-value)
- mismatchReturnValue.setown(createCompound(LINK(cleanupAction), LINK(mismatchReturnValue)));
- }
- }
- translator.buildFilteredReturn(ctx, test, mismatchReturnValue);
- }
- }
- }
- }
- break;
- case no_hqlproject:
- {
- IHqlExpression * dataset = expr->queryChild(0);
- IHqlExpression * selSeq = querySelSeq(expr);
- OwnedHqlExpr leftSelect = createSelector(no_left, dataset, querySelSeq(expr));
- //Following is a bit nasty....
- //Converting the no_hqlproject to a no_newusertable means that some of the expressions
- //are commoned up with expressions calculated by a previous filter, reducing the code.
- //However, it isn't valid if transform contains an instance of newSelect -
- //e.g. project(i(x), transform(exists(i....))) - see jholt25.xhql
- //And unfortunately it fails silently.
- //So we use queryReplaceSelector which fails if an ambiguity is introduced by the replacement
- OwnedHqlExpr newSelect = ensureActiveRow(dataset->queryNormalizedSelector());
- OwnedHqlExpr transform = queryNewReplaceSelector(expr->queryChild(1), leftSelect, newSelect);
- BuildCtx subctx(ctx); // buildTargetCursor adds group if necessary
- Linked<BoundRow> tempRow;
- Linked<BoundRow> rowBuilder;
- buildTargetCursor(tempRow, rowBuilder, subctx, expr);
- if (!transform)
- {
- //The replace introduced an ambiguity => need to use the unmapped expression.
- BoundRow * prevCursor = translator.resolveSelectorDataset(ctx, dataset->queryNormalizedSelector());
- transform.set(expr->queryChild(1));
- translator.rebindTableCursor(subctx, dataset, prevCursor, no_left, selSeq);
- rebindFilepositons(subctx, dataset, no_left, selSeq);
- }
- if (returnIfFilterFails)
- {
- translator.associateSkipReturnMarker(subctx, failedFilterValue, NULL); // failedFilterValue already handles clearing the result
- }
- else
- {
- //MORE: Probably not implementable, should try and prevent this happening!
- //translator.associateSkipReturnMarker(subctx, failedFilterValue, NULL);
- }
- translator.doTransform(subctx, transform, rowBuilder);
- ctx.addGroup();
- associateTargetCursor(subctx, ctx, tempRow, rowBuilder, expr);
- }
- break;
- case no_newusertable:
- {
- BuildCtx subctx(ctx);
- Linked<BoundRow> tempRow;
- Linked<BoundRow> rowBuilder;
- buildTargetCursor(tempRow, rowBuilder, subctx, expr);
- IHqlExpression * transform = expr->queryChild(2);
- if (returnIfFilterFails)
- {
- translator.associateSkipReturnMarker(subctx, failedFilterValue, NULL); // failedFilterValue already handles clearing the result
- }
- else
- {
- //MORE: Probably not implementable
- //translator.associateSkipReturnMarker(subctx, failedFilterValue, NULL);
- }
- translator.doTransform(subctx, transform, rowBuilder);
- ctx.addGroup();
- associateTargetCursor(subctx, ctx, tempRow, rowBuilder, expr);
- }
- break;
- case no_aggregate:
- {
- if (alreadyDoneFlag)
- {
- IHqlExpression * dataset = expr->queryChild(0);
- IHqlExpression * selSeq = querySelSeq(expr);
- IHqlExpression * transform = expr->queryChild(2);
- BuildCtx subctx(ctx);
- //Similar code to no_hqlproject. Should possibly try and do a similar mapping to the transforms
- //(but complicated by having merge/first)
- BoundRow * prevCursor = translator.resolveSelectorDataset(ctx, dataset);
- translator.rebindTableCursor(subctx, dataset, prevCursor, no_left, selSeq);
- rebindFilepositons(subctx, dataset, no_left, selSeq);
- Linked<BoundRow> tempRow;
- Linked<BoundRow> rowBuilder;
- buildTargetCursor(tempRow, rowBuilder, subctx, expr);
- translator.doBuildUserAggregateProcessTransform(subctx, rowBuilder, expr, transform, alreadyDoneFlag);
- subctx.addAssign(alreadyDoneFlag, queryBoolExpr(true));
- associateTargetCursor(subctx, ctx, tempRow, rowBuilder, expr);
- }
- }
- break;
- case no_newaggregate:
- {
- IHqlExpression * transform = expr->queryChild(2);
- if (isCompoundCount)
- {
- //This should really be special cased in a count() baseclass, but can't be bothered at the moment.
- IHqlExpression * rhs = transform->queryChild(0)->queryChild(1);
- IHqlExpression * filter = queryRealChild(rhs, 0);
- if (filter)
- {
- if (!returnIfFilterFails)
- translator.buildFilter(ctx, filter);
- else
- {
- OwnedHqlExpr test = getInverse(filter);
- translator.buildFilteredReturn(ctx, test, failedFilterValue);
- }
- }
- if (compoundCountVar)
- {
- OwnedHqlExpr inc = adjustValue(compoundCountVar, 1);
- ctx.addAssign(compoundCountVar, inc);
- }
- }
- else if (alreadyDoneFlag)
- {
- BuildCtx subctx(ctx);
- Linked<BoundRow> tempRow;
- Linked<BoundRow> rowBuilder;
- buildTargetCursor(tempRow, rowBuilder, subctx, expr);
- translator.doBuildAggregateProcessTransform(subctx, rowBuilder, expr, alreadyDoneFlag);
- subctx.addAssign(alreadyDoneFlag, queryBoolExpr(true));
- associateTargetCursor(subctx, ctx, tempRow, rowBuilder, expr);
- }
- }
- break;
- case no_usertable:
- case no_selectfields:
- UNIMPLEMENTED;
- break;
- case no_fetch:
- {
- BuildCtx subctx(ctx);
- Linked<BoundRow> tempRow;
- Linked<BoundRow> rowBuilder;
- buildTargetCursor(tempRow, rowBuilder, subctx, expr);
- // MORE - don't understand why this is required here but not in hqlproject above
- IHqlExpression * dataset = expr->queryChild(0);
- BoundRow * leftCursor;
- switch (getDatasetKind(tableExpr))
- {
- case no_csv:
- leftCursor = translator.bindCsvTableCursor(subctx, dataset, "Left", no_left, querySelSeq(expr), true, queryCsvTableEncoding(tableExpr));
- break;
- case no_xml:
- case no_json:
- leftCursor = translator.bindXmlTableCursor(subctx, dataset, "xmlLeft", no_left, querySelSeq(expr), true);
- break;
- default:
- leftCursor = translator.bindTableCursor(subctx, dataset, "left", no_left, querySelSeq(expr));
- break;
- }
- BoundRow * rightCursor = NULL;
- LinkedHqlExpr transform = expr->queryChild(3);
- if (!containsOnlyLeft(transform, true))
- {
- IHqlExpression * rhs = expr->queryChild(1);
- IHqlExpression * memoryRhsRecord = rhs->queryRecord();
- //RIGHT is the input data row.
- //a) In hthor the remote file is read locally, so RIGHT is a deserialized unmodified row.
- //b) in roxie it is serialized, and accessed in its serialized form.
- //c) in thor it is serialized, but also (inefficiently) deserialized, so use the deserialized form.
- //NOTE: Currently extractJoinFields either does nothing, or sends the entire row. Needless to say that could be improved -
- //and would mean the roxie/thor code required changing
- if (translator.targetRoxie())
- {
- OwnedHqlExpr serializedRhsRecord = getSerializedForm(memoryRhsRecord, diskAtom);
- OwnedHqlExpr serializedRhs = createDataset(no_null, LINK(serializedRhsRecord));
- rightCursor = translator.bindTableCursor(subctx, serializedRhs, "right", no_right, querySelSeq(expr));
- transform.setown(replaceMemorySelectorWithSerializedSelector(transform, memoryRhsRecord, no_right, querySelSeq(expr), diskAtom));
- }
- else
- {
- rightCursor = translator.bindTableCursor(subctx, rhs, "right", no_right, querySelSeq(expr));
- }
- }
- buildTransformFpos(subctx); // unusual, but this must occur after the row cursor is bound
- translator.associateSkipReturnMarker(subctx, failedFilterValue, NULL); // failedFilterValue already handles clearing the result
- translator.doTransform(subctx, transform, rowBuilder);
- subctx.removeAssociation(leftCursor);
- subctx.removeAssociation(rightCursor);
- associateTargetCursor(subctx, ctx, tempRow, rowBuilder, expr);
- }
- break;
- case no_compound_diskread:
- case no_compound_disknormalize:
- case no_compound_diskaggregate:
- case no_compound_diskcount:
- case no_compound_diskgroupaggregate:
- case no_compound_indexread:
- case no_compound_indexnormalize:
- case no_compound_indexaggregate:
- case no_compound_indexcount:
- case no_compound_indexgroupaggregate:
- case no_compound_childread:
- case no_compound_childnormalize:
- case no_compound_childaggregate:
- case no_compound_childcount:
- case no_compound_childgroupaggregate:
- case no_compound_fetch:
- case no_compound_selectnew:
- break;
- default:
- throwUnexpectedOp(op);
- }
- }
- void SourceBuilder::doBuildAggregateSelectIterator(BuildCtx & ctx, IHqlExpression * expr)
- {
- IHqlExpression * ds = expr->queryChild(0);
- if (isNewSelector(expr))
- buildTransformElements(ctx, ds, false);
- Owned<IHqlCppDatasetCursor> cursor = translator.createDatasetSelector(ctx, expr->queryNormalizedSelector());
- cursor->buildIterateLoop(ctx, false);
- }
- void SourceBuilder::processTransformSelect(BuildCtx & ctx, IHqlExpression * expr)
- {
- throwUnexpected();
- }
- void SourceBuilder::doBuildNormalizeIterators(BuildCtx & ctx, IHqlExpression * expr, bool isChildIterator)
- {
- HqlExprArray iterators;
- IHqlExpression * root = gatherSelectorLevels(iterators, expr);
- //MORE: transform also needs to be inside this iterctx
- BuildCtx iterctx(*globaliterctx);
- MemberFunction firstFunc(translator, instance->startctx);
- CursorArray cursors;
- if (isChildIterator)
- {
- assertex(!root);
- firstFunc.start("virtual bool first() override");
- }
- else
- {
- assertex(root);
- firstFunc.start("virtual bool first(const void * _src) override");
- bool isProjected = (root->queryNormalizedSelector() != tableExpr->queryNormalizedSelector());
- if (!isProjected)
- {
- iterctx.addQuotedLiteral("byte * src;");
- associateFilePositions(iterctx, "activity->fpp", "activity->src"); // in case no projection in first()
- translator.associateBlobHelper(iterctx, tableExpr, "fpp");
- firstFunc.ctx.addQuotedLiteral("src = (byte *)_src;");
- }
- else
- {
- firstFunc.ctx.addQuotedLiteral("byte * src = (byte *)_src;");
- }
- translator.associateBlobHelper(firstFunc.ctx, tableExpr, "fpp");
- BoundRow * tableCursor = translator.bindTableCursor(firstFunc.ctx, tableExpr, "src");
- associateFilePositions(firstFunc.ctx, "fpp", "src");
- TransformSequenceBuilder builder(translator, queryBoolExpr(false));
- builder.buildSequence(firstFunc.ctx, &iterctx, root);
- if (!isProjected)
- cursors.append(*LINK(tableCursor));
- else
- {
- BoundRow * match = translator.resolveSelectorDataset(firstFunc.ctx, root);
- assertex(match);
- cursors.append(*LINK(match));
- }
- }
- CompoundIteratorBuilder iterBuilder(translator, instance->nestedctx, iterctx);
- if (iterators.ordinality() == 1)
- {
- StringBuffer s, iterName, cursorName;
- iterBuilder.createSingleLevelIterator(iterName, cursorName, &iterators.item(0), cursors);
- firstFunc.ctx.addQuoted(s.clear().append("return (").append(cursorName).append(" = (byte *)").append(iterName).append(".first()) != 0;"));
- {
- BuildCtx nextctx(instance->startctx);
- nextctx.addQuotedFunction("virtual bool next() override");
- nextctx.addQuoted(s.clear().append("return (").append(cursorName).append(" = (byte *)").append(iterName).append(".next()) != 0;"));
- }
- }
- else
- {
- iterBuilder.buildCompoundIterator(instance->onlyEvalOnceContext(), iterators, cursors);
- firstFunc.ctx.addQuotedLiteral("return iter.first();");
- {
- BuildCtx nextctx(instance->startctx);
- nextctx.addQuotedFunction("virtual bool next() override");
- nextctx.addQuotedLiteral("return iter.next();");
- }
- }
- ForEachItemIn(i, cursors)
- ctx.associate(cursors.item(i));
- }
- void SourceBuilder::checkDependencies(BuildCtx & ctx, IHqlExpression * expr)
- {
- //Add dependency information
- Owned<ABoundActivity> bound = instance->getBoundActivity();
- translator.addFileDependency(nameExpr, bound);
- }
- void SourceBuilder::extractMonitors(IHqlExpression * ds, SharedHqlExpr & unkeyedFilter, HqlExprArray & conds)
- {
- ForEachItemIn(i, conds)
- {
- IHqlExpression * filter = &conds.item(i);
- #ifdef ALLOW_CHILD_ITERATORS_TO_CALL_CANMATCHANY
- if (isSourceInvariant(ds, filter)) // more actually isSourceInvariant.
- extendAndCondition(globalGuard, filter);
- else
- #endif
- appendFilter(unkeyedFilter, filter);
- }
- }
- void SourceBuilder::analyseGraph(IHqlExpression * expr)
- {
- analyse(expr);
- SourceFieldUsage * fieldUsage = translator.querySourceFieldUsage(tableExpr);
- if (fieldUsage && !fieldUsage->seenAll())
- {
- if (expr->queryNormalizedSelector() == tableExpr->queryNormalizedSelector())
- fieldUsage->noteAll();
- else
- gatherFieldUsage(fieldUsage, expr);
- }
- }
- void SourceBuilder::gatherFieldUsage(SourceFieldUsage * fieldUsage, IHqlExpression * expr)
- {
- for (;;)
- {
- if (expr->queryBody() == tableExpr->queryBody())
- return;
- if (fieldUsage->seenAll())
- return;
- IHqlExpression * ds = expr->queryChild(0);
- switch (expr->getOperator())
- {
- case no_fetch:
- {
- assertex(ds->queryBody() == tableExpr->queryBody());
- IHqlExpression * selSeq = querySelSeq(expr);
- OwnedHqlExpr left = createSelector(no_left, ds, selSeq);
- ::gatherFieldUsage(fieldUsage, expr, left);
- return;
- }
- }
- assertex(getNumChildTables(expr) == 1);
- if (ds->queryNormalizedSelector() == tableExpr->queryNormalizedSelector())
- gatherParentFieldUsage(fieldUsage, expr);
- expr = ds;
- }
- }
- inline bool useDescriptiveGraphLabel(ThorActivityKind kind)
- {
- switch (kind)
- {
- case TAKcsvfetch:
- case TAKxmlfetch:
- case TAKjsonfetch:
- case TAKfetch:
- return false;
- }
- return true;
- }
- static bool expandGraphLabel(ThorActivityKind kind)
- {
- switch (kind)
- {
- case TAKdiskread:
- case TAKcsvread:
- case TAKxmlread:
- case TAKjsonread:
- case TAKdiskcount:
- case TAKdiskexists:
- return true;
- default:
- return false;
- }
- }
- ABoundActivity * SourceBuilder::buildActivity(BuildCtx & ctx, IHqlExpression * expr, ThorActivityKind _activityKind, const char *kind, ABoundActivity *input)
- {
- activityKind = _activityKind;
- translator.gatherActiveCursors(ctx, parentCursors);
- bool isSpill = tableExpr && tableExpr->hasAttribute(_spill_Atom);
- useImplementationClass = translator.queryOptions().minimizeActivityClasses && translator.targetRoxie() && isSpill;
- Owned<ActivityInstance> localInstance = new ActivityInstance(translator, ctx, activityKind, expr, kind);
- if (useImplementationClass)
- localInstance->setImplementationClass(newMemorySpillReadArgId);
- if ((activityKind >= TAKdiskread) && (activityKind <= TAKdiskgroupaggregate))
- {
- IHqlExpression * seq = querySequence(tableExpr);
- translator.noteResultAccessed(ctx, seq, nameExpr);
- }
- instance = localInstance;
- StringBuffer graphLabel;
- graphLabel.append(getActivityText(activityKind));
- bool isFiltered = false;
- double filterLikelihood = 1.0;
- if (expandGraphLabel(activityKind))
- {
- graphLabel.clear();
- if (expr != tableExpr)
- {
- IHqlExpression * cur = expr;
- bool isProjected = false;
- for (;;)
- {
- switch (cur->getOperator())
- {
- case no_filter:
- if (isKnownLikelihood(filterLikelihood))
- {
- double likelihood = queryActivityLikelihood(cur);
- if (isKnownLikelihood(likelihood))
- // Combine the likelihood of the 2 filter conditions
- // N.B. this only works if the filter probability are independent
- filterLikelihood *= likelihood;
- else
- // One of the filter probability is unknown, so the overall probability is unknown
- setUnknownLikelihood(filterLikelihood);
- }
- isFiltered = true;
- break;
- case no_hqlproject:
- case no_newusertable:
- case no_transformascii:
- case no_transformebcdic:
- if (!cur->hasAttribute(_internal_Atom))
- isProjected = true;
- break;
- case no_table:
- cur = NULL;
- break;
- }
- if (!cur)
- break;
- cur = cur->queryChild(0);
- }
- if (isFiltered)
- {
- graphLabel.append("Filtered\n");
- }
- if (isProjected)
- graphLabel.append("Projected\n");
- }
- if ((translator.getTargetClusterType() == RoxieCluster) && isSpill)
- graphLabel.append("Read");
- else if (isExplicitExists() && (activityKind == TAKdiskcount))
- graphLabel.append("Disk Exists");
- else
- graphLabel.append(getActivityText(activityKind));
- }
- if (isExplicitExists())
- {
- if (activityKind == TAKindexcount)
- graphLabel.clear().append("Index Exists");
- }
- if (useDescriptiveGraphLabel(activityKind))
- {
- StringBuffer eclChunk;
- bool isLimited = false;
- bool isStepped = false;
- IHqlExpression * cur = expr;
- instance->graphEclText.clear();
- for (;;)
- {
- eclChunk.clear();
- switch (cur->getOperator())
- {
- case no_hqlproject:
- case no_newusertable:
- case no_transformascii:
- case no_transformebcdic:
- break;
- case no_filter:
- toECL(cur->queryBody(), eclChunk, false, true);
- break;
- case no_table:
- case no_newkeyindex:
- case no_select:
- toECL(cur->queryBody(), eclChunk, false, true);
- cur = NULL;
- break;
- case no_stepped:
- isStepped = true;
- break;
- case no_limit:
- case no_keyedlimit:
- isLimited = true;
- break;
- default:
- if (getNumChildTables(cur) == 0)
- {
- toECL(cur->queryBody(), eclChunk, false, true);
- cur = NULL;
- }
- break;
- }
- instance->graphEclText.insert(0, eclChunk);
- if (!cur)
- break;
- cur = cur->queryChild(0);
- }
- if (isLimited)
- graphLabel.insert(0, "Limited\n");
- if (isStepped)
- graphLabel.insert(0, "Stepped\n");
- if (localInstance->isLocal && !isSpill)
- graphLabel.insert(0, "Local ");
- }
- if (nameExpr && nameExpr->queryValue())
- {
- if (isSpill)
- graphLabel.append("\nSpill");
- else
- {
- StringBuffer filename;
- //Call getStringValue() rather than generateECL() to avoid 't quote \ etc. in the string
- getStringValue(filename.append("'"), nameExpr).append("'");
- const char * coloncolon = strstr(filename, "::");
- if (coloncolon)
- {
- for (;;)
- {
- const char * next = strstr(coloncolon+2, "::");
- if (!next)
- break;
- coloncolon = next;
- }
- graphLabel.newline().append("'...").append(coloncolon);
- }
- else
- graphLabel.newline().append(filename);
- }
- }
- instance->graphLabel.set(graphLabel.str());
- translator.buildActivityFramework(localInstance);
- translator.buildInstancePrefix(localInstance);
- analyseGraph(expr);
- if (!useImplementationClass)
- {
- buildMembers(expr);
- buildTransform(expr);
- buildFlagsMember(expr);
- if (tableExpr && (activityKind < TAKchildread || activityKind > TAKchildthroughnormalize))
- {
- switch (activityKind)
- {
- case TAKindexread:
- case TAKindexnormalize:
- case TAKindexaggregate:
- case TAKindexcount:
- case TAKindexgroupaggregate:
- case TAKindexexists:
- {
- LinkedHqlExpr indexExpr = queryAttributeChild(tableExpr, _original_Atom, 0);
- OwnedHqlExpr serializedRecord;
- unsigned numPayload = numPayloadFields(indexExpr);
- if (numPayload)
- serializedRecord.setown(notePayloadFields(indexExpr->queryRecord(), numPayload));
- else
- serializedRecord.set(indexExpr->queryRecord());
- serializedRecord.setown(getSerializedForm(serializedRecord, diskAtom));
- bool hasFilePosition = getBoolAttribute(indexExpr, filepositionAtom, true);
- serializedRecord.setown(createMetadataIndexRecord(serializedRecord, hasFilePosition));
- translator.buildMetaMember(instance->classctx, serializedRecord, false, "queryDiskRecordSize");
- translator.buildMetaMember(instance->classctx, serializedRecord, false, "queryProjectedDiskRecordSize");
- break;
- }
- default:
- if (fieldInfo.hasVirtualsOrDeserialize())
- {
- OwnedHqlExpr diskTable = createDataset(no_anon, LINK(physicalRecord));
- translator.buildMetaMember(instance->classctx, diskTable, false, "queryDiskRecordSize");
- if (activityKind != TAKpiperead)
- translator.buildMetaMember(instance->classctx, diskTable, false, "queryProjectedDiskRecordSize");
- }
- else
- {
- translator.buildMetaMember(instance->classctx, tableExpr, isGrouped(tableExpr), "queryDiskRecordSize");
- if (activityKind != TAKpiperead)
- translator.buildMetaMember(instance->classctx, tableExpr, isGrouped(tableExpr), "queryProjectedDiskRecordSize");
- }
- }
- }
- }
- else
- {
- assertex(!hasDynamic(tableExpr));
- bool matched = translator.registerGlobalUsage(nameExpr);
- if (!matched)
- {
- StringBuffer spillName;
- getExprECL(nameExpr, spillName);
- if (translator.queryOptions().allowThroughSpill)
- throwError1(HQLERR_ReadSpillBeforeWriteFix, spillName.str());
- else
- throwError1(HQLERR_ReadSpillBeforeWrite, spillName.str());
- }
- translator.addFilenameConstructorParameter(*instance, "getFileName", nameExpr);
- }
- if (steppedExpr)
- buildSteppedHelpers();
- if (translator.targetRoxie())
- instance->addAttributeBool("_isSpill", isSpill);
- else if (needToCallTransform || transformCanFilter)
- instance->addAttributeBool("_isTransformSpill", isSpill);
- else
- instance->addAttributeBool("_isSpill", isSpill);
- if (isFiltered)
- {
- if (isKnownLikelihood(filterLikelihood))
- {
- StringBuffer text;
- filterLikelihood *= 100;
- text.setf("%3.2f%%", filterLikelihood);
- instance->addAttribute("matchLikelihood", text);
- }
- }
- IHqlExpression * spillReason = tableExpr ? queryAttributeChild(tableExpr, _spillReason_Atom, 0) : NULL;
- if (spillReason && !translator.queryOptions().obfuscateOutput)
- {
- StringBuffer text;
- getStringValue(text, spillReason);
- instance->addAttribute("spillReason", text.str());
- }
- if (tableExpr)
- instance->addSignedAttribute(tableExpr->queryAttribute(_signed_Atom));
- checkDependencies(ctx, expr);
- translator.buildInstanceSuffix(localInstance);
- if (input)
- translator.buildConnectInputOutput(ctx, localInstance, input, 0, 0);
- instance = NULL;
- return localInstance->getBoundActivity();
- }
- void SourceBuilder::buildKeyedLimitHelper(IHqlExpression * self)
- {
- if (keyedLimitExpr)
- {
- IHqlExpression * limitValue = keyedLimitExpr->queryChild(1);
- {
- MemberFunction func(translator, instance->startctx, "virtual unsigned __int64 getKeyedLimit() override");
- translator.buildReturn(func.ctx, limitValue);
- if (isZero(limitValue))
- translator.WARNING(CategoryUnusual, HQLWRN_KeyedLimitIsZero);
- }
- LinkedHqlExpr fail = keyedLimitExpr->queryChild(2);
- if (!fail || fail->isAttribute())
- fail.setown(translator.createFailAction("Keyed limit exceeded", limitValue, NULL, instance->activityId));
- {
- MemberFunction func(translator, instance->startctx, "virtual void onKeyedLimitExceeded() override");
- translator.buildStmt(func.ctx, fail);
- }
- IHqlExpression * transform = queryAttributeChild(keyedLimitExpr, onFailAtom, 0);
- if (transform)
- {
- MemberFunction func(translator, instance->startctx, "virtual size32_t transformOnKeyedLimitExceeded(ARowBuilder & crSelf) override");
- translator.ensureRowAllocated(func.ctx, "crSelf");
- translator.buildTransformBody(func.ctx, transform, NULL, NULL, self, NULL);
- }
- }
- }
- void SourceBuilder::buildSteppedHelpers()
- {
- StringBuffer steppedFlags;
- IHqlExpression * priority = steppedExpr->queryAttribute(priorityAtom);
- if (priority)
- {
- steppedFlags.append("|SSFhaspriority");
- translator.doBuildFunction(instance->startctx, doubleType, "getPriority", priority->queryChild(0));
- }
- IHqlExpression * prefetch = steppedExpr->queryAttribute(prefetchAtom);
- if (prefetch)
- {
- steppedFlags.append("|SSFhasprefetch");
- translator.doBuildUnsignedFunction(instance->startctx, "getPrefetchSize", prefetch->queryChild(0));
- }
- IHqlExpression * filtered = steppedExpr->queryAttribute(filteredAtom);
- if (filtered)
- steppedFlags.append("|SSFalwaysfilter");
- if (steppedFlags.length())
- translator.doBuildUnsignedFunction(instance->classctx, "getSteppedFlags", steppedFlags.str()+1);
- }
- void SourceBuilder::assignLocalExtract(BuildCtx & ctx, ParentExtract * extractBuilder, IHqlExpression * dataset, const char * argName)
- {
- if (extractBuilder)
- {
- StringBuffer s;
- s.append("byte * ");
- translator.generateExprCpp(s, extractBuilder->queryExtractName());
- s.append(" = (byte *) ").append(argName).append(";");
- ctx.addQuoted(s);
- }
- else
- {
- StringBuffer s;
- ctx.addQuoted(s.append("unsigned char * left = (unsigned char *) ").append(argName).append(";"));
- translator.bindTableCursor(ctx, dataset, "left");
- }
- }
- void SourceBuilder::buildGroupAggregateHashHelper(ParentExtract * extractBuilder, IHqlExpression * dataset, IHqlExpression * fields)
- {
- instance->classctx.addQuotedLiteral("virtual IHash * queryHash() override { return &hash; }");
- BuildCtx classctx(instance->nestedctx);
- IHqlStmt * classStmt = translator.beginNestedClass(classctx, "hash", "IHash", NULL, extractBuilder);
- {
- MemberFunction func(translator, classctx, "virtual unsigned hash(const void * _self) override");
- assignLocalExtract(func.ctx, extractBuilder, dataset, "_self");
- OwnedHqlExpr hash = createValue(no_hash32, LINK(unsignedType), LINK(fields), createAttribute(internalAtom));
- translator.buildReturn(func.ctx, hash);
- }
- translator.endNestedClass(classStmt);
- }
- void SourceBuilder::buildGroupAggregateCompareHelper(ParentExtract * extractBuilder, IHqlExpression * aggregate, HqlExprArray & recordFields, HqlExprArray & aggregateFields)
- {
- //Special case comparison of IF(<global-bool>, value, some-constant), so that the order expression only compares if <global-bool> is true
- //specifically to improve david's BIRPs code.
- HqlExprArray optimizedLeft, optimizedRight;
- ForEachItemIn(i, recordFields)
- {
- IHqlExpression & curLeft = recordFields.item(i);
- IHqlExpression & curRight = aggregateFields.item(i);
- IHqlExpression * newLeft = NULL;
- IHqlExpression * newRight = NULL;
- if (curLeft.getOperator() == no_if)
- {
- IHqlExpression * cond = curLeft.queryChild(0);
- if (isIndependentOfScope(cond))
- {
- IHqlExpression * elseValue = curLeft.queryChild(2);
- if (elseValue->getOperator() == no_constant)
- {
- newLeft = LINK(&curLeft);
- newRight = createValue(no_if, curRight.getType(), LINK(cond), LINK(&curRight), LINK(elseValue));
- }
- }
- }
- if (newLeft)
- {
- optimizedLeft.append(*newLeft);
- optimizedRight.append(*newRight);
- }
- else
- {
- optimizedLeft.append(OLINK(curLeft));
- optimizedRight.append(OLINK(curRight));
- }
- }
- OwnedHqlExpr leftList = createSortList(optimizedLeft);
- DatasetReference datasetRight(aggregate, no_activetable, NULL);
- OwnedHqlExpr selSeq = createDummySelectorSequence();
- OwnedHqlExpr rightList = createSortList(optimizedRight);
- OwnedHqlExpr rightSelect = datasetRight.getSelector(no_right, selSeq);
- OwnedHqlExpr rightResolved = datasetRight.mapCompound(rightList, rightSelect);
- OwnedHqlExpr order = createValue(no_order, makeIntType(sizeof(signed), true), LINK(leftList), LINK(rightResolved));
- //Now generate the nested class
- instance->classctx.addQuotedLiteral("virtual ICompare * queryCompareRowElement() override { return &compareRowElement; }");
- BuildCtx classctx(instance->nestedctx);
- IHqlStmt * classStmt = translator.beginNestedClass(classctx, "compareRowElement", "ICompare", NULL, extractBuilder);
- {
- MemberFunction func(translator, classctx, "virtual int docompare(const void * _left, const void * _right) const override");
- assignLocalExtract(func.ctx, extractBuilder, aggregate->queryChild(0), "_left");
- func.ctx.addQuotedLiteral("const unsigned char * right = (const unsigned char *) _right;");
- func.ctx.associateExpr(constantMemberMarkerExpr, constantMemberMarkerExpr);
- translator.bindTableCursor(func.ctx, aggregate, "right", no_right, selSeq);
- translator.doBuildReturnCompare(func.ctx, order, no_eq, false, false);
- }
- translator.endNestedClass(classStmt);
- }
- void SourceBuilder::buildGroupAggregateProcessHelper(ParentExtract * extractBuilder, IHqlExpression * aggregate, const char * name, bool doneAny)
- {
- StringBuffer proto;
- IHqlExpression * dataset = aggregate->queryChild(0);
- IHqlExpression * tgtRecord = aggregate->queryChild(1);
- OwnedHqlExpr resultDataset = createDataset(no_anon, LINK(tgtRecord));
- proto.append("virtual size32_t ").append(name).append("(ARowBuilder & crSelf, const void * _src) override");
- MemberFunction validateFunc(translator, instance->nestedctx, proto, MFdynamicproto);
- translator.ensureRowAllocated(validateFunc.ctx, "crSelf");
- assignLocalExtract(validateFunc.ctx, extractBuilder, dataset, "_src");
- BoundRow * selfRow = translator.bindSelf(validateFunc.ctx, resultDataset, "crSelf");
- if (extractBuilder)
- {
- MemberEvalContext * evalContext = new MemberEvalContext(translator, extractBuilder, translator.queryEvalContext(validateFunc.ctx), validateFunc.ctx);
- validateFunc.ctx.associateOwn(*evalContext);
- evalContext->initContext();
- }
- if (aggregate->getOperator() == no_aggregate)
- {
- //It is inefficient to call processUserAggregateTransform() twice, but this is an unusual construct, so leave as it
- //is for the moment.
- OwnedHqlExpr firstTransform;
- OwnedHqlExpr nextTransform;
- translator.processUserAggregateTransform(aggregate, aggregate->queryChild(2), firstTransform, nextTransform);
- IHqlExpression * transform = doneAny ? nextTransform : firstTransform;
- OwnedHqlExpr left = createSelector(no_left, dataset, querySelSeq(aggregate));
- OwnedHqlExpr mappedTransform = replaceSelector(transform, left, dataset);
- translator.doBuildUserAggregateProcessTransform(validateFunc.ctx, selfRow, aggregate, mappedTransform, queryBoolExpr(doneAny));
- }
- else
- translator.doBuildAggregateProcessTransform(validateFunc.ctx, selfRow, aggregate, queryBoolExpr(doneAny));
- translator.buildReturnRecordSize(validateFunc.ctx, selfRow);
- }
- void SourceBuilder::buildGroupAggregateHelpers(ParentExtract * extractBuilder, IHqlExpression * aggregate)
- {
- IHqlExpression * dataset = aggregate->queryChild(0);
- LinkedHqlExpr transform = aggregate->queryChild(2);
- LinkedHqlExpr grouping = aggregate->queryChild(3);
- if (aggregate->getOperator() == no_aggregate)
- {
- OwnedHqlExpr left = createSelector(no_left, dataset, querySelSeq(aggregate));
- grouping.setown(replaceSelector(grouping, left, dataset));
- transform.setown(replaceSelector(transform, left, dataset));
- }
- HqlExprArray recordFields, aggregateFields;
- grouping->unwindList(recordFields, no_sortlist);
- getMappedFields(aggregateFields, transform, recordFields, queryActiveTableSelector());
- OwnedHqlExpr allRecordFields = createValueSafe(no_sortlist, makeSortListType(NULL), recordFields);
- OwnedHqlExpr allAggregateFields = createValueSafe(no_sortlist, makeSortListType(NULL), aggregateFields);
- //virtual size32_t processFirst(void * target, const void * src)
- buildGroupAggregateProcessHelper(extractBuilder, aggregate, "processFirst", false);
- //virtual size32_t processNext(void * target, const void * src)
- buildGroupAggregateProcessHelper(extractBuilder, aggregate, "processNext", true);
- //virtual IHash * queryHash()
- buildGroupAggregateHashHelper(extractBuilder, dataset, allRecordFields);
- //virtual ICompare * queryCompareElements()
- DatasetReference outRef(aggregate, no_activetable, NULL);
- translator.buildCompareMember(instance->nestedctx, "CompareElements", allAggregateFields, outRef); // compare transformed elements
- //virtual ICompare * queryCompareRowElement()
- buildGroupAggregateCompareHelper(extractBuilder, aggregate, recordFields, aggregateFields);
- //virtual IHash * queryHashElement()
- translator.buildHashOfExprsClass(instance->nestedctx, "HashElement", allAggregateFields, outRef, true);
- }
- IHqlExpression * SourceBuilder::ensureAggregateGroupingAliased(IHqlExpression * aggregate)
- {
- IHqlExpression * dataset = aggregate->queryChild(0);
- IHqlExpression * grouping = aggregate->queryChild(3);
- //Force complex grouping fields into aliases to reduce processing...
- HqlMapTransformer transformer;
- transformer.setMapping(dataset, dataset);
- ForEachChild(i, grouping)
- {
- IHqlExpression * cur = grouping->queryChild(i);
- if (translator.requiresTemp(instance->nestedctx, cur, true) && (cur->getOperator() != no_alias))
- {
- OwnedHqlExpr alias = createAliasOwn(LINK(cur), createAttribute(internalAtom));
- transformer.setMapping(cur, alias);
- }
- }
- return transformer.transformRoot(aggregate);
- }
- void SourceBuilder::buildGlobalGroupAggregateHelpers(IHqlExpression * expr)
- {
- IHqlExpression * aggregate = expr->queryChild(0);
- node_operator op = aggregate->getOperator();
- assertex(op == no_newaggregate || op == no_aggregate);
- StringBuffer s;
- //virtual size32_t clearAggregate(void * self) = 0;
- translator.doBuildAggregateClearFunc(instance->startctx, aggregate);
- //virtual size32_t mergeAggregate(ARowBuilder & crSelf, const void * src) = 0; //only call if transform called at least once on src.
- translator.doBuildAggregateMergeFunc(instance->startctx, aggregate, requiresOrderedMerge);
- //virtual void processRow(void * self, const void * src) = 0;
- {
- BuildCtx rowctx(instance->startctx);
- rowctx.addQuotedFunction("virtual void processRow(const void * src, IHThorGroupAggregateCallback * callback) override");
- rowctx.addQuotedLiteral("doProcessRow((byte *)src, callback);");
- }
- //virtual void processRows(void * self, size32_t srcLen, const void * src) = 0;
- {
- OwnedHqlExpr newTableExpr = LINK(tableExpr);
- if (isKey(tableExpr))
- {
- IHqlExpression * record = tableExpr->queryRecord();
- OwnedHqlExpr newRecord = removeChild(record, record->numChildren()-1);
- newTableExpr.setown(replaceChild(tableExpr, 1, newRecord));
- }
- MemberFunction func(translator, instance->startctx, "virtual void processRows(size32_t srcLen, const void * _left, IHThorGroupAggregateCallback * callback) override");
- func.ctx.addQuotedLiteral("unsigned char * left = (unsigned char *)_left;");
- OwnedHqlExpr ds = createVariable("left", makeReferenceModifier(newTableExpr->getType()));
- OwnedHqlExpr len = createVariable("srcLen", LINK(sizetType));
- OwnedHqlExpr fullDs = createTranslated(ds, len);
- BoundRow * curRow = translator.buildDatasetIterate(func.ctx, fullDs, false);
- s.clear().append("doProcessRow(");
- translator.generateExprCpp(s, curRow->queryBound());
- s.append(", callback);");
- func.ctx.addQuoted(s);
- }
- }
- void SourceBuilder::buildGroupingMonitors(IHqlExpression * expr, MonitorExtractor & monitors)
- {
- IHqlExpression * aggregate = expr->queryChild(0);
- node_operator op = aggregate->getOperator();
- assertex(op == no_newaggregate || op == no_aggregate);
- IHqlExpression * dataset = aggregate->queryChild(0);
- IHqlExpression * grouping = aggregate->queryChild(3);
- //virtual void createGroupSegmentMonitors(IIndexReadContext *ctx) = 0;
- MemberFunction func(translator, instance->startctx, "virtual bool createGroupSegmentMonitors(IIndexReadContext * irc) override");
- monitorsForGrouping = true;
- if (op == no_newaggregate)
- translator.bindTableCursor(func.ctx, dataset, "_dummy");
- else
- translator.bindTableCursor(func.ctx, dataset, "_dummy", no_left, querySelSeq(aggregate));
- unsigned maxOffset = 0;
- ForEachChild(i, grouping)
- {
- unsigned nextOffset = 0;
- if (!monitors.createGroupingMonitor(func.ctx, "irc", grouping->queryChild(i), nextOffset))
- {
- monitorsForGrouping = false;
- func.setIncluded(false);
- break;
- }
- if (maxOffset < nextOffset)
- maxOffset = nextOffset;
- }
- if (monitorsForGrouping)
- func.ctx.addReturn(queryBoolExpr(true));
- if (monitorsForGrouping)
- translator.doBuildUnsignedFunction(instance->classctx, "getGroupSegmentMonitorsSize", maxOffset);
- }
- void SourceBuilder::buildAggregateHelpers(IHqlExpression * expr)
- {
- IHqlExpression * aggregate = expr->queryChild(0);
- node_operator op = aggregate->getOperator();
- assertex(op == no_newaggregate || op == no_aggregate);
- StringBuffer s;
- alreadyDoneFlag.setown(instance->startctx.getTempDeclare(queryBoolType(), NULL));
- instance->onstartctx.addAssign(alreadyDoneFlag, queryBoolExpr(false));
- //virtual bool processedAnyRows() = 0;
- translator.doBuildBoolFunction(instance->startctx, "processedAnyRows", alreadyDoneFlag);
- //virtual size32_t clearAggregate(ARowBuilder & crSelf) = 0;
- translator.doBuildAggregateClearFunc(instance->startctx, aggregate);
- //virtual size32_t mergeAggregate(ARowBuilder & crSelf, const void * src) = 0; //only call if transform called at least once on src.
- translator.doBuildAggregateMergeFunc(instance->startctx, aggregate, requiresOrderedMerge);
- }
- void SourceBuilder::buildCountHelpers(IHqlExpression * expr, bool allowMultiple)
- {
- StringBuffer s;
- //---- virtual bool hasFilter() { return <bool>; } ----
- if (transformCanFilter||isNormalize)
- translator.doBuildBoolFunction(instance->classctx, "hasFilter", true);
- if (allowMultiple)
- {
- bool isExists = hasExistChoosenLimit();
- OwnedHqlExpr one = getSizetConstant(1);
- if (transformCanFilter||isNormalize)
- {
- //virtual size32_t numValid(const void * src) = 0;
- {
- BuildCtx rowctx(instance->startctx);
- rowctx.addQuotedFunction("virtual size32_t numValid(const void * src) override");
- rowctx.addQuotedLiteral("return valid((byte *)src);");
- }
- //virtual size32_t numValid(size32_t srcLen, const void * src);
- {
- MemberFunction func(translator, instance->startctx, "virtual size32_t numValid(size32_t srcLen, const void * _src) override");
- func.ctx.addQuotedLiteral("unsigned char * src = (unsigned char *)_src;");
- OwnedHqlExpr ds = createVariable("src", makeReferenceModifier(tableExpr->getType()));
- OwnedHqlExpr len = createVariable("srcLen", LINK(sizetType));
- OwnedHqlExpr fullDs = createTranslated(ds, len);
- if (isExists)
- {
- BuildCtx iterctx(func.ctx);
- BoundRow * curRow = translator.buildDatasetIterate(iterctx, fullDs, false);
- s.clear().append("if (valid(");
- translator.generateExprCpp(s, curRow->queryBound());
- s.append("))");
- iterctx.addQuotedCompound(s, nullptr);
- iterctx.addReturn(one);
- func.ctx.addQuotedLiteral("return 0;");
- }
- else
- {
- func.ctx.addQuotedLiteral("size32_t cnt = 0;");
- BuildCtx iterctx(func.ctx);
- BoundRow * curRow = translator.buildDatasetIterate(iterctx, fullDs, false);
- s.clear().append("cnt += valid(");
- translator.generateExprCpp(s, curRow->queryBound());
- s.append(");");
- iterctx.addQuoted(s);
- func.ctx.addQuotedLiteral("return cnt;");
- }
- }
- }
- else
- {
- //virtual size32_t numValid(size32_t srcLen, const void * src);
- MemberFunction func(translator, instance->startctx, "virtual size32_t numValid(size32_t srcLen, const void * _src) override");
- if (isExists)
- func.ctx.addReturn(one);
- else
- {
- func.ctx.addQuotedLiteral("unsigned char * src = (unsigned char *)_src;");
- CHqlBoundExpr bound;
- bound.length.setown(createVariable("srcLen", LINK(sizetType)));
- bound.expr.setown(createVariable("src", makeReferenceModifier(tableExpr->getType())));
- OwnedHqlExpr count = translator.getBoundCount(bound);
- func.ctx.addReturn(count);
- }
- }
- }
- }
- void SourceBuilder::buildNormalizeHelpers(IHqlExpression * expr)
- {
- }
- void SourceBuilder::buildGroupAggregateTransformBody(BuildCtx & transformCtx, IHqlExpression * expr, bool useExtract, bool bindInputRow)
- {
- buildTransformBody(transformCtx, expr, false, false, bindInputRow);
- IHqlExpression * aggregate = expr->queryChild(0);
- OwnedHqlExpr mappedAggregate = ensureAggregateGroupingAliased(aggregate);
- Owned<ParentExtract> extractBuilder;
- if (useExtract || (aggregate != mappedAggregate))
- {
- extractBuilder.setown(translator.createExtractBuilder(transformCtx, PETcallback, NULL, GraphCoLocal, true));
- if (!translator.queryOptions().serializeRowsetInExtract)
- extractBuilder->setAllowDestructor();
- translator.beginExtract(transformCtx, extractBuilder);
- buildGroupAggregateHelpers(extractBuilder, mappedAggregate);
- translator.endExtract(transformCtx, extractBuilder);
- }
- else
- buildGroupAggregateHelpers(NULL, mappedAggregate);
- HqlExprArray args;
- args.append(*createVariable("callback", makeBoolType()));
- if (extractBuilder)
- {
- CHqlBoundExpr boundExtract;
- extractBuilder->endCreateExtract(boundExtract);
- args.append(*boundExtract.getTranslatedExpr());
- }
- else
- {
- BoundRow * match = translator.resolveDatasetRequired(transformCtx, aggregate->queryChild(0));
- Owned<ITypeInfo> rowType = makeReferenceModifier(makeRowType(queryNullRecord()->getType()));
- OwnedHqlExpr rowAddr = getPointer(match->queryBound());
- OwnedHqlExpr castBound = createValue(no_typetransfer, LINK(rowType), LINK(rowAddr));
- args.append(*createTranslated(castBound));
- }
- OwnedHqlExpr call = translator.bindFunctionCall(addAggregateRowId, args);
- translator.buildStmt(transformCtx, call);
- }
- void SourceBuilder::gatherVirtualFields(bool ignoreVirtuals, bool ensureSerialized)
- {
- IHqlExpression * record = tableExpr->queryRecord();
- fieldInfo.gatherVirtualFields(record, ignoreVirtuals, ensureSerialized);
- if (fieldInfo.hasVirtuals())
- physicalRecord.setown(fieldInfo.createPhysicalRecord());
- else
- physicalRecord.set(record);
- }
- /*
- interface ICompoundSourceSteppingMeta : extends ISteppingMeta
- {
- virtual ISteppingMeta * queryRawSteppingMeta() = 0;
- virtual ISteppingMeta * queryProjectedSteppingMeta() = 0; // if null no projection takes place
- virtual void mapOutputToIInput(void * originalRow, const void * projectedRow, unsigned firstField, unsigned numFields) = 0;
- };
- */
- bool SourceBuilder::containsStepping(IHqlExpression * expr)
- {
- for (;;)
- {
- switch (expr->getOperator())
- {
- case no_stepped:
- return true;
- default:
- {
- if (expr->queryBody() == tableExpr)
- return false;
- unsigned numChildren = getNumChildTables(expr);
- if (numChildren == 0)
- return false;
- assertex(numChildren == 1);
- }
- }
- expr = expr->queryChild(0);
- }
- }
- void SourceBuilder::gatherSteppingMeta(IHqlExpression * expr, SteppingFieldSelection & outputStepping, SteppingFieldSelection & rawStepping)
- {
- for (;;)
- {
- switch (expr->getOperator())
- {
- case no_newusertable:
- case no_hqlproject:
- if (rawStepping.exists())
- {
- rawStepping.expandTransform(expr);
- }
- else
- {
- gatherSteppingMeta(expr->queryChild(0), outputStepping, rawStepping);
- outputStepping.clear();
- return;
- //throwError(HQLERR_CantProjectStepping);
- //gatherSteppingMeta(expr->queryChild(0), outputStepping, rawStepping);
- //apply projection to the output fields and inverse to the raw fields.
- }
- break;
- case no_stepped:
- {
- outputStepping.setStepping(expr);
- rawStepping.setStepping(expr);
- break;
- }
- default:
- {
- if (expr->queryBody() == tableExpr)
- return;
- unsigned numChildren = getNumChildTables(expr);
- if (numChildren == 0)
- return;
- assertex(numChildren == 1);
- }
- }
- expr = expr->queryChild(0);
- }
- }
- void SourceBuilder::gatherSteppingMeta(IHqlExpression * expr, SourceSteppingInfo & info)
- {
- if (!steppedExpr)
- return;
- gatherSteppingMeta(expr, info.outputStepping, info.rawSteppingProject);
- if (info.rawSteppingProject.exists())
- info.extractRaw();
- }
- //-----------------------------------------------------------------------------------------------
- //-- Disk file processing
- //-----------------------------------------------------------------------------------------------
- class DiskReadBuilderBase : public SourceBuilder
- {
- public:
- DiskReadBuilderBase(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr)
- : SourceBuilder(_translator, _tableExpr, _nameExpr), monitors(_tableExpr, _translator, 0, true)
- {
- fpos.setown(getFilepos(tableExpr, false));
- lfpos.setown(getFilepos(tableExpr, true));
- logicalFilenameMarker.setown(getFileLogicalName(tableExpr));
- mode = tableExpr->queryChild(2);
- modeOp = mode->getOperator();
- includeFormatCrc = (modeOp != no_csv && modeOp != no_pipe);
- }
- virtual void buildMembers(IHqlExpression * expr);
- virtual void buildTransformFpos(BuildCtx & transformCtx);
- virtual void extractMonitors(IHqlExpression * ds, SharedHqlExpr & unkeyedFilter, HqlExprArray & conds);
- protected:
- virtual void buildFlagsMember(IHqlExpression * expr);
- protected:
- MonitorExtractor monitors;
- OwnedHqlExpr globalGuard;
- IHqlExpression * mode;
- node_operator modeOp;
- bool includeFormatCrc;
- };
- void DiskReadBuilderBase::buildMembers(IHqlExpression * expr)
- {
- StringBuffer s;
- //Process any KEYED() information
- if (monitors.isFiltered())
- {
- MemberFunction func(translator, instance->startctx, "virtual void createSegmentMonitors(IIndexReadContext *irc) override");
- monitors.buildSegments(func.ctx, "irc", true);
- }
- instance->addAttributeBool("_isKeyed", monitors.isFiltered());
- //---- virtual unsigned getFlags()
- instance->addAttributeBool("preload", isPreloaded);
- bool matched = translator.registerGlobalUsage(tableExpr->queryChild(0));
- if (translator.getTargetClusterType() == RoxieCluster)
- {
- instance->addAttributeBool("_isOpt", tableExpr->hasAttribute(optAtom));
- instance->addAttributeBool("_isSpillGlobal", tableExpr->hasAttribute(jobTempAtom));
- if (tableExpr->hasAttribute(jobTempAtom) && !matched)
- throwUnexpected();
- }
- if (!matched && expr->hasAttribute(_spill_Atom))
- {
- StringBuffer spillName;
- getExprECL(tableExpr->queryChild(0), spillName);
- if (translator.queryOptions().allowThroughSpill)
- throwError1(HQLERR_ReadSpillBeforeWriteFix, spillName.str());
- else
- throwError1(HQLERR_ReadSpillBeforeWrite, spillName.str());
- }
- //---- virtual bool canMatchAny() { return <value>; } ----
- LinkedHqlExpr guard = globalGuard.get();
- extendConditionOwn(guard, no_and, LINK(monitors.queryGlobalGuard()));
- if (guard)
- translator.doBuildBoolFunction(instance->startctx, "canMatchAny", guard);
- translator.buildEncryptHelper(instance->startctx, tableExpr->queryAttribute(encryptAtom));
- //---- preloadSize is passed via the xgmml, not via a member
- if (preloadSize)
- {
- instance->addAttributeInt("_preloadSize", preloadSize->queryValue()->getIntValue());
- }
- if (includeFormatCrc)
- {
- //Spill files can still have virtual attributes in their physical records => remove them.
- OwnedHqlExpr noVirtualRecord = removeVirtualAttributes(physicalRecord);
- translator.buildFormatCrcFunction(instance->classctx, "getFormatCrc", false, noVirtualRecord, NULL, 0);
- }
- buildLimits(instance->startctx, expr, instance->activityId);
- buildKeyedLimitHelper(expr);
- //Note the helper base class contains code like the following
- //IThorDiskCallback * fpp;");
- //virtual void setCallback(IThorDiskCallback * _tc) { fpp = _tc; }");
- }
- void DiskReadBuilderBase::buildFlagsMember(IHqlExpression * expr)
- {
- StringBuffer flags;
- if (tableExpr->hasAttribute(_spill_Atom)) flags.append("|TDXtemporary");
- if (tableExpr->hasAttribute(jobTempAtom)) flags.append("|TDXjobtemp");
- if (tableExpr->hasAttribute(groupedAtom)) flags.append("|TDXgrouped");
- if (tableExpr->hasAttribute(__compressed__Atom)) flags.append("|TDXcompress");
- if (tableExpr->hasAttribute(unsortedAtom)) flags.append("|TDRunsorted");
- if (tableExpr->hasAttribute(optAtom)) flags.append("|TDRoptional");
- if (tableExpr->hasAttribute(_workflowPersist_Atom)) flags.append("|TDXupdateaccessed");
- if (isPreloaded) flags.append("|TDRpreload");
- if (monitors.isFiltered()) flags.append("|TDRkeyed");
- if (limitExpr)
- {
- if (limitExpr->hasAttribute(onFailAtom))
- flags.append("|TDRlimitcreates");
- else if (limitExpr->hasAttribute(skipAtom))
- flags.append("|TDRlimitskips");
- }
- if (keyedLimitExpr)
- {
- if (keyedLimitExpr->hasAttribute(onFailAtom))
- flags.append("|TDRkeyedlimitcreates|TDRcountkeyedlimit"); // is count correct?
- else if (keyedLimitExpr->hasAttribute(skipAtom))
- flags.append("|TDRkeyedlimitskips|TDRcountkeyedlimit");
- else if (keyedLimitExpr->hasAttribute(countAtom))
- flags.append("|TDRcountkeyedlimit");
- }
- if (onlyExistsAggreate) flags.append("|TDRaggregateexists");
- if (monitorsForGrouping) flags.append("|TDRgroupmonitors");
- if (!nameExpr->isConstant()) flags.append("|TDXvarfilename");
- if (translator.hasDynamicFilename(tableExpr)) flags.append("|TDXdynamicfilename");
- if (isUnfilteredCount) flags.append("|TDRunfilteredcount");
- if (isVirtualLogicalFilenameUsed) flags.append("|TDRfilenamecallback");
- if (requiresOrderedMerge) flags.append("|TDRorderedmerge");
- if (flags.length())
- translator.doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);
- }
- void DiskReadBuilderBase::buildTransformFpos(BuildCtx & transformCtx)
- {
- if (modeOp == no_csv)
- associateFilePositions(transformCtx, "fpp", "dataSrc[0]");
- else
- associateFilePositions(transformCtx, "fpp", "left");
- }
- void DiskReadBuilderBase::extractMonitors(IHqlExpression * ds, SharedHqlExpr & unkeyedFilter, HqlExprArray & conds)
- {
- ForEachItemIn(i, conds)
- {
- IHqlExpression * filter = &conds.item(i);
- if (isSourceInvariant(ds, filter)) // more actually isSourceInvariant.
- extendConditionOwn(globalGuard, no_and, LINK(filter));
- else
- {
- node_operator op = filter->getOperator();
- switch (op)
- {
- case no_assertkeyed:
- case no_assertwild:
- {
- //MORE: This needs to test that the fields are at fixed offsets, fixed length, and collatable.
- OwnedHqlExpr extraFilter;
- monitors.extractFilters(filter, extraFilter);
- //NB: Even if it is keyed then (part of) the test condition might be duplicated.
- appendFilter(unkeyedFilter, extraFilter);
- break;
- }
- default:
- // Add this condition to the catchall filter
- appendFilter(unkeyedFilter, filter);
- break;
- }
- }
- }
- }
- //---------------------------------------------------------------------------
- class DiskReadBuilder : public DiskReadBuilderBase
- {
- public:
- DiskReadBuilder(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr)
- : DiskReadBuilderBase(_translator, _tableExpr, _nameExpr)
- {
- }
- virtual void buildTransform(IHqlExpression * expr);
- virtual void buildMembers(IHqlExpression * expr);
- };
- void DiskReadBuilder::buildMembers(IHqlExpression * expr)
- {
- if (modeOp == no_csv)
- buildFilenameMember();
- else if (modeOp != no_pipe)
- buildReadMembers(expr);
- DiskReadBuilderBase::buildMembers(expr);
- //---- virtual const char * getPipeProgram() { return "grep"; } ----
- if (modeOp==no_pipe)
- {
- if (expr->hasAttribute(_disallowed_Atom))
- throwError(HQLERR_PipeNotAllowed);
- {
- MemberFunction func(translator, instance->startctx, "virtual const char * getPipeProgram() override");
- translator.buildReturn(func.ctx, mode->queryChild(0), unknownVarStringType);
- }
- IHqlExpression * csvFromPipe = tableExpr->queryAttribute(csvAtom);
- IHqlExpression * xmlFromPipe = tableExpr->queryAttribute(xmlAtom);
- bool usesContents = false;
- if (csvFromPipe)
- {
- if (isValidCsvRecord(tableExpr->queryRecord()))
- {
- StringBuffer csvInstanceName;
- translator.buildCsvReadTransformer(tableExpr, csvInstanceName, csvFromPipe);
- StringBuffer s;
- s.append("virtual ICsvToRowTransformer * queryCsvTransformer() override { return &").append(csvInstanceName).append("; }");
- instance->classctx.addQuoted(s);
- }
- else
- {
- throwUnexpected(); // should be caught earlier
- }
- }
- else if (xmlFromPipe)
- {
- translator.doBuildXmlReadMember(*instance, expr, "queryXmlTransformer", usesContents);
- translator.doBuildVarStringFunction(instance->classctx, "getXmlIteratorPath", queryAttributeChild(xmlFromPipe, rowAtom, 0));
- }
-
- StringBuffer flags;
- if (tableExpr->hasAttribute(groupAtom)) // not supported in parser?
- flags.append("|TPFgroupeachrow");
- if (tableExpr->hasAttribute(optAtom)) // not supported in parser?
- flags.append("|TPFnofail");
- if (csvFromPipe)
- flags.append("|TPFreadcsvfrompipe");
- if (xmlFromPipe)
- flags.append("|TPFreadxmlfrompipe");
- if (usesContents)
- flags.append("|TPFreadusexmlcontents");
- if (flags.length())
- translator.doBuildUnsignedFunction(instance->classctx, "getPipeFlags", flags.str()+1);
- }
- }
- void DiskReadBuilder::buildTransform(IHqlExpression * expr)
- {
- if (modeOp == no_pipe)
- {
- assertex(!(needToCallTransform || transformCanFilter));
- return;
- }
- if (modeOp == no_csv)
- {
- translator.buildCsvParameters(instance->nestedctx, mode, NULL, true);
- {
- MemberFunction func(translator, instance->startctx, "virtual size32_t transform(ARowBuilder & crSelf, unsigned * lenSrc, const char * * dataSrc) override");
- translator.ensureRowAllocated(func.ctx, "crSelf");
- //associateVirtualCallbacks(*this, func.ctx, tableExpr);
- buildTransformBody(func.ctx, expr, true, false, true);
- }
- rootSelfRow = NULL;
- unsigned maxColumns = countTotalFields(tableExpr->queryRecord(), false);
- translator.doBuildUnsignedFunction(instance->classctx, "getMaxColumns", maxColumns);
- return;
- }
- MemberFunction func(translator, instance->startctx);
- if (instance->kind == TAKdiskread)
- func.start("virtual size32_t transform(ARowBuilder & crSelf, const void * _left) override");
- else
- func.start("virtual size32_t transform(ARowBuilder & crSelf, const void * _left, IFilePositionProvider * fpp) override");
- translator.ensureRowAllocated(func.ctx, "crSelf");
- func.ctx.addQuotedLiteral("unsigned char * left = (unsigned char *)_left;");
- buildTransformBody(func.ctx, expr, true, false, true);
- }
- //---------------------------------------------------------------------------
- ABoundActivity * HqlCppTranslator::doBuildActivityDiskRead(BuildCtx & ctx, IHqlExpression * expr)
- {
- // assertex(!isGroupedActivity(expr));
- IHqlExpression *tableExpr = queryPhysicalRootTable(expr);
- if (!tableExpr)
- return buildCachedActivity(ctx, expr->queryChild(0)); // Somehow a null appeared.
- HqlExprAttr mode = tableExpr->queryChild(2);
- node_operator modeOp = mode->getOperator();
- bool isPiped = modeOp==no_pipe;
- DiskReadBuilder info(*this, tableExpr, tableExpr->queryChild(0));
- info.gatherVirtualFields(tableExpr->hasAttribute(_noVirtual_Atom) || isPiped, needToSerializeRecord(modeOp));
- unsigned optFlags = (options.foldOptimized ? HOOfold : 0);
- if (!info.fieldInfo.simpleVirtualsAtEnd || info.fieldInfo.requiresDeserialize ||
- (info.recordHasVirtuals() && (modeOp == no_csv || !isSimpleSource(expr))))
- {
- OwnedHqlExpr transformed = buildTableWithoutVirtuals(info.fieldInfo, expr);
- //Need to wrap a possible no_usertable, otherwise the localisation can go wrong.
- if (expr->getOperator() == no_table)
- transformed.setown(createDataset(no_compound_diskread, LINK(transformed)));
- OwnedHqlExpr optimized = optimizeHqlExpression(queryErrorProcessor(), transformed, optFlags);
- traceExpression("after disk optimize", optimized);
- return doBuildActivityDiskRead(ctx, optimized);
- }
- OwnedHqlExpr optimized;
- if (expr->getOperator() == no_table)
- optimized.set(expr);
- else
- optimized.setown(optimizeHqlExpression(queryErrorProcessor(), expr, optFlags));
- if (optimized != expr)
- return buildActivity(ctx, optimized, false);
- if (optimized->getOperator() != no_compound_diskread)
- optimized.setown(createDataset(no_compound_diskread, LINK(optimized)));
- if (isPiped)
- return info.buildActivity(ctx, expr, TAKpiperead, "PipeRead", NULL);
- ensureDiskAccessAllowed(tableExpr);
- if (modeOp == no_csv)
- return info.buildActivity(ctx, expr, TAKcsvread, "CsvRead", NULL);
- return info.buildActivity(ctx, expr, TAKdiskread, "DiskRead", NULL);
- }
- //---------------------------------------------------------------------------
- class DiskNormalizeBuilder : public DiskReadBuilderBase
- {
- public:
- DiskNormalizeBuilder(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr)
- : DiskReadBuilderBase(_translator, _tableExpr, _nameExpr)
- {
- }
- virtual void buildTransform(IHqlExpression * expr);
- virtual void buildMembers(IHqlExpression * expr);
- protected:
- virtual void analyseGraph(IHqlExpression * expr);
- virtual void processTransformSelect(BuildCtx & ctx, IHqlExpression * expr)
- {
- doBuildNormalizeIterators(ctx, expr, false);
- }
- };
- void DiskNormalizeBuilder::analyseGraph(IHqlExpression * expr)
- {
- DiskReadBuilderBase::analyseGraph(expr);
- needDefaultTransform = (expr->queryNormalizedSelector()->getOperator() == no_select);
- }
- void DiskNormalizeBuilder::buildMembers(IHqlExpression * expr)
- {
- buildFilenameMember();
- DiskReadBuilderBase::buildMembers(expr);
- buildNormalizeHelpers(expr);
- }
- void DiskNormalizeBuilder::buildTransform(IHqlExpression * expr)
- {
- globaliterctx.setown(new BuildCtx(instance->startctx));
- globaliterctx->addGroup();
- MemberFunction func(translator, instance->startctx, "virtual size32_t transform(ARowBuilder & crSelf) override");
- translator.ensureRowAllocated(func.ctx, "crSelf");
- buildTransformBody(func.ctx, expr, true, false, false);
- }
- //---------------------------------------------------------------------------
- ABoundActivity * HqlCppTranslator::doBuildActivityDiskNormalize(BuildCtx & ctx, IHqlExpression * expr)
- {
- assertex(!isGroupedActivity(expr));
- IHqlExpression *tableExpr = queryPhysicalRootTable(expr);
- ensureDiskAccessAllowed(tableExpr);
- HqlExprAttr mode = tableExpr->queryChild(2);
- assertex(mode->getOperator()!=no_pipe);
- DiskNormalizeBuilder info(*this, tableExpr, tableExpr->queryChild(0));
- info.gatherVirtualFields(tableExpr->hasAttribute(_noVirtual_Atom), needToSerializeRecord(mode));
- LinkedHqlExpr transformed = expr;
- if (info.recordHasVirtualsOrDeserialize())
- transformed.setown(buildTableWithoutVirtuals(info.fieldInfo, expr));
- unsigned optFlags = (options.foldOptimized ? HOOfold : 0);
- OwnedHqlExpr optimized = optimizeHqlExpression(queryErrorProcessor(), transformed, optFlags);
- if (optimized != expr)
- return buildActivity(ctx, optimized, false);
- return info.buildActivity(ctx, expr, TAKdisknormalize, "DiskNormalize", NULL);
- }
- //---------------------------------------------------------------------------
- class DiskAggregateBuilder : public DiskReadBuilderBase
- {
- public:
- DiskAggregateBuilder(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr)
- : DiskReadBuilderBase(_translator, _tableExpr, _nameExpr)
- {
- failedFilterValue.clear();
- }
- virtual void buildTransform(IHqlExpression * expr);
- virtual void buildMembers(IHqlExpression * expr);
- protected:
- virtual void processTransformSelect(BuildCtx & ctx, IHqlExpression * expr)
- {
- doBuildAggregateSelectIterator(ctx, expr);
- }
- virtual void analyseGraph(IHqlExpression * expr)
- {
- DiskReadBuilderBase::analyseGraph(expr);
- returnIfFilterFails = !isNormalize;
- }
- };
- void DiskAggregateBuilder::buildMembers(IHqlExpression * expr)
- {
- StringBuffer s;
- buildFilenameMember();
- DiskReadBuilderBase::buildMembers(expr);
- buildAggregateHelpers(expr);
- //virtual void processRow(void * self, const void * src) = 0;
- {
- BuildCtx rowctx(instance->startctx);
- rowctx.addQuotedFunction("virtual void processRow(ARowBuilder & crSelf, const void * src) override");
- rowctx.addQuotedLiteral("doProcessRow(crSelf, (const byte *)src);");
- }
- //virtual void processRows(void * self, size32_t srcLen, const void * src) = 0;
- {
- MemberFunction func(translator, instance->startctx, "virtual void processRows(ARowBuilder & crSelf, size32_t srcLen, const void * _left) override");
- func.ctx.addQuotedLiteral("unsigned char * left = (unsigned char *)_left;");
- OwnedHqlExpr ds = createVariable("left", makeReferenceModifier(tableExpr->getType()));
- OwnedHqlExpr len = createVariable("srcLen", LINK(sizetType));
- OwnedHqlExpr fullDs = createTranslated(ds, len);
- Owned<IHqlCppDatasetCursor> iter = translator.createDatasetSelector(func.ctx, fullDs);
- BoundRow * curRow = iter->buildIterateLoop(func.ctx, false);
- s.clear().append("doProcessRow(crSelf, ");
- translator.generateExprCpp(s, curRow->queryBound());
- s.append(");");
- func.ctx.addQuoted(s);
- }
- }
- void DiskAggregateBuilder::buildTransform(IHqlExpression * expr)
- {
- MemberFunction func(translator, instance->startctx, "void doProcessRow(ARowBuilder & crSelf, const byte * left)");
- translator.ensureRowAllocated(func.ctx, "crSelf");
- buildTransformBody(func.ctx, expr, false, false, true);
- }
- //---------------------------------------------------------------------------
- class DiskCountBuilder : public DiskReadBuilderBase
- {
- public:
- DiskCountBuilder(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr, node_operator _aggOp)
- : DiskReadBuilderBase(_translator, _tableExpr, _nameExpr)
- {
- aggOp = _aggOp;
- isCompoundCount = true;
- failedFilterValue.set(queryZero());
- }
- virtual void buildTransform(IHqlExpression * expr);
- virtual void buildMembers(IHqlExpression * expr);
- virtual bool isExplicitExists() { return (aggOp == no_existsgroup); }
- protected:
- virtual void processTransformSelect(BuildCtx & ctx, IHqlExpression * expr)
- {
- doBuildAggregateSelectIterator(ctx, expr);
- }
- virtual void analyseGraph(IHqlExpression * expr)
- {
- DiskReadBuilderBase::analyseGraph(expr);
- returnIfFilterFails = !isNormalize;
- if (aggOp == no_existsgroup)
- choosenValue.setown(getSizetConstant(1));
- }
- protected:
- node_operator aggOp;
- };
- void DiskCountBuilder::buildMembers(IHqlExpression * expr)
- {
- isUnfilteredCount = !(transformCanFilter||isNormalize);
- buildFilenameMember();
- DiskReadBuilderBase::buildMembers(expr);
- buildCountHelpers(expr, true);
- }
- void DiskCountBuilder::buildTransform(IHqlExpression * expr)
- {
- if (transformCanFilter||isNormalize)
- {
- MemberFunction func(translator, instance->startctx, "size32_t valid(const byte * left)");
- OwnedHqlExpr cnt;
- if (isNormalize)
- {
- compoundCountVar.setown(func.ctx.getTempDeclare(sizetType, queryZero()));
- cnt.set(compoundCountVar);
- }
- else
- cnt.setown(getSizetConstant(1));
- BuildCtx subctx(func.ctx);
- buildTransformBody(subctx, expr, false, false, true);
- func.ctx.addReturn(cnt);
- }
- }
- //---------------------------------------------------------------------------
- ABoundActivity * HqlCppTranslator::doBuildActivityDiskAggregate(BuildCtx & ctx, IHqlExpression * expr)
- {
- assertex(!isGroupedActivity(expr));
- IHqlExpression *tableExpr = queryPhysicalRootTable(expr);
- ensureDiskAccessAllowed(tableExpr);
- HqlExprAttr mode = tableExpr->queryChild(2);
- assertex(mode->getOperator()!=no_pipe);
- DiskAggregateBuilder info(*this, tableExpr, tableExpr->queryChild(0));
- info.gatherVirtualFields(tableExpr->hasAttribute(_noVirtual_Atom), needToSerializeRecord(mode));
- LinkedHqlExpr transformed = expr;
- if (info.recordHasVirtualsOrDeserialize())
- transformed.setown(buildTableWithoutVirtuals(info.fieldInfo, expr));
- transformed.setown(optimizeHqlExpression(queryErrorProcessor(), transformed, getSourceAggregateOptimizeFlags()));
- if (transformed != expr)
- return buildActivity(ctx, transformed, false);
- node_operator aggOp = querySimpleAggregate(expr, true, false);
- if (aggOp == no_countgroup || aggOp == no_existsgroup)
- {
- DiskCountBuilder info(*this, tableExpr, tableExpr->queryChild(0), aggOp);
- info.gatherVirtualFields(tableExpr->hasAttribute(_noVirtual_Atom), needToSerializeRecord(mode));
- return info.buildActivity(ctx, expr, TAKdiskcount, "DiskCount", NULL);
- }
- else
- return info.buildActivity(ctx, expr, TAKdiskaggregate, "DiskAggregate", NULL);
- }
- //---------------------------------------------------------------------------
- class DiskGroupAggregateBuilder : public DiskReadBuilderBase
- {
- public:
- DiskGroupAggregateBuilder(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr)
- : DiskReadBuilderBase(_translator, _tableExpr, _nameExpr)
- {
- failedFilterValue.clear();
- }
- virtual void buildTransform(IHqlExpression * expr);
- virtual void buildMembers(IHqlExpression * expr);
- protected:
- virtual void processTransformSelect(BuildCtx & ctx, IHqlExpression * expr)
- {
- doBuildAggregateSelectIterator(ctx, expr);
- }
- virtual void analyseGraph(IHqlExpression * expr)
- {
- DiskReadBuilderBase::analyseGraph(expr);
- returnIfFilterFails = !isNormalize;
- }
- };
- void DiskGroupAggregateBuilder::buildMembers(IHqlExpression * expr)
- {
- buildFilenameMember();
- buildGroupingMonitors(expr, monitors);
- DiskReadBuilderBase::buildMembers(expr);
- buildGlobalGroupAggregateHelpers(expr);
- }
- void DiskGroupAggregateBuilder::buildTransform(IHqlExpression * expr)
- {
- MemberFunction func(translator, instance->startctx, "void doProcessRow(const byte * left, IHThorGroupAggregateCallback * callback)");
- bool accessesCallback = containsOperator(expr, no_filepos) || containsOperator(expr, no_file_logicalname);
- buildGroupAggregateTransformBody(func.ctx, expr, isNormalize || accessesCallback, true);
- }
- //---------------------------------------------------------------------------
- ABoundActivity * HqlCppTranslator::doBuildActivityDiskGroupAggregate(BuildCtx & ctx, IHqlExpression * expr)
- {
- IHqlExpression *tableExpr = queryPhysicalRootTable(expr);
- ensureDiskAccessAllowed(tableExpr);
- HqlExprAttr mode = tableExpr->queryChild(2);
- assertex(mode->getOperator()!=no_pipe);
- DiskGroupAggregateBuilder info(*this, tableExpr, tableExpr->queryChild(0));
- info.gatherVirtualFields(tableExpr->hasAttribute(_noVirtual_Atom), needToSerializeRecord(mode));
- LinkedHqlExpr transformed = expr;
- if (info.recordHasVirtualsOrDeserialize())
- transformed.setown(buildTableWithoutVirtuals(info.fieldInfo, expr));
- transformed.setown(optimizeHqlExpression(queryErrorProcessor(), transformed, getSourceAggregateOptimizeFlags()));
- if (transformed != expr)
- return buildActivity(ctx, transformed, false);
- return info.buildActivity(ctx, expr, TAKdiskgroupaggregate, "DiskGroupAggregate", NULL);
- }
- //-----------------------------------------------------------------------------------------------
- //-- Child dataset processing
- //-----------------------------------------------------------------------------------------------
- class ChildBuilderBase : public SourceBuilder
- {
- public:
- ChildBuilderBase(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr)
- : SourceBuilder(_translator, _tableExpr, _nameExpr)
- {
- }
- virtual void buildMembers(IHqlExpression * expr) {}
- virtual void buildTransformFpos(BuildCtx & transformCtx) {}
- };
- class ChildNormalizeBuilder : public ChildBuilderBase
- {
- public:
- ChildNormalizeBuilder(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr)
- : ChildBuilderBase(_translator, _tableExpr, _nameExpr)
- {
- }
- virtual void buildTransform(IHqlExpression * expr);
- virtual void buildMembers(IHqlExpression * expr);
- protected:
- virtual void analyseGraph(IHqlExpression * expr);
- virtual void processTransformSelect(BuildCtx & ctx, IHqlExpression * expr)
- {
- doBuildNormalizeIterators(ctx, expr, true);
- }
- };
- void ChildNormalizeBuilder::analyseGraph(IHqlExpression * expr)
- {
- ChildBuilderBase::analyseGraph(expr);
- needDefaultTransform = (expr->queryNormalizedSelector()->getOperator() == no_select);
- }
- void ChildNormalizeBuilder::buildMembers(IHqlExpression * expr)
- {
- ChildBuilderBase::buildMembers(expr);
- buildNormalizeHelpers(expr);
- }
- void ChildNormalizeBuilder::buildTransform(IHqlExpression * expr)
- {
- globaliterctx.setown(new BuildCtx(instance->startctx));
- globaliterctx->addGroup();
- MemberFunction func(translator, instance->startctx, "virtual size32_t transform(ARowBuilder & crSelf) override");
- translator.ensureRowAllocated(func.ctx, "crSelf");
- buildTransformBody(func.ctx, expr, true, false, false);
- }
- //---------------------------------------------------------------------------
- ABoundActivity * HqlCppTranslator::doBuildActivityChildNormalize(BuildCtx & ctx, IHqlExpression * expr)
- {
- ChildNormalizeBuilder info(*this, NULL, NULL);
- OwnedHqlExpr optimized = optimizeHqlExpression(queryErrorProcessor(), expr, HOOfold);
- if (optimized != expr)
- return buildActivity(ctx, optimized, false);
- return info.buildActivity(ctx, expr, TAKchildnormalize, "ChildNormalize", NULL);
- }
- //---------------------------------------------------------------------------
- class ChildAggregateBuilder : public ChildBuilderBase
- {
- public:
- ChildAggregateBuilder(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr)
- : ChildBuilderBase(_translator, _tableExpr, _nameExpr)
- {
- failedFilterValue.clear();
- }
- virtual void buildTransform(IHqlExpression * expr);
- virtual void buildMembers(IHqlExpression * expr);
- protected:
- virtual void processTransformSelect(BuildCtx & ctx, IHqlExpression * expr)
- {
- doBuildAggregateSelectIterator(ctx, expr);
- }
- virtual void analyseGraph(IHqlExpression * expr)
- {
- ChildBuilderBase::analyseGraph(expr);
- returnIfFilterFails = false;
- }
- };
- void ChildAggregateBuilder::buildMembers(IHqlExpression * expr)
- {
- ChildBuilderBase::buildMembers(expr);
- buildAggregateHelpers(expr);
- }
- void ChildAggregateBuilder::buildTransform(IHqlExpression * expr)
- {
- MemberFunction func(translator, instance->startctx, "virtual void processRows(ARowBuilder & crSelf) override");
- translator.ensureRowAllocated(func.ctx, "crSelf");
- buildTransformBody(func.ctx, expr, false, false, false);
- }
- //---------------------------------------------------------------------------
- ABoundActivity * HqlCppTranslator::doBuildActivityChildAggregate(BuildCtx & ctx, IHqlExpression * expr)
- {
- ChildAggregateBuilder info(*this, NULL, NULL);
- OwnedHqlExpr transformed = optimizeHqlExpression(queryErrorProcessor(), expr, getSourceAggregateOptimizeFlags());
- if (transformed != expr)
- return buildActivity(ctx, transformed, false);
- return info.buildActivity(ctx, expr, TAKchildaggregate, "ChildAggregate", NULL);
- }
- //---------------------------------------------------------------------------
- class ChildGroupAggregateBuilder : public ChildBuilderBase
- {
- public:
- ChildGroupAggregateBuilder(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr)
- : ChildBuilderBase(_translator, _tableExpr, _nameExpr)
- {
- failedFilterValue.clear();
- }
- virtual void buildTransform(IHqlExpression * expr);
- virtual void buildMembers(IHqlExpression * expr);
- protected:
- virtual void processTransformSelect(BuildCtx & ctx, IHqlExpression * expr)
- {
- doBuildAggregateSelectIterator(ctx, expr);
- }
- virtual void analyseGraph(IHqlExpression * expr)
- {
- ChildBuilderBase::analyseGraph(expr);
- returnIfFilterFails = false;
- }
- };
- void ChildGroupAggregateBuilder::buildMembers(IHqlExpression * expr)
- {
- ChildBuilderBase::buildMembers(expr);
- IHqlExpression * aggregate = expr->queryChild(0);
- assertex(aggregate->getOperator() == no_newaggregate);
- StringBuffer s;
- //virtual size32_t clearAggregate(void * self) = 0;
- translator.doBuildAggregateClearFunc(instance->startctx, aggregate);
- //virtual size32_t mergeAggregate(ARowBuilder & crSelf, const void * src) - never actually called.
- instance->startctx.addQuotedLiteral("virtual size32_t mergeAggregate(ARowBuilder & crSelf, const void * src) override { return 0; }");
- }
- void ChildGroupAggregateBuilder::buildTransform(IHqlExpression * expr)
- {
- MemberFunction func(translator, instance->startctx, "void processRows(IHThorGroupAggregateCallback * callback)");
- buildGroupAggregateTransformBody(func.ctx, expr, true, false);
- }
- //---------------------------------------------------------------------------
- ABoundActivity * HqlCppTranslator::doBuildActivityChildGroupAggregate(BuildCtx & ctx, IHqlExpression * expr)
- {
- ChildGroupAggregateBuilder info(*this, NULL, NULL);
- OwnedHqlExpr transformed = optimizeHqlExpression(queryErrorProcessor(), expr, getSourceAggregateOptimizeFlags());
- if (transformed != expr)
- return buildActivity(ctx, transformed, false);
- return info.buildActivity(ctx, expr, TAKchildgroupaggregate, "ChildGroupAggregate", NULL);
- }
- //---------------------------------------------------------------------------
- class ChildThroughNormalizeBuilder : public ChildBuilderBase
- {
- public:
- ChildThroughNormalizeBuilder(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr)
- : ChildBuilderBase(_translator, _tableExpr, _nameExpr)
- {
- }
- virtual void buildTransform(IHqlExpression * expr);
- virtual void buildMembers(IHqlExpression * expr);
- protected:
- virtual void analyseGraph(IHqlExpression * expr);
- virtual void processTransformSelect(BuildCtx & ctx, IHqlExpression * expr)
- {
- translator.bindTableCursor(ctx, expr, "left");
- doBuildNormalizeIterators(ctx, expr, false);
- }
- };
- void ChildThroughNormalizeBuilder::analyseGraph(IHqlExpression * expr)
- {
- ChildBuilderBase::analyseGraph(expr);
- needDefaultTransform = (expr->queryNormalizedSelector()->getOperator() == no_select);
- }
- void ChildThroughNormalizeBuilder::buildMembers(IHqlExpression * expr)
- {
- ChildBuilderBase::buildMembers(expr);
- buildNormalizeHelpers(expr);
- }
- void ChildThroughNormalizeBuilder::buildTransform(IHqlExpression * expr)
- {
- globaliterctx.setown(new BuildCtx(instance->startctx));
- globaliterctx->addGroup();
- MemberFunction func(translator, instance->startctx, "virtual size32_t transform(ARowBuilder & crSelf) override");
- translator.ensureRowAllocated(func.ctx, "crSelf");
- buildTransformBody(func.ctx, expr, true, false, false);
- }
- ABoundActivity * HqlCppTranslator::doBuildActivityCompoundSelectNew(BuildCtx & ctx, IHqlExpression * expr)
- {
- OwnedHqlExpr optimized = optimizeHqlExpression(queryErrorProcessor(), expr, HOOfold);
- if (optimized->getOperator() == no_null)
- return buildCachedActivity(ctx, optimized);
- IHqlExpression * select = queryRoot(optimized);
- if (!(select && select->getOperator() == no_select && select->hasAttribute(newAtom)))
- {
- if (optimized->getOperator() == no_compound_selectnew)
- return buildCachedActivity(ctx, optimized->queryChild(0));
- return buildCachedActivity(ctx, optimized);
- }
- IHqlExpression * ds = select->queryChild(0);
- Owned<ABoundActivity> childActivity = buildCachedActivity(ctx, ds);
- OwnedHqlExpr fakeDataset = createDataset(no_anon, LINK(ds->queryRecord()));
- OwnedHqlExpr fakeSelect = createNewSelectExpr(LINK(fakeDataset), LINK(select->queryChild(1)));
- OwnedHqlExpr activityExpr = replaceExpression(optimized, select, fakeSelect);
- ChildThroughNormalizeBuilder info(*this, fakeDataset, NULL);
- info.gatherVirtualFields(true, true); // ,false?
- return info.buildActivity(ctx, activityExpr, TAKchildthroughnormalize, "ChildThroughNormalize", childActivity);
- }
- //-----------------------------------------------------------------------------------------------
- //-- Index processing
- //-----------------------------------------------------------------------------------------------
- /*
- Note on generating segment monitors for conditions (ty)x = Y
- 1) casting from Tx to Ty loses information. E.g., (int1)string2field = int1value.
- In this case it is almost impossible to generate a segment monitor because we would need to work out all
- the possibly values for x which could generate the value y.
- The only exception is an inequality (and no_notin), which we can use to remove some of the candidates. The test will always need
- duplicating since we cannot remove all the expected values.
- 2) Casting from Ty to Tx loses information. E.g., (string)string2field = stringvalue
- In this case we can process the filter without prefiltering by testing whether isExact: (Ty)(Tx)Y == Y, and following the following rules:
- a) no_eq. If isExact, add value else nothing.
- b) no_ne. If isExtact, remove value else nothing.
- c) no_gt. Always add > value
- d) no_ge. If isExact add >= value else add > value
- e) no_lt. If isExact add < value else add <= value
- f) no_le. Always add <= value
- 3) Note casts must be present on both sides to indicate exactly what type the comparison will be done as. This includes (string)which would
- normally be missing if the field was of type string<n>.
- */
- static node_operator getModifiedOp(node_operator op, bool duplicate)
- {
- if (!duplicate)
- return op;
- switch (op)
- {
- case no_eq:
- case no_le:
- case no_ge:
- case no_in:
- return op;
- //err on the side of caution for the segment monitors -
- //the test later which check it more accurately.
- case no_gt:
- return no_ge;
- case no_lt:
- return no_le;
- case no_ne:
- case no_notin:
- return no_none;
- default:
- UNIMPLEMENTED;
- }
- }
- void KeyFailureInfo::merge(const KeyFailureInfo & other)
- {
- if (code < other.code)
- set(other.code, other.field);
- }
- void KeyFailureInfo::reportError(HqlCppTranslator & translator, IHqlExpression * condition)
- {
- StringBuffer ecl;
- getExprECL(condition, ecl);
- switch (code)
- {
- case KFRunknown:
- translator.throwError1(HQLERR_KeyedJoinTooComplex, ecl.str());
- case KFRnokey:
- translator.throwError1(HQLERR_KeyAccessNoKeyField, ecl.str());
- case KFRtoocomplex:
- translator.throwError1(HQLERR_KeyedJoinTooComplex, ecl.str());
- case KFRcast:
- translator.throwError2(HQLERR_KeyAccessNeedCast, ecl.str(), str(field->queryName()));
- case KFRor:
- translator.throwError1(HQLERR_OrMultipleKeyfields, ecl.str());
- }
- }
- const char * BuildMonitorState::getSetName(bool createValueSets)
- {
- if (!setNames.isItem(numActiveSets))
- {
- StringBuffer name;
- getUniqueId(name.append("set"));
- setNames.append(*new StringAttrItem(name.str()));
- StringBuffer s;
- funcctx.setNextConstructor();
- if (createValueSets)
- funcctx.addQuoted(s.append("Owned<IValueSet> ").append(name).append(";"));
- else
- funcctx.addQuoted(s.append("Owned<IStringSet> ").append(name).append(";"));
- }
- return setNames.item(numActiveSets++).text;
- }
- void BuildMonitorState::popSetName()
- {
- numActiveSets--;
- }
- IHqlExpression * KeyConditionInfo::createConjunction()
- {
- LinkedHqlExpr result = preFilter;
- ForEachItemIn(i, conditions)
- extendAndCondition(result, conditions.item(i).expr);
- extendAndCondition(result, postFilter);
- return result.getClear();
- }
- //---------------------------------------------------------------------------------------------------------------------
- const char * KeySelectorInfo::getFFOptions()
- {
- switch (keyedKind)
- {
- case KeyedExtend:
- return "FFopt";
- default:
- return "FFkeyed";
- }
- }
- //---------------------------------------------------------------------------------------------------------------------
- MonitorExtractor::MonitorExtractor(IHqlExpression * _tableExpr, HqlCppTranslator & _translator, int _numKeyableFields, bool _isDiskRead) : translator(_translator)
- {
- tableExpr = _tableExpr;
- if (_numKeyableFields <= 0)
- {
- //-ve number means remove a certain number of fields from the record
- IHqlExpression * record = tableExpr->queryRecord();
- numKeyableFields = 0;
- ForEachChild(i, record)
- if (!record->queryChild(i)->isAttribute())
- numKeyableFields++;
- numKeyableFields += _numKeyableFields; // remove payload fields.
- }
- else
- numKeyableFields = (unsigned)_numKeyableFields;
- onlyHozedCompares = !_isDiskRead;
- expandKeyableFields();
- cleanlyKeyedExplicitly = false;
- keyedExplicitly = false;
- allowDynamicFormatChange = !tableExpr->hasAttribute(fixedAtom);
- createValueSets = translator.queryOptions().createValueSets;
- if (createValueSets)
- {
- addRangeFunc = addRawRangeId;
- killRangeFunc = killRawRangeId;
- }
- else
- {
- addRangeFunc = addRangeId;
- killRangeFunc = killRangeId;
- }
- }
- void MonitorExtractor::callAddAll(BuildCtx & ctx, IHqlExpression * targetVar)
- {
- HqlExprArray args;
- args.append(*LINK(targetVar));
- translator.callProcedure(ctx, addAllId, args);
- }
- static IHqlExpression * createExpandedRecord(IHqlExpression * expr);
- static ITypeInfo * getExpandedFieldType(ITypeInfo * type, IHqlExpression * expr)
- {
- Linked<ITypeInfo> expandedType = type;
- if (type->getSize() == UNKNOWN_LENGTH)
- expandedType.clear();
- switch (type->getTypeCode())
- {
- case type_packedint:
- expandedType.setown(makeIntType(type->queryPromotedType()->getSize(), type->isSigned()));
- break;
- case type_bitfield:
- expandedType.set(type->queryPromotedType());
- break;
- case type_varstring:
- case type_varunicode:
- #if 0
- if (type->getSize() != UNKNOWN_LENGTH)
- {
- unsigned len = type->getStringLen();
- switch (type->getTypeCode())
- {
- case type_varstring:
- expandedType.setown(makeStringType(len, LINK(type->queryCharset()), LINK(type->queryCollation())));
- break;
- case type_varunicode:
- expandedType.setown(makeUnicodeType(len, type->queryLocale()));
- break;
- }
- break;
- }
- #endif //fall through
- case type_data:
- case type_qstring:
- case type_string:
- case type_unicode:
- case type_utf8:
- if (type->getSize() == UNKNOWN_LENGTH)
- {
- unsigned maxLength = UNKNOWN_LENGTH;
- IHqlExpression * maxSizeExpr = expr ? queryAttributeChild(expr, maxSizeAtom, 0) : NULL;
- if (maxSizeExpr)
- {
- unsigned maxSize = (unsigned)maxSizeExpr->queryValue()->getIntValue();
- switch (type->getTypeCode())
- {
- case type_data:
- case type_string:
- maxLength = maxSize - sizeof(size32_t);
- break;
- case type_qstring:
- maxLength = rtlQStrLength(maxSize - sizeof(size32_t));
- break;
- case type_unicode:
- maxLength = (maxSize-sizeof(size32_t))/sizeof(UChar);
- break;
- case type_utf8:
- maxLength = (maxSize-sizeof(size32_t))/4;
- break;
- case type_varstring:
- maxLength = maxSize - 1;
- break;
- case type_varunicode:
- maxLength = (maxSize/sizeof(UChar)) - 1;
- break;
- }
- }
- else
- {
- IHqlExpression * maxLengthExpr = expr ? queryAttributeChild(expr, maxLengthAtom, 0) : NULL;
- if (maxLengthExpr)
- maxLength = (unsigned)maxLengthExpr->queryValue()->getIntValue();
- }
- if (maxLength != UNKNOWN_LENGTH)
- {
- switch (type->getTypeCode())
- {
- case type_data:
- expandedType.setown(makeDataType(maxLength));
- break;
- case type_qstring:
- expandedType.setown(makeQStringType(maxLength));
- break;
- case type_string:
- expandedType.setown(makeStringType(maxLength, LINK(type->queryCharset()), LINK(type->queryCollation())));
- break;
- case type_unicode:
- expandedType.setown(makeUnicodeType(maxLength, type->queryLocale()));
- break;
- case type_utf8:
- expandedType.setown(makeUtf8Type(maxLength, type->queryLocale()));
- break;
- case type_varstring:
- expandedType.setown(makeVarStringType(maxLength, LINK(type->queryCharset()), LINK(type->queryCollation())));
- break;
- case type_varunicode:
- expandedType.setown(makeVarUnicodeType(maxLength, type->queryLocale()));
- break;
- }
- }
- }
- else
- {
- //This could ensure the strings are ascii, but the ebcdic strings are still comparable, and the order will be more logical
- //if they remain as ebcdic.
- }
- break;
- case type_table:
- case type_groupedtable:
- case type_set:
- expandedType.clear();
- break;
- case type_row:
- {
- OwnedHqlExpr newRecord = createExpandedRecord(queryRecord(type));
- if (isEmptyRecord(newRecord))
- expandedType.clear();
- else
- expandedType.setown(makeRowType(LINK(newRecord->queryRecordType())));
- break;
- }
- case type_alien:
- {
- IHqlAlienTypeInfo * alien = queryAlienType(type);
- expandedType.set(alien->queryLogicalType());
- break;
- }
- }
- return expandedType.getClear();
- }
- static void createExpanded(HqlExprArray & fields, IHqlExpression * expr)
- {
- switch (expr->getOperator())
- {
- case no_ifblock:
- //if blocks need to generate translated segment monitors to be keyed, so don't expand them
- break;
- case no_record:
- {
- ForEachChild(i, expr)
- createExpanded(fields, expr->queryChild(i));
- break;
- }
- case no_field:
- {
- ITypeInfo * type = expr->queryType();
- Owned<ITypeInfo> expandedType = getExpandedFieldType(type, expr);
- if (expandedType)
- {
- if (expandedType == type)
- fields.append(*LINK(expr));
- else
- {
- HqlExprArray attrs;
- unwindChildren(attrs, expr);
- //MORE: Any default will now have the wrong type => remove it for the moment (ideally it would be projected)
- removeAttribute(attrs, defaultAtom);
- fields.append(*createField(expr->queryId(), LINK(expandedType), attrs));
- }
- }
- break;
- }
- case no_attr:
- case no_attr_link:
- case no_attr_expr:
- fields.append(*LINK(expr));
- break;
- }
- }
- static IHqlExpression * createExpandedRecord(IHqlExpression * expr)
- {
- HqlExprArray fields;
- createExpanded(fields, expr);
- return cloneOrLink(expr, fields);
- }
- bool MonitorExtractor::createGroupingMonitor(BuildCtx ctx, const char * listName, IHqlExpression * expr, unsigned & maxOffset)
- {
- switch (expr->getOperator())
- {
- case no_if:
- {
- IHqlExpression * cond = expr->queryChild(0);
- if (expr->queryChild(2)->isConstant() && isIndependentOfScope(cond))
- {
- BuildCtx subctx(ctx);
- translator.buildFilter(subctx, expr->queryChild(0));
- createGroupingMonitor(subctx, listName, expr->queryChild(1), maxOffset);
- return true; // may still be keyed
- }
- break;
- }
- case no_select:
- {
- size32_t offset = 0;
- ForEachItemIn(i, keyableSelects)
- {
- IHqlExpression & cur = keyableSelects.item(i);
- size32_t curSize = cur.queryType()->getSize();
- if (curSize == UNKNOWN_LENGTH)
- break;
- if (expr == &cur)
- {
- //MORE: Check the type of the field is legal.
- ctx.addQuotedF("%s->append(createWildKeySegmentMonitor(%u, %u, %u));", listName, 0, offset, curSize);
- maxOffset = offset+curSize;
- return true;
- }
- offset += curSize;
- }
- break;
- }
- case no_constant:
- return true;
- }
- ctx.addReturn(queryBoolExpr(false));
- return false;
- }
- void MonitorExtractor::expandKeyableFields()
- {
- HqlExprArray fields;
- IHqlExpression * tableRecord = tableExpr->queryRecord();
- unsigned cnt = 0;
- ForEachChild(i, tableRecord)
- {
- if (cnt == numKeyableFields)
- break;
- IHqlExpression * cur = tableRecord->queryChild(i);
- if (!cur->isAttribute())
- {
- fields.append(*LINK(cur));
- cnt++;
- }
- }
- keyableRecord.setown(createRecord(fields));
- expandedRecord.setown(createExpandedRecord(keyableRecord));
- IHqlExpression * selector = tableExpr->queryNormalizedSelector();
- OwnedHqlExpr expandedSelector = createDataset(no_anon, LINK(expandedRecord), createUniqueId());
- firstOffsetField = NotFound;
- expandSelects(keyableRecord, expandedRecord->querySimpleScope(), selector, expandedSelector);
- if (firstOffsetField == NotFound)
- firstOffsetField = keyableSelects.ordinality();
- }
- void MonitorExtractor::expandSelects(IHqlExpression * expr, IHqlSimpleScope * expandedScope, IHqlExpression * keySelector, IHqlExpression * expandedSelector)
- {
- switch (expr->getOperator())
- {
- case no_record:
- {
- ForEachChild(i, expr)
- expandSelects(expr->queryChild(i), expandedScope, keySelector, expandedSelector);
- break;
- }
- case no_ifblock:
- expandSelects(expr->queryChild(1), expandedScope, keySelector, expandedSelector);
- break;
- case no_field:
- {
- OwnedHqlExpr match = expandedScope->lookupSymbol(expr->queryId());
- if (match)
- {
- OwnedHqlExpr keySelected = createSelectExpr(LINK(keySelector), LINK(expr));
- OwnedHqlExpr expandedSelected = createSelectExpr(LINK(expandedSelector), LINK(match));
- IHqlExpression * record = expr->queryRecord();
- if (record)
- expandSelects(record, match->queryRecord()->querySimpleScope(), keySelected, expandedSelected);
- else
- {
- if ((expr != match) && (firstOffsetField == NotFound))
- {
- ITypeInfo * exprType = expr->queryType();
- ITypeInfo * matchType = match->queryType();
- if ((exprType->getSize() != matchType->getSize()) ||
- (exprType->getTypeCode() == type_bitfield || exprType->getTypeCode() == type_bitfield))
- firstOffsetField = keyableSelects.ordinality();
- }
- keyableSelects.append(*LINK(keySelected));
- expandedSelects.append(*LINK(expandedSelected));
- }
- }
- else
- {
- if (firstOffsetField == NotFound)
- firstOffsetField = keyableSelects.ordinality();
- }
- break;
- }
- }
- }
- void MonitorExtractor::buildKeySegmentInExpr(BuildMonitorState & buildState, KeySelectorInfo & selectorInfo, BuildCtx & ctx, const char * target, IHqlExpression & thisKey, MonitorFilterKind filterKind)
- {
- //Generally this slightly increases the code size, but reduces the number of
- //temporary sets which is generally more efficient.
- OwnedHqlExpr simplified = querySimplifyInExpr(&thisKey);
- if (simplified)
- {
- OwnedHqlExpr folded = foldHqlExpression(simplified);
- buildKeySegmentExpr(buildState, selectorInfo, ctx, target, *folded, filterKind);
- return;
- }
- IHqlExpression * expandedSelector = selectorInfo.expandedSelector;
- ITypeInfo * fieldType = expandedSelector->queryType();
- unsigned curSize = fieldType->getSize();
- createStringSet(ctx, target, curSize, expandedSelector);
- OwnedHqlExpr targetVar = createVariable(target, makeVoidType());
- IHqlExpression * lhs = thisKey.queryChild(0);
- OwnedHqlExpr values = normalizeListCasts(thisKey.queryChild(1));
- IIdAtom * func = addRangeFunc;
- if (thisKey.getOperator() == no_notin)
- {
- callAddAll(ctx, targetVar);
- func = killRangeFunc;
- }
- if (values->getOperator() != no_list)
- {
- //iterate through the set
- BuildCtx subctx(ctx);
- CHqlBoundExpr boundCurElement;
- Owned<IHqlCppSetCursor> cursor = translator.createSetSelector(ctx, values);
- bool done = false;
- CHqlBoundExpr isAll;
- cursor->buildIsAll(subctx, isAll);
- if (isAll.expr->queryValue())
- {
- if (isAll.expr->queryValue()->getBoolValue())
- {
- callAddAll(subctx, targetVar);
- done = true;
- //If ALL allowed exceptions then we would need to do more....
- }
- }
- else
- {
- IHqlStmt * stmt = subctx.addFilter(isAll.expr);
- callAddAll(subctx, targetVar);
- subctx.selectElse(stmt);
- }
- if (!done)
- {
- cursor->buildIterateLoop(subctx, boundCurElement, false);
- OwnedHqlExpr curValue = boundCurElement.getTranslatedExpr();
- OwnedHqlExpr test = createBoolExpr(no_eq, LINK(lhs), LINK(curValue));
- OwnedHqlExpr promoted = getExplicitlyPromotedCompare(test);
- OwnedHqlExpr compare, normalized;
- extractCompareInformation(subctx, promoted, compare, normalized, expandedSelector);
- if (compare)
- translator.buildFilter(subctx, compare);
- HqlExprArray args;
- args.append(*LINK(targetVar));
- unsigned srcSize = normalized->queryType()->getSize();
- if (srcSize < curSize)
- {
- OwnedHqlExpr lengthExpr = getSizetConstant(srcSize);
- OwnedHqlExpr rangeLower = getRangeLimit(fieldType, lengthExpr, normalized, -1);
- OwnedHqlExpr rangeUpper = getRangeLimit(fieldType, lengthExpr, normalized, +1);
- CHqlBoundExpr boundLower, boundUpper;
- translator.buildExpr(subctx, rangeLower, boundLower);
- translator.buildExpr(subctx, rangeUpper, boundUpper);
- args.append(*getPointer(boundLower.expr));
- args.append(*getPointer(boundUpper.expr));
- }
- else
- {
- OwnedHqlExpr address = getMonitorValueAddress(subctx, normalized);
- args.append(*LINK(address));
- args.append(*LINK(address));
- }
- translator.callProcedure(subctx, func, args);
- }
- }
- else
- {
- ForEachChild(idx2, values)
- {
- BuildCtx subctx(ctx);
- IHqlExpression * cur = values->queryChild(idx2);
- OwnedHqlExpr test = createBoolExpr(no_eq, LINK(lhs), LINK(cur));
- OwnedHqlExpr promoted = getExplicitlyPromotedCompare(test);
- OwnedHqlExpr compare, normalized;
- extractCompareInformation(subctx, promoted, compare, normalized, expandedSelector);
- if (compare)
- translator.buildFilter(subctx, compare);
- OwnedHqlExpr address = getMonitorValueAddress(subctx, normalized);
- HqlExprArray args;
- args.append(*LINK(targetVar));
- args.append(*LINK(address));
- args.append(*LINK(address));
- translator.callProcedure(subctx, func, args);
- }
- }
- }
- static IHqlExpression * createCompareRecast(node_operator op, IHqlExpression * value, IHqlExpression * recastValue)
- {
- if (recastValue->queryValue())
- return LINK(queryBoolExpr(op == no_ne));
- return createValue(op, makeBoolType(), LINK(value), LINK(recastValue));
- }
- void MonitorExtractor::extractCompareInformation(BuildCtx & ctx, IHqlExpression * expr, SharedHqlExpr & compare, SharedHqlExpr & normalized, IHqlExpression * expandedSelector)
- {
- extractCompareInformation(ctx, expr->queryChild(0), expr->queryChild(1), compare, normalized, expandedSelector);
- }
- void MonitorExtractor::extractCompareInformation(BuildCtx & ctx, IHqlExpression * lhs, IHqlExpression * value, SharedHqlExpr & compare, SharedHqlExpr & normalized, IHqlExpression * expandedSelector)
- {
- LinkedHqlExpr compareValue = value->queryBody();
- OwnedHqlExpr recastValue;
- if ((lhs->getOperator() != no_select) || (lhs->queryType() != compareValue->queryType()))
- {
- OwnedHqlExpr temp = castToFieldAndBack(lhs, compareValue);
- if (temp != compareValue)
- {
- //Force into a temporary variable since it will be used more than once, and reapply the field casting/
- compareValue.setown(translator.buildSimplifyExpr(ctx, compareValue));
- //cast to promoted type because sometimes evaluating can convert string to string<n>
- Owned<ITypeInfo> promotedType = getPromotedECLType(lhs->queryType(), compareValue->queryType());
- compareValue.setown(ensureExprType(compareValue, promotedType));
- recastValue.setown(castToFieldAndBack(lhs, compareValue));
- }
- }
- normalized.setown(invertTransforms(lhs, compareValue));
- normalized.setown(foldHqlExpression(normalized));
- if (recastValue && recastValue != compareValue)
- compare.setown(createCompareRecast(no_eq, compareValue, recastValue));
- }
- void MonitorExtractor::createStringSet(BuildCtx & ctx, const char * target, unsigned size, IHqlExpression * selector)
- {
- assertex(selector->getOperator() == no_select);
- if (createValueSets)
- {
- StringBuffer type;
- translator.buildRtlFieldType(type, selector->queryChild(1), queryRecord(tableExpr));
- ctx.addQuotedF("%s.setown(createValueSet(%s));", target, type.str());
- }
- else
- {
- if (onlyHozedCompares)
- ctx.addQuotedF("%s.setown(createRtlStringSet(%u));", target, size);
- else
- {
- ITypeInfo * type = selector->queryType();
- bool isBigEndian = !type->isInteger() || !isLittleEndian(type);
- ctx.addQuotedF("%s.setown(createRtlStringSetEx(%u,%d,%d));", target, size, isBigEndian, type->isSigned());
- }
- }
- }
- void MonitorExtractor::buildKeySegmentCompareExpr(BuildMonitorState & buildState, KeySelectorInfo & selectorInfo, BuildCtx & ctx, const char * targetSet, IHqlExpression & thisKey)
- {
- OwnedHqlExpr targetVar = createVariable(targetSet, makeVoidType());
- createStringSet(ctx, targetSet, selectorInfo.size, selectorInfo.expandedSelector);
- if (!exprReferencesDataset(&thisKey, tableExpr))
- {
- BuildCtx subctx(ctx);
- translator.buildFilter(subctx, &thisKey);
- callAddAll(subctx, targetVar);
- return;
- }
- OwnedHqlExpr compare;
- OwnedHqlExpr normalized;
- BuildCtx subctx(ctx);
- extractCompareInformation(subctx, &thisKey, compare, normalized, selectorInfo.expandedSelector);
- OwnedHqlExpr address = getMonitorValueAddress(subctx, normalized);
- HqlExprArray args;
- args.append(*LINK(targetVar));
- node_operator op = thisKey.getOperator();
- switch (op)
- {
- case no_eq:
- if (compare)
- translator.buildFilter(subctx, compare);
- args.append(*LINK(address));
- args.append(*LINK(address));
- translator.callProcedure(subctx, addRangeFunc, args);
- break;
- case no_ne:
- subctx.addQuoted(StringBuffer().appendf("%s->addAll();", targetSet));
- if (compare)
- translator.buildFilter(subctx, compare);
- args.append(*LINK(address));
- args.append(*LINK(address));
- translator.callProcedure(subctx, killRangeFunc, args);
- break;
- case no_le:
- args.append(*createValue(no_nullptr, makeVoidType()));
- args.append(*LINK(address));
- translator.callProcedure(subctx, addRangeFunc, args);
- break;
- case no_lt:
- // e) no_lt. If isExact add < value else add <= value
- if (compare)
- {
- OwnedHqlExpr invCompare = getInverse(compare);
- IHqlStmt * cond = translator.buildFilterViaExpr(subctx, invCompare);
- //common this up...
- args.append(*createValue(no_nullptr, makeVoidType()));
- args.append(*LINK(address));
- translator.callProcedure(subctx, addRangeFunc, args);
- subctx.selectElse(cond);
- args.append(*LINK(targetVar));
- }
- subctx.addQuoted(StringBuffer().appendf("%s->addAll();", targetSet));
- args.append(*LINK(address));
- args.append(*createValue(no_nullptr, makeVoidType()));
- translator.callProcedure(subctx, killRangeFunc, args);
- break;
- case no_ge:
- // d) no_ge. If isExact add >= value else add > value
- if (compare)
- {
- OwnedHqlExpr invCompare = getInverse(compare);
- IHqlStmt * cond = translator.buildFilterViaExpr(subctx, invCompare);
- //common this up...
- subctx.addQuoted(StringBuffer().appendf("%s->addAll();", targetSet));
- args.append(*createValue(no_nullptr, makeVoidType()));
- args.append(*LINK(address));
- translator.callProcedure(subctx, killRangeFunc, args);
- subctx.selectElse(cond);
- args.append(*LINK(targetVar));
- }
- args.append(*LINK(address));
- args.append(*createValue(no_nullptr, makeVoidType()));
- translator.callProcedure(subctx, addRangeFunc, args);
- break;
- case no_gt:
- subctx.addQuoted(StringBuffer().appendf("%s->addAll();", targetSet));
- args.append(*createValue(no_nullptr, makeVoidType()));
- args.append(*LINK(address));
- translator.callProcedure(subctx, killRangeFunc, args);
- break;
- case no_between:
- case no_notbetween:
- {
- //NB: This should only be generated for substring queries. User betweens are converted
- //to two separate comparisons to cope with range issues.
- args.append(*LINK(address));
- CHqlBoundExpr rhs2;
- OwnedHqlExpr adjustedUpper = invertTransforms(thisKey.queryChild(0), thisKey.queryChild(2));
- OwnedHqlExpr foldedUpper = foldHqlExpression(adjustedUpper);
- OwnedHqlExpr hozedValue = getHozedKeyValue(foldedUpper);
- IIdAtom * name = hozedValue->queryId();
- if ((name != createRangeHighId) && (name != createQStrRangeHighId))
- hozedValue.setown(ensureExprType(hozedValue, selectorInfo.expandedSelector->queryType()));
- translator.buildExpr(subctx, hozedValue, rhs2);
- translator.ensureHasAddress(subctx, rhs2);
- args.append(*getPointer(rhs2.expr));
- if (op == no_between)
- translator.callProcedure(subctx, addRangeFunc, args);
- else
- {
- subctx.addQuoted(StringBuffer().appendf("%s->addAll();", targetSet));
- translator.callProcedure(subctx, killRangeFunc, args);
- }
- break;
- }
- default:
- throwUnexpectedOp(op);
- }
- }
- IHqlExpression * MonitorExtractor::unwindConjunction(HqlExprArray & matches, IHqlExpression * expr)
- {
- node_operator op = expr->getOperator();
- expr->unwindList(matches, op);
- OwnedHqlExpr invariant;
- ForEachItemInRev(i, matches)
- {
- IHqlExpression & cur = matches.item(i);
- if (isIndexInvariant(&cur, false))
- {
- invariant.setown(extendConditionOwn(op, LINK(&cur), invariant.getClear()));
- matches.remove(i);
- }
- }
- return invariant.getClear();
- }
- //Note this function may change the incoming ctx if filterKind is not NoMonitorFilter
- void MonitorExtractor::buildKeySegmentExpr(BuildMonitorState & buildState, KeySelectorInfo & selectorInfo, BuildCtx & ctx, const char * requiredSet, IHqlExpression & thisKey, MonitorFilterKind filterKind)
- {
- const char * targetSet = requiredSet;
- StringBuffer s;
- unsigned curSize = selectorInfo.size;
- node_operator op = thisKey.getOperator();
- BuildCtx subctx(ctx);
- BuildCtx * appendCtx = &ctx;
- StringBuffer createMonitorText;
- switch (op)
- {
- case no_in:
- case no_notin:
- {
- if (!targetSet)
- targetSet = buildState.getSetName(createValueSets);
- buildKeySegmentInExpr(buildState, selectorInfo, ctx, targetSet, thisKey, filterKind);
- break;
- }
- case no_if:
- {
- MonitorFilterKind childFilter = targetSet ? NoMonitorFilter : filterKind;
- IHqlStmt * ifStmt = translator.buildFilterViaExpr(subctx, thisKey.queryChild(0));
- buildKeySegmentExpr(buildState, selectorInfo, subctx, targetSet, *thisKey.queryChild(1), childFilter);
- subctx.selectElse(ifStmt);
- buildKeySegmentExpr(buildState, selectorInfo, subctx, targetSet, *thisKey.queryChild(2), childFilter);
- break;
- }
- case no_and:
- {
- HqlExprArray matches;
- OwnedHqlExpr invariant = unwindConjunction(matches, &thisKey);
- unsigned numMatches = matches.ordinality();
- if (!targetSet && numMatches > 1)
- targetSet = buildState.getSetName(createValueSets);
- IHqlStmt * ifStmt = NULL;
- if (invariant)
- {
- ifStmt = translator.buildFilterViaExpr(subctx, invariant);
- if (filterKind == MonitorFilterSkipEmpty)
- ctx.set(subctx);
- }
-
- buildKeySegmentExpr(buildState, selectorInfo, subctx, targetSet, matches.item(0), NoMonitorFilter);
- for (unsigned i=1; i< numMatches; i++)
- {
- IHqlExpression & cur = matches.item(i);
- const char * curTarget = buildState.getSetName(createValueSets);
- BuildCtx childctx(subctx);
- buildKeySegmentExpr(buildState, selectorInfo, childctx, curTarget, cur, MonitorFilterSkipAll);
- if (createValueSets)
- childctx.addQuotedF("%s->intersectSet(%s);", targetSet, curTarget);
- else
- childctx.addQuotedF("%s.setown(rtlIntersectSet(%s,%s));", targetSet, targetSet, curTarget);
- buildState.popSetName();
- }
- if (invariant && (filterKind != MonitorFilterSkipEmpty))
- {
- subctx.selectElse(ifStmt);
- if (targetSet)
- createStringSet(subctx, targetSet, curSize, selectorInfo.selector);
- else
- buildEmptyKeySegment(buildState, subctx, selectorInfo);
- }
- break;
- }
- case no_or:
- {
- HqlExprArray matches;
- OwnedHqlExpr invariant = unwindConjunction(matches, &thisKey);
- unsigned numMatches = matches.ordinality();
- if (invariant)
- {
- if (filterKind == MonitorFilterSkipAll)
- {
- OwnedHqlExpr test = getInverse(invariant);
- translator.buildFilter(subctx, test);
- ctx.set(subctx);
- }
- else
- {
- IHqlStmt * ifStmt = translator.buildFilterViaExpr(subctx, invariant);
- if (targetSet)
- {
- createStringSet(subctx, targetSet, curSize, selectorInfo.selector);
- OwnedHqlExpr targetVar = createVariable(targetSet, makeVoidType());
- callAddAll(subctx, targetVar);
- }
- subctx.selectElse(ifStmt);
- }
- }
-
- appendCtx = &subctx;
- if (!targetSet && numMatches > 1)
- targetSet = buildState.getSetName(createValueSets);
- buildKeySegmentExpr(buildState, selectorInfo, subctx, targetSet, matches.item(0), NoMonitorFilter);
- for (unsigned i=1; i < numMatches; i++)
- {
- IHqlExpression & cur = matches.item(i);
- const char * curTarget = buildState.getSetName(createValueSets);
- BuildCtx childctx(subctx);
- buildKeySegmentExpr(buildState, selectorInfo, childctx, curTarget, cur, MonitorFilterSkipEmpty);
- if (createValueSets)
- childctx.addQuotedF("%s->unionSet(%s);", targetSet, curTarget);
- else
- childctx.addQuotedF("%s.setown(rtlUnionSet(%s, %s));", targetSet, targetSet, curTarget);
- buildState.popSetName();
- }
- break;
- }
- case no_eq:
- {
- if (!targetSet)
- {
- if (buildSingleKeyMonitor(createMonitorText, selectorInfo, subctx, thisKey))
- break;
- targetSet = buildState.getSetName(createValueSets);
- }
- buildKeySegmentCompareExpr(buildState, selectorInfo, ctx, targetSet, thisKey);
- break;
- }
- default:
- {
- if (!targetSet)
- targetSet = buildState.getSetName(createValueSets);
- buildKeySegmentCompareExpr(buildState, selectorInfo, ctx, targetSet, thisKey);
- break;
- }
- }
- if (targetSet && !requiredSet)
- {
- if (createValueSets)
- createMonitorText.appendf("createFieldFilter(%u, %s.getClear())", selectorInfo.fieldIdx, targetSet);
- else
- createMonitorText.appendf("createKeySegmentMonitor(%s, %s.getClear(), %u, %u, %u)",
- boolToText(selectorInfo.keyedKind != KeyedYes), targetSet, selectorInfo.fieldIdx, selectorInfo.offset, selectorInfo.size);
- buildState.popSetName();
- }
- if (createMonitorText.length())
- {
- if (createValueSets)
- appendCtx->addQuotedF("%s->append(%s, %s);", buildState.listName, selectorInfo.getFFOptions(), createMonitorText.str());
- else
- appendCtx->addQuotedF("%s->append(%s);", buildState.listName, createMonitorText.str());
- }
- }
- IHqlExpression * MonitorExtractor::getMonitorValueAddress(BuildCtx & ctx, IHqlExpression * _value)
- {
- LinkedHqlExpr value = _value;
- CHqlBoundExpr bound;
- ITypeInfo * type = value->queryType();
- switch (type->getTypeCode())
- {
- case type_varstring: case type_varunicode:
- {
- assertex(type->getSize() != UNKNOWN_LENGTH);
- CHqlBoundTarget tempTarget;
- translator.createTempFor(ctx, type, tempTarget, typemod_none, FormatNatural);
- //clear the variable.
- HqlExprArray args;
- args.append(*getPointer(tempTarget.expr));
- args.append(*getZero());
- args.append(*getSizetConstant(type->getSize()));
- OwnedHqlExpr call = translator.bindTranslatedFunctionCall(memsetId, args);
- ctx.addExpr(call);
- //then assign over the top
- translator.buildExprAssign(ctx, tempTarget, value);
- bound.setFromTarget(tempTarget);
- break;
- }
- default:
- translator.buildExpr(ctx, value, bound);
- translator.ensureHasAddress(ctx, bound);
- break;
- }
- return getPointer(bound.expr);
- }
- bool MonitorExtractor::buildSingleKeyMonitor(StringBuffer & createMonitorText, KeySelectorInfo & selectorInfo, BuildCtx & ctx, IHqlExpression & thisKey)
- {
- BuildCtx subctx(ctx);
- OwnedHqlExpr compare, normalized;
- StringBuffer funcName;
- extractCompareInformation(subctx, &thisKey, compare, normalized, selectorInfo.expandedSelector);
- if (compare)
- return false;
- if (createValueSets)
- {
- StringBuffer type;
- translator.buildRtlFieldType(type, selectorInfo.selector->queryChild(1), queryRecord(selectorInfo.selector->queryChild(0)));
- //MORE: Need to ensure it is exactly the correct format - e.g. variable length strings are length prefixed
- OwnedHqlExpr address = getMonitorValueAddress(subctx, normalized);
- StringBuffer addrText;
- translator.generateExprCpp(addrText, address);
- createMonitorText.appendf("createFieldFilter(%u, %s, %s)", selectorInfo.fieldIdx, type.str(), addrText.str());
- }
- else
- {
- ITypeInfo * type = selectorInfo.expandedSelector->queryType();
- type_t tc = type->getTypeCode();
- if ((tc == type_int) || (tc == type_swapint))
- {
- if (isLittleEndian(type))
- {
- if (type->isSigned())
- funcName.append("createSingleLittleSignedKeySegmentMonitor");
- else if (type->getSize() != 1)
- funcName.append("createSingleLittleKeySegmentMonitor");
- }
- else
- {
- if (type->isSigned())
- funcName.append("createSingleBigSignedKeySegmentMonitor");
- else
- funcName.append("createSingleKeySegmentMonitor");
- }
- }
- if (!funcName.length())
- funcName.append("createSingleKeySegmentMonitor");
- OwnedHqlExpr address = getMonitorValueAddress(subctx, normalized);
- StringBuffer addrText;
- translator.generateExprCpp(addrText, address);
- createMonitorText.append(funcName)
- .appendf("(%s, %u, %u, %u, %s)",
- boolToText(selectorInfo.keyedKind != KeyedYes), selectorInfo.fieldIdx, selectorInfo.offset, selectorInfo.size, addrText.str());
- }
- return true;
- }
- KeyedKind getKeyedKind(HqlCppTranslator & translator, KeyConditionArray & matches)
- {
- KeyedKind keyedKind = KeyedNo;
- ForEachItemIn(i, matches)
- {
- KeyCondition & cur = matches.item(i);
- if (cur.keyedKind != keyedKind)
- {
- if (keyedKind == KeyedNo)
- keyedKind = cur.keyedKind;
- else
- translator.throwError1(HQLERR_InconsistentKeyedOpt, str(cur.selector->queryChild(1)->queryName()));
- }
- }
- return keyedKind;
- }
- void MonitorExtractor::buildEmptyKeySegment(BuildMonitorState & buildState, BuildCtx & ctx, KeySelectorInfo & selectorInfo)
- {
- StringBuffer s;
- if (createValueSets)
- {
- StringBuffer type;
- translator.buildRtlFieldType(type, selectorInfo.selector->queryChild(1), queryRecord(selectorInfo.selector->queryChild(0)));
- ctx.addQuoted(s.appendf("%s->append(%s, createEmptyFieldFilter(%u, %s));", buildState.listName, selectorInfo.getFFOptions(), selectorInfo.fieldIdx, type.str()));
- }
- else
- ctx.addQuoted(s.appendf("%s->append(createEmptyKeySegmentMonitor(%s, %u, %u, %u));", buildState.listName, boolToText(selectorInfo.keyedKind != KeyedYes), selectorInfo.fieldIdx, selectorInfo.offset, selectorInfo.size));
- }
- void MonitorExtractor::buildKeySegment(BuildMonitorState & buildState, BuildCtx & ctx, unsigned whichField, unsigned curSize)
- {
- IHqlExpression * selector = &keyableSelects.item(whichField);
- IHqlExpression * expandedSelector = &expandedSelects.item(whichField);
- IHqlExpression * field = selector->queryChild(1);
- KeyConditionArray matches;
- bool isImplicit = true;
- bool prevWildWasKeyed = buildState.wildWasKeyed;
- buildState.wildWasKeyed = false;
- ForEachItemIn(cond, keyed.conditions)
- {
- KeyCondition & cur = keyed.conditions.item(cond);
- if (cur.selector == selector)
- {
- cur.generated = true;
- if (cur.isWild)
- {
- isImplicit = false;
- if (cur.wasKeyed)
- buildState.wildWasKeyed = true;
- else if (buildState.implicitWildField && !ignoreUnkeyed)
- {
- StringBuffer s, keyname;
- translator.throwError3(HQLERR_WildFollowsGap, getExprECL(field, s).str(), str(buildState.implicitWildField->queryChild(1)->queryName()), queryKeyName(keyname));
- }
- }
- else
- {
- matches.append(OLINK(cur));
- if (buildState.implicitWildField && !ignoreUnkeyed)
- {
- StringBuffer s,keyname;
- if (cur.isKeyed())
- translator.throwError3(HQLERR_KeyedFollowsGap, getExprECL(field, s).str(), str(buildState.implicitWildField->queryChild(1)->queryName()), queryKeyName(keyname));
- else if (!buildState.doneImplicitWarning)
- {
- translator.WARNING3(CategoryEfficiency, HQLWRN_KeyedFollowsGap, getExprECL(field, s).str(), str(buildState.implicitWildField->queryChild(1)->queryName()), queryKeyName(keyname));
- buildState.doneImplicitWarning = true;
- }
- }
- }
- }
- }
- if (buildState.wildWasKeyed && (matches.ordinality() == 0))
- {
- StringBuffer keyname;
- translator.WARNING2(CategoryFolding, HQLWRN_FoldRemoveKeyed, str(field->queryName()), queryKeyName(keyname));
- }
- StringBuffer s;
- KeyedKind keyedKind = getKeyedKind(translator, matches);
- if (whichField >= firstOffsetField)
- translator.throwError1(HQLERR_KeyedNotKeyed, getExprECL(field, s).str());
- KeySelectorInfo selectorInfo(keyedKind, selector, expandedSelector, buildState.curFieldIdx, buildState.curOffset, curSize);
- bool ignoreKeyedExtend = false;
- if ((keyedKind == KeyedExtend) && buildState.wildPending() && !ignoreUnkeyed)
- {
- if (keyedKind == KeyedExtend)
- {
- if (prevWildWasKeyed)
- buildState.wildWasKeyed = true;
- else
- {
- StringBuffer keyname;
- translator.WARNING2(CategoryEfficiency, HQLERR_OptKeyedFollowsWild, getExprECL(field, s).str(), queryKeyName(keyname));
- }
- }
- //previous condition folded so always true, so keyed,opt will always be a wildcard.
- if (!allowDynamicFormatChange)
- ignoreKeyedExtend = true;
- isImplicit = false;
- }
- if (matches.ordinality() && !ignoreKeyedExtend)
- {
- if (buildState.wildPending() && !ignoreUnkeyed)
- buildState.clearWild();
- HqlExprArray args;
- ForEachItemIn(i, matches)
- {
- KeyCondition & cur = matches.item(i);
- args.append(*LINK(cur.expr));
- }
- OwnedHqlExpr fullExpr = createBalanced(no_and, queryBoolType(), args);
- BuildCtx subctx(ctx);
- buildKeySegmentExpr(buildState, selectorInfo, subctx, NULL, *fullExpr, ignoreUnkeyed ? MonitorFilterSkipAll : NoMonitorFilter);
- }
- else
- {
- if (isImplicit)
- {
- buildState.implicitWildField.set(selector);
- buildState.doneImplicitWarning = false;
- }
- if (buildState.wildPending() && noMergeSelects.contains(*selector))
- buildState.clearWild();
- if (!buildState.wildPending())
- buildState.wildOffset = buildState.curOffset;
- }
- buildState.curOffset += selectorInfo.size;
- buildState.curFieldIdx++;
- }
- void MonitorExtractor::spotSegmentCSE(BuildCtx & ctx)
- {
- //This could make things much better, but needs some thought
- HqlExprArray conditions;
- ForEachItemIn(cond, keyed.conditions)
- {
- KeyCondition & cur = keyed.conditions.item(cond);
- if (cur.expr)
- conditions.append(*LINK(cur.expr));
- }
- HqlExprArray associated;
- IHqlExpression * selector = tableExpr->queryNormalizedSelector();
- translator.traceExpressions("before seg spot", conditions);
- spotScalarCSE(conditions, associated, NULL, selector, translator.queryOptions().spotCseInIfDatasetConditions);
- translator.traceExpressions("after seg spot", conditions);
- unsigned curCond = 0;
- ForEachItemIn(i, conditions)
- {
- IHqlExpression * cur = &conditions.item(i);
- switch (cur->getOperator())
- {
- case no_alias:
- translator.buildStmt(ctx, cur);
- break;
- case no_alias_scope:
- translator.expandAliasScope(ctx, cur);
- cur = cur->queryChild(0);
- //fallthrough
- default:
- for (;;)
- {
- if (!keyed.conditions.isItem(curCond))
- throwUnexpected();
- KeyCondition & keyCond = keyed.conditions.item(curCond++);
- if (keyCond.expr)
- {
- keyCond.expr.set(cur);
- break;
- }
- }
- break;
- }
- }
- for (;;)
- {
- if (!keyed.conditions.isItem(curCond))
- break;
- KeyCondition & keyCond = keyed.conditions.item(curCond++);
- assertex(!keyCond.expr);
- }
- }
- void MonitorExtractor::buildSegments(BuildCtx & ctx, const char * listName, bool _ignoreUnkeyed)
- {
- translator.useInclude("rtlkey.hpp");
- ignoreUnkeyed = _ignoreUnkeyed;
- if (translator.queryOptions().spotCSE)
- spotSegmentCSE(ctx);
- BuildMonitorState buildState(ctx, listName);
- ForEachItemIn(idx, keyableSelects)
- {
- IHqlExpression * selector = &keyableSelects.item(idx);
- IHqlExpression * expandedSelector = &expandedSelects.item(idx);
- IHqlExpression * field = selector->queryChild(1);
- unsigned curSize = expandedSelector->queryType()->getSize();
- assertex(curSize != UNKNOWN_LENGTH);
- //MORE: Should also allow nested record structures, and allow keying on first elements.
- // and field->queryType()->getSize() doesn't work for alien datatypes etc.
- if(!field->hasAttribute(virtualAtom))
- {
- if (curSize)
- buildKeySegment(buildState, ctx, idx, curSize);
- else
- {
- ForEachItemIn(cond, keyed.conditions)
- {
- KeyCondition & cur = keyed.conditions.item(cond);
- if (cur.selector == selector)
- cur.generated = true;
- }
- }
- }
- }
- //check that all keyed entries have been matched
- ForEachItemIn(cond, keyed.conditions)
- {
- KeyCondition & cur = keyed.conditions.item(cond);
- if (!cur.generated)
- translator.throwError(HQLERR_OnlyKeyFixedField);
- }
- }
- bool MonitorExtractor::isKeySelect(IHqlExpression * select)
- {
- return (keyableSelects.find(*select) != NotFound);
- }
- bool MonitorExtractor::isEqualityFilter(IHqlExpression * search)
- {
- bool matched = false;
- ForEachItemIn(cond, keyed.conditions)
- {
- KeyCondition & cur = keyed.conditions.item(cond);
- if (cur.selector == search)
- {
- if (!cur.isWild)
- {
- if (matched)
- return false;
- matched = true;
- IHqlExpression * matchExpr = cur.expr;
- if (matchExpr->getOperator() != no_eq)
- return false;
- }
- }
- }
- return matched;
- }
- bool MonitorExtractor::isEqualityFilterBefore(IHqlExpression * select)
- {
- ForEachItemIn(i, keyableSelects)
- {
- IHqlExpression & cur = keyableSelects.item(i);
- if (select == &cur)
- return true;
- if (!isEqualityFilter(&cur))
- return false;
- }
- throwUnexpected();
- }
- bool MonitorExtractor::isPrevSelectKeyed(IHqlExpression * select)
- {
- unsigned match = keyableSelects.find(*select);
- assertex(match != NotFound);
- if (match == 0)
- return true;
- IHqlExpression * prev = &keyableSelects.item(match-1);
- ForEachItemIn(i, keyed.conditions)
- {
- KeyCondition & cur = keyed.conditions.item(i);
- if (cur.selector == prev)
- {
- if (!cur.isWild && cur.isKeyed())
- return true;
- if (cur.wasKeyed)
- return true;
- }
- }
- return false;
- }
- bool MonitorExtractor::okToKey(IHqlExpression * select, KeyedKind keyedKind)
- {
- if (keyedKind == KeyedYes)
- return true;
- ForEachItemIn(i, keyed.conditions)
- {
- KeyCondition & cur = keyed.conditions.item(i);
- if (cur.selector == select && cur.isWild)
- return false;
- }
- return true;
- }
- bool MonitorExtractor::isIndexInvariant(IHqlExpression * expr, bool includeRoot)
- {
- if (containsAssertKeyed(expr))
- return false;
- HqlExprCopyArray scopeUsed;
- expr->gatherTablesUsed(NULL, &scopeUsed);
- IHqlExpression * search = tableExpr->queryNormalizedSelector();
- ForEachItemIn(i, scopeUsed)
- {
- IHqlExpression * cur = &scopeUsed.item(i);
- for (;;)
- {
- if (cur == search)
- return false;
- if (includeRoot && (queryRoot(cur) == search))
- return false;
- IHqlExpression * parent = queryNextMultiLevelDataset(cur, true);
- if (!parent)
- break;
- cur = parent;
- }
- }
- return true;
- }
- IHqlExpression * MonitorExtractor::castToFieldAndBack(IHqlExpression * left, IHqlExpression * right)
- {
- node_operator op = left->getOperator();
- switch (op)
- {
- case no_cast:
- case no_implicitcast:
- {
- IHqlExpression * uncast = left->queryChild(0);
- ITypeInfo * castType = right->queryType();
- ITypeInfo * uncastType = uncast->queryType();
- OwnedHqlExpr castRight = ensureExprType(right, uncastType);
- OwnedHqlExpr base = castToFieldAndBack(uncast, castRight);
- //If this cast doesn't lose any information and child didn't change then don't bother
- //casting back and forwards.
- if ((base == castRight) && !castLosesInformation(uncastType, castType))
- return LINK(right);
- return ensureExprType(base, castType);
- }
- case no_select:
- {
- ITypeInfo * leftType = left->queryType();
- ITypeInfo * rightType = right->queryType();
- if (leftType == rightType || !castLosesInformation(leftType, rightType))
- return LINK(right);
- return ensureExprType(right, leftType);
- }
- case no_substring:
- case no_add:
- case no_sub:
- return castToFieldAndBack(left->queryChild(0), right);
- default:
- UNIMPLEMENTED;
- }
- }
- IHqlExpression * MonitorExtractor::invertTransforms(IHqlExpression * left, IHqlExpression * right)
- {
- node_operator op = left->getOperator();
- switch (op)
- {
- case no_cast:
- case no_implicitcast:
- {
- assertex(right->queryType()->getTypeCode() != type_set);
- IHqlExpression * uncast = left->queryChild(0);
- ITypeInfo * uncastType = uncast->queryType();
- OwnedHqlExpr castRight = ensureExprType(right, uncastType);
- return invertTransforms(uncast, castRight);
- }
- case no_select:
- {
- assertex(isKeySelect(left));
- ITypeInfo * leftType = left->queryType();
- ITypeInfo * rightType = right->queryType();
- if (leftType == rightType || !castLosesInformation(leftType, rightType))
- return LINK(right);
- return ensureExprType(right, leftType);
- }
- case no_add:
- case no_sub:
- {
- assertex(right->getOperator() != no_list);
- OwnedHqlExpr adjusted = createValue(op == no_sub ? no_add : no_sub, right->getType(), LINK(right), LINK(left->queryChild(1)));
- return invertTransforms(left->queryChild(0), adjusted);
- }
- case no_substring:
- {
- assertex(right->getOperator() != no_list);
- return invertTransforms(left->queryChild(0), right);
- }
- default:
- UNIMPLEMENTED;
- }
- }
- IHqlExpression * MonitorExtractor::queryKeyableSelector(IHqlExpression * expr)
- {
- switch (expr->getOperator())
- {
- case no_cast:
- case no_implicitcast:
- case no_add:
- case no_sub:
- return queryKeyableSelector(expr->queryChild(0));
- case no_select:
- if (isKeySelect(expr))
- return expr;
- return NULL;
- }
- return NULL;
- }
- IHqlExpression * MonitorExtractor::isKeyableFilter(IHqlExpression * left, IHqlExpression * right, bool & duplicate, node_operator compareOp, KeyFailureInfo & reason, KeyedKind keyedKind)
- {
- node_operator op = left->getOperator();
- switch (op)
- {
- case no_cast:
- case no_implicitcast:
- {
- IHqlExpression * uncast = left->queryChild(0);
- ITypeInfo * castType = left->queryType();
- ITypeInfo * uncastType = uncast->queryType();
- //(ty)x = y. E.g., (int1)string2field = int1value
- //if more than one value of x[uncastType] corresponds to a single value in y[castType] then we can't sensibly create
- //the key segment monitor. Because we will get false negatives. If it is an inverse then duplicate (see below)
- bool canRemoveCast = true;
- if (castLosesInformation(castType, uncastType))
- {
- if ((compareOp != no_ne) && (compareOp != no_notin))
- canRemoveCast = false;
- duplicate = true;
- }
- //if more than one value of y corresponds to a single value of x then need to duplicate the test condition.
- //or pretest whether (ty)(tx)y == y.
- //if (castLosesInformation(uncastType, castType))
- //Now taken care of when the segment monitors are created
- //If the comparison is non equality and the cast changes the collation sequence then you can't remove it.
- switch (compareOp)
- {
- case no_eq:
- case no_ne:
- case no_in:
- case no_notin:
- break;
- default:
- if (!preservesOrder(castType, uncastType))
- canRemoveCast = false;
- break;
- }
- Linked<ITypeInfo> newType = uncastType;
- if (right->queryType()->getTypeCode() == type_set)
- newType.setown(makeSetType(newType.getLink()));
- OwnedHqlExpr castRight = ensureExprType(right, newType);
- IHqlExpression * ret = isKeyableFilter(uncast, castRight, duplicate, compareOp, reason, keyedKind);
- if (canRemoveCast || !ret)
- return ret;
- reason.set(KFRcast, ret->queryChild(1));
- return NULL;
- }
- case no_select:
- if (isKeySelect(left) && okToKey(left, keyedKind))
- {
- if (isIndexInvariant(right, false))
- return left;
- reason.set(KFRtoocomplex, left);
- }
- else
- reason.set(KFRnokey);
- return NULL;
- case no_substring:
- {
- IHqlExpression * range = left->queryChild(1);
- if (range->getOperator() == no_rangeto)
- {
- IValue *end = range->queryChild(0)->queryValue();
- if (!end)
- break;
- return isKeyableFilter(left->queryChild(0), right, duplicate, compareOp, reason, keyedKind);
- }
- else if (range->getOperator() == no_range)
- {
- IValue *start = range->queryChild(0)->queryValue();
- IValue *end = range->queryChild(1)->queryValue();
- if (!start || !end || start->getIntValue() != 1)
- break;
- return isKeyableFilter(left->queryChild(0), right, duplicate, compareOp, reason, keyedKind);
- }
- reason.set(KFRtoocomplex, right);
- return NULL;
- }
- case no_add:
- case no_sub:
- if (isIndexInvariant(left->queryChild(1), false))
- return isKeyableFilter(left->queryChild(0), right, duplicate, compareOp, reason, keyedKind);
- reason.set(KFRtoocomplex, left);
- return NULL;
- }
- reason.set(KFRnokey);
- return NULL;
- }
- static IHqlExpression * getCompareValue(ITypeInfo * fieldType, unsigned subStringLen, IValue * value, int whichBoundary)
- {
- type_t ftc = fieldType->getTypeCode();
- unsigned fieldLen = fieldType->getStringLen();
- unsigned lenValue = value->queryType()->getStringLen();
- const void * rawValue = value->queryValue();
- size32_t resultLen;
- rtlDataAttr result;
- if (whichBoundary < 0)
- {
- switch (ftc)
- {
- case type_qstring:
- rtlCreateQStrRangeLow(resultLen, result.refstr(), fieldLen, subStringLen, lenValue, static_cast<const char *>(rawValue));
- break;
- case type_string:
- rtlCreateStrRangeLow(resultLen, result.refstr(), fieldLen, subStringLen, lenValue, static_cast<const char *>(rawValue));
- break;
- case type_data:
- rtlCreateDataRangeLow(resultLen, result.refdata(), fieldLen, subStringLen, lenValue, rawValue);
- break;
- case type_unicode:
- rtlCreateUnicodeRangeLow(resultLen, result.refustr(), fieldLen, subStringLen, lenValue, static_cast<const UChar *>(rawValue));
- break;
- default:
- //should this generate a warning/error instead?
- rtlCreateRange(resultLen, result.refstr(), fieldLen, subStringLen, fieldType->getSize(), static_cast<const char *>(rawValue), 0, 0);
- break;
- }
- }
- else
- {
- switch (ftc)
- {
- case type_qstring:
- rtlCreateQStrRangeHigh(resultLen, result.refstr(), fieldLen, subStringLen, lenValue, static_cast<const char *>(rawValue));
- break;
- case type_string:
- rtlCreateStrRangeHigh(resultLen, result.refstr(), fieldLen, subStringLen, lenValue, static_cast<const char *>(rawValue));
- break;
- case type_data:
- rtlCreateDataRangeHigh(resultLen, result.refdata(), fieldLen, subStringLen, lenValue, rawValue);
- break;
- case type_unicode:
- rtlCreateUnicodeRangeHigh(resultLen, result.refustr(), fieldLen, subStringLen, lenValue, static_cast<const UChar *>(rawValue));
- break;
- default:
- rtlCreateRange(resultLen, result.refstr(), fieldLen, subStringLen, fieldType->getSize(), static_cast<const char *>(rawValue), 255, 0);
- break;
- }
- }
- assertex(resultLen == fieldLen);
- return createConstant(createValueFromMem(LINK(fieldType), result.getdata()));
- }
- IHqlExpression * MonitorExtractor::getRangeLimit(ITypeInfo * fieldType, IHqlExpression * lengthExpr, IHqlExpression * value, int whichBoundary)
- {
- type_t ftc = fieldType->getTypeCode();
- unsigned fieldLength = fieldType->getStringLen();
- IValue * constValue = value->queryValue();
- if (constValue && lengthExpr->queryValue())
- {
- unsigned subStringLen = (unsigned)lengthExpr->queryValue()->getIntValue();
- if ((int)subStringLen < 0) subStringLen = 0;
- if (subStringLen > fieldLength)
- translator.throwError1(HQLERR_SubstringOutOfRange, subStringLen);
- return getCompareValue(fieldType, subStringLen, constValue, whichBoundary);
- }
- IIdAtom * func;
- if (whichBoundary < 0)
- {
- switch (ftc)
- {
- case type_qstring:
- func = createQStrRangeLowId;
- break;
- case type_string:
- func = createStrRangeLowId;
- break;
- case type_data:
- func = createDataRangeLowId;
- break;
- case type_unicode:
- func = createUnicodeRangeLowId;
- break;
- default:
- func = createRangeLowId;
- break;
- }
- }
- else
- {
- switch (ftc)
- {
- case type_qstring:
- func = createQStrRangeHighId;
- break;
- case type_string:
- func = createStrRangeHighId;
- break;
- case type_data:
- func = createDataRangeHighId;
- break;
- case type_unicode:
- func = createUnicodeRangeHighId;
- break;
- default:
- func = createRangeHighId;
- break;
- }
- }
- HqlExprArray args;
- args.append(*getSizetConstant(fieldLength));
- args.append(*LINK(lengthExpr));
- args.append(*LINK(value));
- //Note: I can't change the return type of the function - because if fixed length then wrong call is made, and variable length is worse code.
- OwnedHqlExpr call = translator.bindFunctionCall(func, args);
- return createValue(no_typetransfer, LINK(fieldType), LINK(call));
- }
- IHqlExpression * MonitorExtractor::createRangeCompare(IHqlExpression * selector, IHqlExpression * value, IHqlExpression * lengthExpr, bool compareEqual)
- {
- OwnedHqlExpr foldedValue = foldHqlExpression(value);
- ITypeInfo * fieldType = selector->queryType();
- IHqlExpression * lowExpr = getRangeLimit(fieldType, lengthExpr, foldedValue, -1);
- IHqlExpression * highExpr = getRangeLimit(fieldType, lengthExpr, foldedValue, +1);
- //Could convert to two separate tests, but code is worse, and boundary conditions aren't going to happen.
- return createValue(compareEqual ? no_between : no_notbetween, makeBoolType(), LINK(selector), lowExpr, highExpr);
- }
- static IHqlExpression * removeCastTrim(IHqlExpression * expr)
- {
- for (;;)
- {
- if ((expr->getOperator() == no_trim) && !expr->queryChild(1))
- expr = expr->queryChild(0);
- else if (isLengthPreservingCast(expr))
- expr = expr->queryChild(0);
- else
- return expr;
- expr = queryNonAliased(expr);
- }
- }
- static IHqlExpression * queryLengthFromRange(IHqlExpression * range)
- {
- switch (range->getOperator())
- {
- case no_rangefrom:
- case no_rangecommon:
- return NULL;
- case no_rangeto:
- return range->queryChild(0);
- case no_range:
- if (getIntValue(range->queryChild(0), 0) == 1)
- return range->queryChild(1);
- return NULL;
- default:
- if (getIntValue(range, 0) == 1)
- return range;
- return NULL;
- }
- }
- static void extendRangeCheck(SharedHqlExpr & globalGuard, SharedHqlExpr & localCond, IHqlExpression * selector, IHqlExpression * lengthExpr, bool compareEqual)
- {
- #if 0
- //This might be a good idea, but probably doesn't make a great deal of difference at runtime
- //Optimize the case where you check zero length to use a wild carded range instead
- //x[1..len=0] = y from x in range 0000000..FFFFFF to len==0 || x in range....
- if (compareEqual)
- {
- if (!lengthExpr->queryValue())
- {
- OwnedHqlExpr testLength = createBoolExpr(no_eq, LINK(lengthExpr), ensureExprType(queryZero(), lengthExpr->queryType()));
- localCond.setown(createBoolExpr(no_or, testLength.getClear(), LINK(localCond)));
- }
- }
- #endif
- //For a range check x[1..m] = y we generate
- //x in range(sizeof(x), m, y, 0) and range(maxlength(x), m, y, 255)
- //need to guard with condition len(trim(y)) <= m
- //If x[1..length(trim(y))] == y then don't add a condition
- IHqlExpression * cur = queryNonAliased(lengthExpr);
- IHqlExpression * compare = removeCastTrim(queryNonAliased(selector));
- if (cur->getOperator() == no_charlen)
- {
- cur = queryNonAliased(cur->queryChild(0));
- if (cur->getOperator() == no_trim)
- {
- cur = queryNonAliased(cur->queryChild(0));
- while (isLengthPreservingCast(cur))
- cur = queryNonAliased(cur->queryChild(0));
- IHqlExpression * compare = selector;
- for (;;)
- {
- compare = removeCastTrim(queryNonAliased(compare));
- if (cur->queryBody() == compare->queryBody())
- return;
- //Casts between strings that reduce/increase the number of characters don't matter as long as
- //they eventually match the search string
- if (!isCast(compare) || !isStringType(compare->queryType()))
- break;
- compare = compare->queryChild(0);
- if (!isStringType(compare->queryType()))
- break;
- }
- }
- }
- // if x[1..n] = z[1..n] then no need for a condition
- if (compare->getOperator() == no_substring)
- {
- IHqlExpression * range = queryLengthFromRange(compare->queryChild(1));
- if (range == lengthExpr)
- return;
- }
- // if x[1..n] = (string<m>y) where m<n then no need for condition
- unsigned selectorLength = selector->queryType()->getStringLen();
- if (selectorLength <= getIntValue(lengthExpr, 0))
- return;
- //otherwise, if x[1..y] == z then add check length(trim(z)) <= y
- OwnedHqlExpr trim = createTrimExpr(selector, NULL);
- OwnedHqlExpr len = createValue(no_charlen, LINK(unsignedType), LINK(trim));
- ITypeInfo * lengthType = lengthExpr->queryType();
- Owned<ITypeInfo> compareType = getPromotedECLCompareType(unsignedType, lengthType);
- node_operator compOp = compareEqual ? no_le : no_gt;
- OwnedHqlExpr positiveLen = createValue(no_maxlist, lengthExpr->getType(), createValue(no_list, makeSetType(LINK(lengthType)), LINK(lengthExpr), createConstant(lengthType->castFrom(false, I64C(0)))));
- OwnedHqlExpr test = createValue(no_le, makeBoolType(), ensureExprType(len, compareType), ensureExprType(positiveLen, compareType));
- test.setown(foldHqlExpression(test));
- if (compareEqual)
- extendConditionOwn(globalGuard, no_and, test.getClear());
- else
- extendConditionOwn(localCond, no_or, getInverse(test));
- }
- bool MonitorExtractor::matchSubstringFilter(KeyConditionInfo & matches, node_operator op, IHqlExpression * left, IHqlExpression * right, KeyedKind keyedKind, bool & duplicate)
- {
- LinkedHqlExpr value = right;
- duplicate = false;
- OwnedHqlExpr guard;
- ITypeInfo * guardCastType = NULL;
- if ((left->getOperator() == no_cast) || (left->getOperator() == no_implicitcast))
- {
- //code is extracted and simplified from isKeyableFilter() above - should be commoned up.
- IHqlExpression * uncast = left->queryChild(0);
- ITypeInfo * castType = left->queryType();
- ITypeInfo * uncastType = uncast->queryType();
- //(ty)x = y.
- //if more than one value of x[uncastType] corresponds to a single value in y[castType] then we can't sensibly create
- //the key segment monitor. Because we will get false negatives. If it is an inverse then duplicate (see below)
- bool canRemoveCast = true;
- if (castLosesInformation(castType, uncastType))
- canRemoveCast = false;
- //if more than one value of y corresponds to a single value of x then need to duplicate the test condition.
- if (!preservesOrder(castType, uncastType))
- canRemoveCast = false;
- if (!canRemoveCast)
- {
- // reason.set(KFRcast, ret->queryChild(1));
- return false;
- }
- if ((op != no_in) && (op != no_notin))
- {
- value.setown(ensureExprType(right, uncastType));
- //If a simple equality test then create a global guard to check that we aren't matching a false positive
- if (castLosesInformation(uncastType, castType))
- guard.setown(createBoolExpr(no_eq, ensureExprType(value, castType), LINK(right)));
- }
- else
- {
- //if an IN then add guards to each comparison - generated later...
- if (castLosesInformation(uncastType, castType))
- guardCastType = uncastType;
- else
- {
- Owned<ITypeInfo> targetType = makeSetType(LINK(uncastType));
- value.setown(ensureExprType(right, targetType));
- }
- }
- left = uncast;
- }
- if (left->getOperator() != no_substring)
- return false;
- if ((op == no_in) || (op == no_notin))
- {
- value.setown(normalizeListCasts(value));
- if (value->getOperator() != no_list)
- return false;
- }
- IHqlExpression * selector = left->queryChild(0);
- if (!isKeySelect(selector) || !okToKey(selector, keyedKind))
- return false;
- if (!isIndexInvariant(right, false))
- return false;
- ITypeInfo * fieldType = selector->queryType();
- unsigned fieldLength = fieldType->getStringLen();
- if (fieldLength == UNKNOWN_LENGTH)
- return false;
- OwnedHqlExpr range = foldHqlExpression(left->queryChild(1));
- IHqlExpression * lengthExpr = queryLengthFromRange(range);
- if (!lengthExpr)
- return false;
- OwnedHqlExpr newTest;
- if ((op == no_eq) || (op == no_ne))
- {
- newTest.setown(createRangeCompare(selector, value, lengthExpr, op == no_eq));
- extendRangeCheck(guard, newTest, right, lengthExpr, op == no_eq);
- }
- else //no_in, no_notin
- {
- HqlExprArray compares;
- ForEachChild(i, value)
- {
- IHqlExpression * cur = value->queryChild(i);
- LinkedHqlExpr castValue = cur;
- OwnedHqlExpr valueGuard;
- if (guardCastType)
- {
- castValue.setown(ensureExprType(castValue, guardCastType));
- valueGuard.setown(createBoolExpr(no_eq, ensureExprType(castValue, cur->queryType()), LINK(cur)));
- extendRangeCheck(valueGuard, valueGuard, cur, lengthExpr, (op == no_in));
- }
- OwnedHqlExpr cond = createRangeCompare(selector, castValue, lengthExpr, (op == no_in));
- if (valueGuard)
- cond.setown(createValue(no_and, makeBoolType(), valueGuard.getClear(), cond.getClear()));
- compares.append(*cond.getClear());
- }
- node_operator combineOp = (op == no_in) ? no_or : no_and;
- newTest.setown(createBalanced(combineOp, queryBoolType(), compares));
- }
-
- matches.appendCondition(*new KeyCondition(selector, newTest, keyedKind));
- if (guard)
- matches.appendPreFilter(guard);
- return true;
- }
- bool MonitorExtractor::extractSimpleCompareFilter(KeyConditionInfo & matches, IHqlExpression * expr, KeyedKind keyedKind)
- {
- OwnedHqlExpr promoted = getExplicitlyPromotedCompare(expr);
- IHqlExpression * l = promoted->queryChild(0);
- IHqlExpression * r = promoted->queryChild(1);
- bool duplicate = false;
- KeyFailureInfo reasonl, reasonr;
- node_operator op = expr->getOperator();
- IHqlExpression * matchedSelector = isKeyableFilter(l, r, duplicate, op, reasonl, keyedKind);
- Owned<KeyCondition> result;
- if (matchedSelector)
- {
- node_operator newOp = getModifiedOp(op, duplicate);
- if (newOp != no_none)
- {
- OwnedHqlExpr newFilter = createValue(newOp, expr->getType(), LINK(l), LINK(r));
- result.setown(new KeyCondition(matchedSelector, newFilter, keyedKind));
- }
- }
- else
- {
- duplicate = false;
- matchedSelector = isKeyableFilter(r, l, duplicate, op, reasonr, keyedKind);
- if (matchedSelector)
- {
- node_operator newOp = getModifiedOp(getReverseOp(op), duplicate);
- if (newOp != no_none)
- {
- OwnedHqlExpr newFilter = createValue(newOp, expr->getType(), LINK(r), LINK(l));
- result.setown(new KeyCondition(matchedSelector, newFilter, keyedKind));
- }
- }
- }
- bool extracted = (result != NULL);
- if (extracted)
- {
- matches.appendCondition(*result.getClear());
- }
- else
- {
- failReason.merge(reasonl);
- failReason.merge(reasonr);
- }
- if (duplicate || !extracted)
- matches.appendPostFilter(expr);
- return extracted;
- }
- bool MonitorExtractor::extractOrFilter(KeyConditionInfo & matches, IHqlExpression * expr, KeyedKind keyedKind)
- {
- HqlExprArray conds;
- expr->unwindList(conds, no_or);
- bool validOrFilter = true;
- HqlExprAttr invariant;
- CIArrayOf<KeyConditionInfo> branches;
- ForEachItemIn(idx, conds)
- {
- IHqlExpression & cur = conds.item(idx);
- if (isIndexInvariant(&cur, false))
- extendOrCondition(invariant, &cur);
- else
- {
- KeyConditionInfo & branch = * new KeyConditionInfo;
- branches.append(branch);
- //Can't generate an OR with a pure post-filter
- if (!extractFilters(branch, &cur, keyedKind))
- validOrFilter = false;
- }
- }
- //check all the conditions that are ORd together don't contain references to multiple fields.
- KeyCondition * firstBranch = NULL;
- bool multipleBranches = branches.ordinality() > 1;
- bool multipleSelectors = false;
- bool multipleConditions = false;
- bool hasPostFilter = false;
- ForEachItemIn(i1, branches)
- {
- KeyConditionInfo & branch = branches.item(i1);
- if (branch.postFilter)
- hasPostFilter = true;
- ForEachItemIn(i2, branch.conditions)
- {
- KeyCondition & cur = branch.conditions.item(i2);
- if (!firstBranch)
- firstBranch = &cur;
- else
- {
- multipleConditions = true;
- if (i1 == 0)
- {
- if (firstBranch->selector != cur.selector)
- multipleSelectors = true;
- }
- else
- {
- if ((firstBranch->selector != cur.selector) || multipleSelectors)
- validOrFilter = false;
- }
- }
- }
- }
- if (multipleBranches && hasPostFilter)
- validOrFilter = false;
- if (validOrFilter && firstBranch)
- {
- bool optimizeSingleBranch = true;
- if (multipleSelectors || hasPostFilter || (optimizeSingleBranch && !multipleConditions))
- {
- //Invariant ored with a conjunction
- //X or (A and B) -> (X or A) AND (X or B)
- assertex(branches.ordinality() == 1);
- KeyConditionInfo & branch = branches.item(0);
- OwnedHqlExpr preFilter = branch.preFilter ? extendCondition(no_or, invariant, branch.preFilter) : NULL;
- OwnedHqlExpr postFilter = branch.postFilter ? extendCondition(no_or, invariant, branch.postFilter) : NULL;
- matches.appendPreFilter(preFilter);
- matches.appendPostFilter(postFilter);
- ForEachItemIn(i2, branch.conditions)
- {
- KeyCondition & cur = branch.conditions.item(i2);
- OwnedHqlExpr filter = extendCondition(no_or, invariant, cur.expr);
- matches.conditions.append(*new KeyCondition(cur.selector, filter, keyedKind));
- }
- }
- else
- {
- LinkedHqlExpr combinedCondition = invariant;
- ForEachItemIn(i1, branches)
- {
- KeyConditionInfo & branch = branches.item(i1);
- OwnedHqlExpr conjunction = branch.createConjunction();
- extendOrCondition(combinedCondition, conjunction);
- }
-
- matches.conditions.append(*new KeyCondition(firstBranch->selector, combinedCondition, keyedKind));
- }
- return true;
- }
- else
- {
- matches.appendPostFilter(expr);
- KeyFailureInfo reason;
- reason.set(KFRor);
- failReason.merge(reason);
- return false;
- }
- }
- bool MonitorExtractor::extractIfFilter(KeyConditionInfo & matches, IHqlExpression * expr, KeyedKind keyedKind)
- {
- //MORE: This could generate better code, but I don't think it is worth the effort at the moment.
- //Really, I should analyse left and right. Iterate each selector referenced. If there are no post conditions then
- //generate IF(a, X, Y) compound expression, otherwise generate the default below.
- IHqlExpression * cond = expr->queryChild(0);
- if ((keyedKind != KeyedNo) && isIndexInvariant(cond, false))
- {
- //Convert IF(a, X, Y) to... IF (a, X, true) AND IF (a, true, Y) to... (NOT a OR X) AND (a OR Y)
- OwnedHqlExpr inverseCond = getInverse(cond);
- OwnedHqlExpr trueValue = createBoolExpr(no_or, LINK(inverseCond), LINK(expr->queryChild(1)));
- OwnedHqlExpr falseValue = createBoolExpr(no_or, LINK(cond), LINK(expr->queryChild(2)));
- OwnedHqlExpr combined = createBoolExpr(no_and, LINK(trueValue), LINK(falseValue));
- return extractFilters(matches, combined, keyedKind);
- }
- matches.appendPostFilter(expr);
- return false;
- }
- static HqlTransformerInfo selectSpotterInfo("SelectSpotter");
- MonitorExtractor::SelectSpotter::SelectSpotter(const HqlExprArray & _selects) : NewHqlTransformer(selectSpotterInfo), selects(_selects)
- {
- hasSelects = false;
- }
- void MonitorExtractor::SelectSpotter::analyseExpr(IHqlExpression * expr)
- {
- if (hasSelects || alreadyVisited(expr))
- return;
- if (selects.find(*expr) != NotFound)
- {
- hasSelects = true;
- return;
- }
- NewHqlTransformer::analyseExpr(expr);
- }
- bool MonitorExtractor::containsTableSelects(IHqlExpression * expr)
- {
- HqlExprCopyArray inScope;
- expr->gatherTablesUsed(NULL, &inScope);
- //Check that cursors for all inScope tables are already bound in the start context
- return inScope.find(*tableExpr->queryNormalizedSelector()) != NotFound;
- }
- void MonitorExtractor::extractFilters(IHqlExpression * expr, SharedHqlExpr & extraFilter)
- {
- HqlExprArray conds;
- expr->unwindList(conds, no_and);
- extractFilters(conds, extraFilter);
- }
- void MonitorExtractor::extractFilters(HqlExprArray & exprs, SharedHqlExpr & extraFilter)
- {
- OwnedHqlExpr savedFilter = keyed.postFilter.getClear();
- ForEachItemIn(i1, exprs)
- {
- IHqlExpression & cur = exprs.item(i1);
- switch (cur.getOperator())
- {
- case no_assertkeyed:
- case no_assertwild:
- extractFilters(keyed, &cur, KeyedNo);
- break;
- }
- }
- keyedExplicitly = (keyed.conditions.ordinality() != 0);
- cleanlyKeyedExplicitly = keyedExplicitly && !keyed.postFilter;
- ForEachItemIn(i2, exprs)
- {
- IHqlExpression & cur = exprs.item(i2);
- switch (cur.getOperator())
- {
- case no_assertkeyed:
- case no_assertwild:
- break;
- default:
- if (!keyedExplicitly)
- extractFilters(keyed, &cur, KeyedNo);
- else if (!cur.isAttribute() && isIndexInvariant(&cur, true))
- keyed.appendPreFilter(&cur);
- else
- keyed.appendPostFilter(&cur);
- break;
- }
- }
- extraFilter.set(keyed.postFilter);
- keyed.postFilter.setown(extendConditionOwn(no_and, savedFilter.getClear(), LINK(extraFilter)));
- }
- void MonitorExtractor::extractFiltersFromFilterDs(IHqlExpression * expr)
- {
- HqlExprArray conds;
- HqlExprAttr dummy;
- unwindFilterConditions(conds, expr);
- extractFilters(conds, dummy);
- }
- void MonitorExtractor::extractFoldedWildFields(IHqlExpression * expr)
- {
- node_operator op = expr->getOperator();
- switch (op)
- {
- case no_cast:
- case no_implicitcast:
- case no_add:
- case no_sub:
- //fields may have been transformed since folding...
- extractFoldedWildFields(expr->queryChild(0));
- break;
- case no_select:
- if (isKeySelect(expr))
- {
- KeyCondition * condition = new KeyCondition;
- condition->selector.set(expr);
- condition->isWild = true;
- condition->wasKeyed = true;
- keyed.conditions.append(*condition);
- }
- break;
- }
- }
- bool MonitorExtractor::extractBoolFieldFilter(KeyConditionInfo & matches, IHqlExpression * selector, KeyedKind keyedKind, bool compareValue)
- {
- if (selector->isBoolean())
- {
- if (isKeySelect(selector) && okToKey(selector, keyedKind))
- {
- OwnedHqlExpr newFilter = createValue(no_eq, makeBoolType(), LINK(selector), createConstant(compareValue));
- matches.appendCondition(*new KeyCondition(selector, newFilter, keyedKind));
- return true;
- }
- }
- return false;
- }
- bool MonitorExtractor::extractFilters(KeyConditionInfo & matches, IHqlExpression * expr, KeyedKind keyedKind)
- {
- if (!expr->isAttribute() && isIndexInvariant(expr, true))
- {
- extendAndCondition(matches.preFilter, expr);
- return true;
- }
- IHqlExpression *l = expr->queryChild(0);
- IHqlExpression *r = expr->queryChild(1);
- node_operator op = expr->getOperator();
- switch (op)
- {
- case no_and:
- {
- bool extracted = extractFilters(matches, l, keyedKind);
- if (!extractFilters(matches, r, keyedKind)) extracted = false;
- return extracted;
- }
- case no_or:
- return extractOrFilter(matches, expr, keyedKind);
- case no_attr:
- case no_attr_expr:
- case no_attr_link:
- return true;
- case no_not:
- {
- IHqlExpression * arg = expr->queryChild(0);
- OwnedHqlExpr inverse = getInverse(arg);
- if (inverse->queryBody() != expr->queryBody())
- return extractFilters(matches, inverse, keyedKind);
- if ((arg->getOperator() == no_select) && arg->isBoolean() &&
- extractBoolFieldFilter(matches, arg, keyedKind, false))
- return true;
- matches.appendPostFilter(expr);
- return false;
- }
- case no_between:
- case no_notbetween:
- {
- //Convert this into two comparisons because that will handle weird boundary conditions much better.
- OwnedHqlExpr normalized = expandBetween(expr);
- return extractFilters(matches, normalized, keyedKind);
- }
- case no_eq:
- case no_ne:
- {
- bool duplicate = false;
- if (matchSubstringFilter(matches, op, l, r, keyedKind, duplicate) || matchSubstringFilter(matches, op, r, l, keyedKind, duplicate))
- {
- if (duplicate)
- matches.appendPostFilter(expr);
- return true;
- }
- return extractSimpleCompareFilter(matches, expr, keyedKind);
- }
- case no_in:
- case no_notin:
- {
- bool duplicate = false;
- if (matchSubstringFilter(matches, op, l, r, keyedKind, duplicate))
- {
- if (duplicate)
- matches.appendPostFilter(expr);
- return true;
- }
- return extractSimpleCompareFilter(matches, expr, keyedKind);
- }
- case no_gt:
- case no_lt:
- case no_ge:
- case no_le:
- return extractSimpleCompareFilter(matches, expr, keyedKind);
- case no_assertkeyed:
- {
- KeyFailureInfo reason;
- reason.merge(failReason);
- failReason.clear();
- bool extend = expr->hasAttribute(extendAtom);
- if (!extractFilters(matches, l, extend ? KeyedExtend : KeyedYes))
- {
- if (!extend)
- failReason.reportError(translator, expr);
- }
- IHqlExpression * original = expr->queryAttribute(_selectors_Atom);
- if (original)
- {
- ForEachChild(i, original)
- extractFoldedWildFields(original->queryChild(i));
- }
- failReason.merge(reason);
- return true;
- }
- case no_assertwild:
- {
- if (l->getOperator() == no_all)
- {
- IHqlExpression * original = expr->queryAttribute(_selectors_Atom);
- assertex(original);
- ForEachChild(i, original)
- extractFoldedWildFields(original->queryChild(i));
- }
- else
- {
- IHqlExpression * selector = queryKeyableSelector(l);
- if (!selector)
- {
- StringBuffer keyname;
- translator.throwError1(HQLERR_WildNotReferenceIndex, queryKeyName(keyname));
- }
- KeyCondition * condition = new KeyCondition;
- condition->selector.set(selector);
- condition->isWild = true;
- matches.appendCondition(*condition);
- }
- return true;
- }
- case no_if:
- return extractIfFilter(matches, expr, keyedKind);
- case no_select:
- {
- if (expr->isBoolean() && extractBoolFieldFilter(matches, expr, keyedKind, true))
- return true;
- matches.appendPostFilter(expr);
- return false;
- }
- default:
- // Add this condition to the catchall expr
- matches.appendPostFilter(expr);
- return false;
- }
- }
- void MonitorExtractor::extractAllFilters(IHqlExpression * dataset)
- {
- for (;;)
- {
- switch (dataset->getOperator())
- {
- case no_newkeyindex:
- return;
- case no_filter:
- extractAllFilters(dataset->queryChild(0));
- extractFiltersFromFilterDs(dataset);
- return;
- case no_compound_indexread:
- case no_newusertable:
- case no_hqlproject:
- case no_distributed:
- case no_preservemeta:
- case no_unordered:
- case no_sorted:
- case no_stepped:
- case no_grouped:
- case no_alias_scope:
- case no_dataset_alias:
- break;
- default:
- UNIMPLEMENTED;
- }
- dataset = dataset->queryChild(0);
- }
- }
- bool MonitorExtractor::isKeyed()
- {
- ForEachItemIn(i, keyed.conditions)
- {
- if (!keyed.conditions.item(i).isWild)
- return true;
- }
- return false;
- }
- bool expandFilename(StringBuffer & s, IHqlExpression * expr)
- {
- switch (expr->getOperator())
- {
- case no_constant:
- expr->toString(s);
- return true;
- case no_getresult:
- //more
- break;
- case no_concat:
- {
- bool hadString = expandFilename(s, expr->queryChild(0));
- unsigned oldLength = s.length();
- s.append("+");
- if (expandFilename(s, expr->queryChild(1)) || hadString)
- return true;
- s.setLength(oldLength);
- return false;
- }
- case no_alias:
- case no_cast:
- case no_implicitcast:
- return expandFilename(s, expr->queryChild(0));
- }
- if (hasNamedSymbol(expr))
- {
- s.append(expr->queryName());
- return true;
- }
- s.append("...");
- return false;
- }
- const char * MonitorExtractor::queryKeyName(StringBuffer & s)
- {
- IAtom * name = tableExpr->queryName();
- if (name)
- s.append(" \'").append(name).append("'");
- else
- {
- IHqlExpression * filename = queryTableFilename(tableExpr);
- if (filename)
- {
- if (!expandFilename(s.append(' '), filename))
- s.clear();
- }
- }
- return s.str();
- }
- static bool isNextField(IHqlExpression * record, IHqlExpression * prevExpr, IHqlExpression * nextExpr)
- {
- if ((prevExpr->getOperator() != no_select) || (nextExpr->getOperator() != no_select))
- return false;
- IHqlExpression * prevSelect = prevExpr->queryChild(0);
- if (prevSelect != nextExpr->queryChild(0))
- return false;
- if (!containsOnlyLeft(prevExpr))
- return false;
- if (prevSelect->getOperator() != no_left)
- record = prevSelect->queryRecord();
- IHqlExpression * prevField = prevExpr->queryChild(1);
- IHqlExpression * nextField = nextExpr->queryChild(1);
- //Slow, but probably doesn't matter...
- ForEachChild(i, record)
- {
- IHqlExpression * cur = record->queryChild(i);
- if (cur == prevField)
- return (record->queryChild(i+1) == nextField);
- }
- return false;
- }
- IHqlExpression * MonitorExtractor::querySimpleJoinValue(IHqlExpression * selector)
- {
- IHqlExpression * matched = NULL;
- ForEachItemIn(cond, keyed.conditions)
- {
- KeyCondition & cur = keyed.conditions.item(cond);
- if (cur.selector == selector)
- {
- if (!cur.isWild)
- {
- if (matched)
- return NULL;
- IHqlExpression * matchExpr = cur.expr;
- if (matchExpr->getOperator() != no_eq)
- return NULL;
- if (matchExpr->queryChild(0) != selector)
- return NULL;
- matched = matchExpr->queryChild(1);
- }
- }
- }
- return matched;
- }
- //---------------------------------------------------------------------------
- static ABoundActivity * buildNullIndexActivity(HqlCppTranslator & translator, BuildCtx & ctx, IHqlExpression * expr)
- {
- while (isCompoundSource(expr))
- expr = expr->queryChild(0);
- return translator.buildCachedActivity(ctx, expr);
- }
- class IndexReadBuilderBase : public SourceBuilder
- {
- friend class MonitorRemovalTransformer;
- public:
- IndexReadBuilderBase(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr)
- : SourceBuilder(_translator, _tableExpr, _nameExpr), monitors(_tableExpr, _translator, -(int)numPayloadFields(_tableExpr), false)
- {
- }
- virtual void buildMembers(IHqlExpression * expr);
- virtual void extractMonitors(IHqlExpression * ds, SharedHqlExpr & unkeyedFilter, HqlExprArray & conds);
- protected:
- virtual void buildFlagsMember(IHqlExpression * expr);
- virtual void buildTransformFpos(BuildCtx & transformCtx)
- {
- associateFilePositions(transformCtx, "fpp", "left");
- }
- IHqlExpression * removeMonitors(IHqlExpression * expr);
- protected:
- MonitorExtractor monitors;
- SourceSteppingInfo steppingInfo;
- };
- void IndexReadBuilderBase::buildMembers(IHqlExpression * expr)
- {
- //---- virtual void createSegmentMonitors(struct IIndexReadContext *) { ... } ----
- {
- MemberFunction func(translator, instance->startctx, "virtual void createSegmentMonitors(IIndexReadContext *irc) override");
- monitors.buildSegments(func.ctx, "irc", false);
- }
- buildLimits(instance->startctx, expr, instance->activityId);
- if (!limitExpr && !keyedLimitExpr && !choosenValue && (instance->kind == TAKindexread || instance->kind == TAKindexnormalize) && !steppedExpr)
- {
- unsigned implicitLimit = translator.getDefaultImplicitIndexReadLimit();
- if (translator.checkIndexReadLimit())
- {
- StringBuffer keyname;
- if (implicitLimit)
- translator.WARNINGAT2(CategoryLimit, queryLocation(expr), HQLWRN_ImplicitReadAddLimit, implicitLimit, monitors.queryKeyName(keyname));
- else
- translator.WARNINGAT1(CategoryLimit, queryLocation(expr), HQLWRN_ImplicitReadLimit, monitors.queryKeyName(keyname));
- }
- if (implicitLimit)
- {
- OwnedHqlExpr limit = getSizetConstant(implicitLimit);
- translator.buildLimitHelpers(instance->startctx, limit, NULL, false, nameExpr, instance->activityId);
- }
- }
- instance->addAttributeBool("preload", isPreloaded);
- if (translator.getTargetClusterType() == RoxieCluster)
- instance->addAttributeBool("_isIndexOpt", tableExpr->hasAttribute(optAtom));
- if (monitors.queryGlobalGuard())
- translator.doBuildBoolFunction(instance->startctx, "canMatchAny", monitors.queryGlobalGuard());
- buildKeyedLimitHelper(expr);
- translator.buildFormatCrcFunction(instance->classctx, "getFormatCrc", true, tableExpr, tableExpr, 1);
- IHqlExpression * originalKey = queryAttributeChild(tableExpr, _original_Atom, 0);
- translator.buildSerializedLayoutMember(instance->classctx, originalKey->queryRecord(), "getIndexLayout", numKeyedFields(originalKey));
- //Note the helper base class contains code like the following
- //IThorIndexCallback * fpp;");
- //virtual void setCallback(IThorIndexCallback * _tc) { fpp = _tc; }");
- }
- void IndexReadBuilderBase::buildFlagsMember(IHqlExpression * expr)
- {
- StringBuffer flags;
- if (tableExpr->hasAttribute(sortedAtom))
- flags.append("|TIRsorted");
- else if (!isOrdered(tableExpr))
- flags.append("|TIRunordered");
- if (!monitors.isFiltered())
- flags.append("|TIRnofilter");
- if (isPreloaded)
- flags.append("|TIRpreload");
- if (tableExpr->hasAttribute(optAtom))
- flags.append("|TIRoptional");
- if (limitExpr && limitExpr->hasAttribute(skipAtom))
- flags.append("|TIRlimitskips");
- if (limitExpr && limitExpr->hasAttribute(onFailAtom))
- flags.append("|TIRlimitcreates");
- if (generateUnfilteredTransform)
- flags.append("|TIRunfilteredtransform");
- if (keyedLimitExpr)
- {
- if (keyedLimitExpr->hasAttribute(onFailAtom))
- flags.append("|TIRkeyedlimitcreates|TIRcountkeyedlimit");
- else if (keyedLimitExpr->hasAttribute(skipAtom))
- flags.append("|TIRkeyedlimitskips|TIRcountkeyedlimit");
- else if (keyedLimitExpr->hasAttribute(countAtom))
- flags.append("|TIRcountkeyedlimit");
- }
- IHqlExpression * firstStepped = steppingInfo.firstStepped();
- if (firstStepped && monitors.isEqualityFilterBefore(firstStepped))
- flags.append("|TIRstepleadequality");
- if (onlyExistsAggreate) flags.append("|TIRaggregateexists");
- if (monitorsForGrouping) flags.append("|TIRgroupmonitors");
- if (!nameExpr->isConstant()) flags.append("|TIRvarfilename");
- if (translator.hasDynamicFilename(tableExpr)) flags.append("|TIRdynamicfilename");
- if (requiresOrderedMerge) flags.append("|TIRorderedmerge");
- if (flags.length())
- translator.doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);
- }
- void IndexReadBuilderBase::extractMonitors(IHqlExpression * ds, SharedHqlExpr & unkeyedFilter, HqlExprArray & conds)
- {
- OwnedHqlExpr extraFilter;
- monitors.extractFilters(conds, extraFilter);
- appendFilter(unkeyedFilter, extraFilter);
- }
- class MonitorRemovalTransformer : public HqlMapTransformer
- {
- public:
- MonitorRemovalTransformer(IndexReadBuilderBase & _builder) : builder(_builder) {}
- virtual IHqlExpression * createTransformed(IHqlExpression * expr);
- protected:
- IndexReadBuilderBase & builder;
- };
- IHqlExpression * MonitorRemovalTransformer::createTransformed(IHqlExpression * expr)
- {
- switch (expr->getOperator())
- {
- case no_filter:
- {
- IHqlExpression * ds = expr->queryChild(0);
- IHqlExpression * body = expr->queryBody();
- unsigned match = builder.originalFilters.find(*body);
- if (match == NotFound)
- {
- if (builder.removedFilters.find(*body))
- return transform(ds);
- return NewHqlTransformer::createTransformed(expr);
- }
- IHqlExpression & newFilter = builder.mappedFilters.item(match);
- HqlExprArray args;
- args.append(*transform(ds));
- args.append(*transform(&newFilter));
- return expr->clone(args);
- }
- }
- return NewHqlTransformer::createTransformed(expr);
- }
- IHqlExpression * IndexReadBuilderBase::removeMonitors(IHqlExpression * expr)
- {
- MonitorRemovalTransformer mapper(*this);
- mapper.setMapping(tableExpr, tableExpr);
- return mapper.transformRoot(expr);
- }
- //---------------------------------------------------------------------------
- class NewIndexReadBuilder : public IndexReadBuilderBase
- {
- public:
- NewIndexReadBuilder(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr)
- : IndexReadBuilderBase(_translator, _tableExpr, _nameExpr)
- {
- }
- virtual void analyseGraph(IHqlExpression * expr)
- {
- IndexReadBuilderBase::analyseGraph(expr);
- gatherSteppingMeta(expr, steppingInfo);
- if (steppedExpr && transformCanFilter && translator.queryOptions().optimizeSteppingPostfilter)
- {
- //If sorted, but can't step output no point in duplicating the transform
- if (steppingInfo.outputStepping.exists())
- generateUnfilteredTransform = true;
- }
- returnIfFilterFails = !isNormalize;
- }
- virtual void buildTransform(IHqlExpression * expr);
- virtual void buildMembers(IHqlExpression * expr);
- };
- void NewIndexReadBuilder::buildMembers(IHqlExpression * expr)
- {
- if (steppingInfo.exists())
- {
- steppingInfo.checkKeyable(monitors);
- monitors.preventMerge(steppingInfo.firstStepped());
- }
- buildReadMembers(expr);
- if (steppingInfo.exists())
- steppingInfo.generateMembers(translator, instance->classctx);
- IndexReadBuilderBase::buildMembers(expr);
- }
- void NewIndexReadBuilder::buildTransform(IHqlExpression * expr)
- {
- if (true)
- {
- MemberFunction func(translator, instance->startctx, "virtual size32_t transform(ARowBuilder & crSelf, const void * _left) override");
- translator.ensureRowAllocated(func.ctx, "crSelf");
- func.ctx.addQuotedLiteral("unsigned char * left = (unsigned char *)_left;");
- translator.associateBlobHelper(func.ctx, tableExpr, "fpp");
- buildTransformBody(func.ctx, expr, true, false, true);
- }
- if (generateUnfilteredTransform)
- {
- MemberFunction func(translator, instance->startctx, "virtual size32_t unfilteredTransform(ARowBuilder & crSelf, const void * _left) override");
- translator.ensureRowAllocated(func.ctx, "crSelf");
- func.ctx.addQuotedLiteral("unsigned char * left = (unsigned char *)_left;");
- translator.associateBlobHelper(func.ctx, tableExpr, "fpp");
- buildTransformBody(func.ctx, expr, true, true, true);
- }
- }
- //---------------------------------------------------------------------------
- ABoundActivity * HqlCppTranslator::doBuildActivityIndexRead(BuildCtx & ctx, IHqlExpression * expr)
- {
- OwnedHqlExpr transformed = buildIndexFromPhysical(expr);
- OwnedHqlExpr optimized = optimizeHqlExpression(queryErrorProcessor(), transformed, HOOfold);
- IHqlExpression *tableExpr = queryPhysicalRootTable(optimized);
- //If the filter is false, then it may get reduced to a NULL operation!
- if (!tableExpr)
- return buildNullIndexActivity(*this, ctx, optimized);
- ensureDiskAccessAllowed(tableExpr);
- if (optimized->getOperator() != no_compound_indexread)
- optimized.setown(createDataset(no_compound_indexread, LINK(optimized)));
- traceExpression("before index read", optimized);
- assertex(tableExpr->getOperator() == no_newkeyindex);
- NewIndexReadBuilder info(*this, tableExpr, tableExpr->queryChild(3));
- info.gatherVirtualFields(true, true);
- if (info.containsStepping(optimized))
- return info.buildActivity(ctx, optimized, TAKindexread, "SteppedIndexRead", NULL);
- return info.buildActivity(ctx, optimized, TAKindexread, "IndexRead", NULL);
- }
- //---------------------------------------------------------------------------
- class IndexNormalizeBuilder : public IndexReadBuilderBase
- {
- public:
- IndexNormalizeBuilder(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr)
- : IndexReadBuilderBase(_translator, _tableExpr, _nameExpr)
- {
- }
- virtual void buildTransform(IHqlExpression * expr);
- virtual void buildMembers(IHqlExpression * expr);
- protected:
- virtual void analyseGraph(IHqlExpression * expr);
- virtual void processTransformSelect(BuildCtx & ctx, IHqlExpression * expr)
- {
- doBuildNormalizeIterators(ctx, expr, false);
- }
- };
- void IndexNormalizeBuilder::analyseGraph(IHqlExpression * expr)
- {
- IndexReadBuilderBase::analyseGraph(expr);
- needDefaultTransform = (expr->queryNormalizedSelector()->getOperator() == no_select);
- }
- void IndexNormalizeBuilder::buildMembers(IHqlExpression * expr)
- {
- buildFilenameMember();
- IndexReadBuilderBase::buildMembers(expr);
- buildNormalizeHelpers(expr);
- }
- void IndexNormalizeBuilder::buildTransform(IHqlExpression * expr)
- {
- globaliterctx.setown(new BuildCtx(instance->startctx));
- globaliterctx->addGroup();
- MemberFunction func(translator, instance->startctx, "virtual size32_t transform(ARowBuilder & crSelf) override");
- translator.ensureRowAllocated(func.ctx, "crSelf");
- //Because this transform creates iterator classes for the child iterators the expression tree needs to be modified
- //instead of using an inline tests. We could switch to using this all the time for indexes once I trust it!
- OwnedHqlExpr simplified = removeMonitors(expr);
- lastTransformer.set(queryExpression(simplified->queryDataset()->queryTable()));
- useFilterMappings=false;
- buildTransformBody(func.ctx, simplified, true, false, false);
- }
- //---------------------------------------------------------------------------
- ABoundActivity * HqlCppTranslator::doBuildActivityIndexNormalize(BuildCtx & ctx, IHqlExpression * expr)
- {
- OwnedHqlExpr transformed = buildIndexFromPhysical(expr);
- OwnedHqlExpr optimized = optimizeHqlExpression(queryErrorProcessor(), transformed, HOOfold);
- traceExpression("after optimize", optimized);
- IHqlExpression *tableExpr = queryPhysicalRootTable(optimized);
- if (!tableExpr)
- return buildNullIndexActivity(*this, ctx, optimized);
- ensureDiskAccessAllowed(tableExpr);
- assertex(tableExpr->getOperator() == no_newkeyindex);
- IndexNormalizeBuilder info(*this, tableExpr, tableExpr->queryChild(3));
- info.gatherVirtualFields(true, true);
- return info.buildActivity(ctx, optimized, TAKindexnormalize, "IndexNormalize", NULL);
- }
- //---------------------------------------------------------------------------
- class IndexAggregateBuilder : public IndexReadBuilderBase
- {
- public:
- IndexAggregateBuilder(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr)
- : IndexReadBuilderBase(_translator, _tableExpr, _nameExpr)
- {
- failedFilterValue.clear();
- }
- virtual void buildTransform(IHqlExpression * expr);
- virtual void buildMembers(IHqlExpression * expr);
- protected:
- virtual void processTransformSelect(BuildCtx & ctx, IHqlExpression * expr)
- {
- doBuildAggregateSelectIterator(ctx, expr);
- }
- virtual void analyseGraph(IHqlExpression * expr)
- {
- IndexReadBuilderBase::analyseGraph(expr);
- returnIfFilterFails = !isNormalize;
- }
- };
- void IndexAggregateBuilder::buildMembers(IHqlExpression * expr)
- {
- StringBuffer s;
- buildFilenameMember();
- IndexReadBuilderBase::buildMembers(expr);
- buildAggregateHelpers(expr);
- //virtual void processRow(void * self, const void * src) = 0;
- {
- BuildCtx rowctx(instance->startctx);
- rowctx.addQuotedFunction("virtual void processRow(ARowBuilder & crSelf, const void * src) override");
- rowctx.addQuotedLiteral("doProcessRow(crSelf, (const byte *)src);");
- }
- {
- IHqlExpression * record = tableExpr->queryRecord();
- OwnedHqlExpr newRecord = removeChild(record, record->numChildren()-1);
- OwnedHqlExpr newTableExpr = replaceChild(tableExpr, 1, newRecord);
- MemberFunction func(translator, instance->startctx, "virtual void processRows(ARowBuilder & crSelf, size32_t srcLen, const void * _left) override");
- func.ctx.addQuotedLiteral("unsigned char * left = (unsigned char *)_left;");
- OwnedHqlExpr ds = createVariable("left", makeReferenceModifier(newTableExpr->getType()));
- OwnedHqlExpr len = createVariable("srcLen", LINK(sizetType));
- OwnedHqlExpr fullDs = createTranslated(ds, len);
- Owned<IHqlCppDatasetCursor> iter = translator.createDatasetSelector(func.ctx, fullDs);
- BoundRow * curRow = iter->buildIterateLoop(func.ctx, false);
- s.clear().append("doProcessRow(crSelf, ");
- translator.generateExprCpp(s, curRow->queryBound());
- s.append(");");
- func.ctx.addQuoted(s);
- }
- }
- void IndexAggregateBuilder::buildTransform(IHqlExpression * expr)
- {
- MemberFunction func(translator, instance->startctx, "void doProcessRow(ARowBuilder & crSelf, const byte * left)");
- translator.ensureRowAllocated(func.ctx, "crSelf");
- translator.associateBlobHelper(func.ctx, tableExpr, "fpp");
- buildTransformBody(func.ctx, expr, false, false, true);
- }
- //---------------------------------------------------------------------------
- class IndexCountBuilder : public IndexReadBuilderBase
- {
- public:
- IndexCountBuilder(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr, node_operator _aggOp)
- : IndexReadBuilderBase(_translator, _tableExpr, _nameExpr)
- {
- aggOp = _aggOp;
- isCompoundCount = true;
- failedFilterValue.set(queryZero());
- }
- virtual void buildTransform(IHqlExpression * expr);
- virtual void buildMembers(IHqlExpression * expr);
- virtual bool isExplicitExists() { return (aggOp == no_existsgroup); }
- protected:
- virtual void processTransformSelect(BuildCtx & ctx, IHqlExpression * expr)
- {
- doBuildAggregateSelectIterator(ctx, expr);
- }
- virtual void analyseGraph(IHqlExpression * expr)
- {
- IndexReadBuilderBase::analyseGraph(expr);
- returnIfFilterFails = !isNormalize;
- IHqlExpression * aggregate = expr->queryChild(0);
- if (isKeyedCountAggregate(aggregate))
- {
- if (isNormalize)
- translator.throwError(HQLERR_KeyedCountCantNormalize);
- if (!monitors.isKeyedExplicitly())
- translator.throwError(HQLERR_KeyedCountNotKeyed);
- if (transformCanFilter)
- {
- ForEachItemIn(i, originalFilters)
- removedFilters.append(OLINK(originalFilters.item(i)));
- originalFilters.kill();
- mappedFilters.kill();
- transformCanFilter = false;
- }
- }
- if (aggOp == no_existsgroup)
- choosenValue.setown(getSizetConstant(1));
- }
- protected:
- node_operator aggOp;
- };
- void IndexCountBuilder::buildMembers(IHqlExpression * expr)
- {
- buildFilenameMember();
- IndexReadBuilderBase::buildMembers(expr);
- buildCountHelpers(expr, false);
- }
- void IndexCountBuilder::buildTransform(IHqlExpression * expr)
- {
- if (transformCanFilter||isNormalize)
- {
- MemberFunction func(translator, instance->startctx, "virtual size32_t numValid(const void * _left) override");
- func.ctx.addQuotedLiteral("unsigned char * left = (unsigned char *)_left;");
- translator.associateBlobHelper(func.ctx, tableExpr, "fpp");
- OwnedHqlExpr cnt;
- if (isNormalize)
- {
- compoundCountVar.setown(func.ctx.getTempDeclare(sizetType, queryZero()));
- cnt.set(compoundCountVar);
- }
- else
- cnt.setown(getSizetConstant(1));
- BuildCtx subctx(func.ctx);
- buildTransformBody(subctx, expr, false, false, true);
- func.ctx.addReturn(cnt);
- }
- }
- //---------------------------------------------------------------------------
- ABoundActivity * HqlCppTranslator::doBuildActivityIndexAggregate(BuildCtx & ctx, IHqlExpression * expr)
- {
- OwnedHqlExpr transformed = buildIndexFromPhysical(expr);
- OwnedHqlExpr optimized = optimizeHqlExpression(queryErrorProcessor(), transformed, getSourceAggregateOptimizeFlags());
- IHqlExpression *tableExpr = queryPhysicalRootTable(optimized);
- if (!tableExpr)
- return buildNullIndexActivity(*this, ctx, optimized);
- ensureDiskAccessAllowed(tableExpr);
- assertex(tableExpr->getOperator() == no_newkeyindex);
- node_operator aggOp = querySimpleAggregate(expr, true, false);
- if (aggOp == no_countgroup || aggOp == no_existsgroup)
- {
- IndexCountBuilder info(*this, tableExpr, tableExpr->queryChild(3), aggOp);
- info.gatherVirtualFields(true, true);
- return info.buildActivity(ctx, optimized, TAKindexcount, "IndexCount", NULL);
- }
- else
- {
- IndexAggregateBuilder info(*this, tableExpr, tableExpr->queryChild(3));
- info.gatherVirtualFields(true, true);
- return info.buildActivity(ctx, optimized, TAKindexaggregate, "IndexAggregate", NULL);
- }
- }
- //---------------------------------------------------------------------------
- class IndexGroupAggregateBuilder : public IndexReadBuilderBase
- {
- public:
- IndexGroupAggregateBuilder(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr)
- : IndexReadBuilderBase(_translator, _tableExpr, _nameExpr)
- {
- failedFilterValue.clear();
- transformAccessesCallback = false;
- }
- virtual void buildTransform(IHqlExpression * expr);
- virtual void buildMembers(IHqlExpression * expr);
- protected:
- void doBuildProcessCountMembers(BuildCtx & ctx, IHqlExpression * aggregate);
- virtual void processTransformSelect(BuildCtx & ctx, IHqlExpression * expr)
- {
- doBuildAggregateSelectIterator(ctx, expr);
- }
- virtual void analyseGraph(IHqlExpression * expr)
- {
- IndexReadBuilderBase::analyseGraph(expr);
- returnIfFilterFails = !isNormalize;
- }
- protected:
- bool transformAccessesCallback;
- };
- void IndexGroupAggregateBuilder::buildMembers(IHqlExpression * expr)
- {
- transformAccessesCallback = containsOperator(expr, no_filepos) || containsOperator(expr, no_id2blob);
- buildFilenameMember();
- buildGroupingMonitors(expr, monitors);
- IndexReadBuilderBase::buildMembers(expr);
- buildGlobalGroupAggregateHelpers(expr);
- if (!isNormalize && !transformCanFilter && monitorsForGrouping && !transformAccessesCallback)
- {
- IHqlExpression * aggregate = expr->queryChild(0);
- ThorActivityKind newKind = TAKnone;
- switch (querySingleAggregate(aggregate, false, true, true))
- {
- case no_countgroup:
- newKind = TAKindexgroupcount;
- break;
- case no_existsgroup:
- newKind = TAKindexgroupexists;
- break;
- }
- if (newKind)
- {
- instance->changeActivityKind(newKind);
- doBuildProcessCountMembers(instance->startctx, aggregate);
- }
- }
- }
- void IndexGroupAggregateBuilder::doBuildProcessCountMembers(BuildCtx & ctx, IHqlExpression * aggregate)
- {
- IHqlExpression * dataset = aggregate->queryChild(0);
- IHqlExpression * tgtRecord = aggregate->queryChild(1);
- IHqlExpression * transform = aggregate->queryChild(2);
- OwnedHqlExpr resultDataset = createDataset(no_anon, LINK(tgtRecord));
- {
- MemberFunction func(translator, ctx, "virtual size32_t initialiseCountGrouping(ARowBuilder & crSelf, const void * _src) override");
- translator.ensureRowAllocated(func.ctx, "crSelf");
- func.ctx.addQuotedLiteral("unsigned char * src = (unsigned char *) _src;");
- translator.associateBlobHelper(func.ctx, tableExpr, "fpp");
- BoundRow * selfCursor = translator.bindSelf(func.ctx, resultDataset, "crSelf");
- translator.bindTableCursor(func.ctx, dataset, "src");
- //Replace count() with 0, exists() with true and call as a transform - which will error if the replacement fails.
- OwnedHqlExpr count = createValue(no_countgroup, LINK(defaultIntegralType));
- OwnedHqlExpr exists = createValue(no_existsgroup, makeBoolType());
- OwnedHqlExpr newCount = createNullExpr(count);
- OwnedHqlExpr newTransform = replaceExpression(transform, count, newCount);
- newTransform.setown(replaceExpression(newTransform, exists, queryBoolExpr(true)));
- translator.doTransform(func.ctx, newTransform, selfCursor);
- translator.buildReturnRecordSize(func.ctx, selfCursor);
- }
- {
- MemberFunction func(translator, ctx, "virtual size32_t processCountGrouping(ARowBuilder & crSelf, unsigned __int64 count) override");
- translator.ensureRowAllocated(func.ctx, "crSelf");
- BoundRow * selfCursor = translator.bindSelf(func.ctx, resultDataset, "crSelf");
- OwnedHqlExpr newCount = createTranslatedOwned(createVariable("count", LINK(defaultIntegralType)));
- OwnedHqlExpr self = getSelf(aggregate);
- ForEachChild(idx, transform)
- {
- IHqlExpression * cur = transform->queryChild(idx);
- if (cur->isAttribute())
- continue;
- OwnedHqlExpr target = selfCursor->bindToRow(cur->queryChild(0), self);
- IHqlExpression * src = cur->queryChild(1);
- IHqlExpression * arg = queryRealChild(src, 0);
- BuildCtx condctx(func.ctx);
- node_operator srcOp = src->getOperator();
- switch (srcOp)
- {
- case no_countgroup:
- {
- if (arg)
- translator.buildFilter(condctx, arg);
- OwnedHqlExpr newValue = createValue(no_add, target->getType(), LINK(target), ensureExprType(newCount, target->queryType()));
- translator.buildAssign(condctx, target, newValue);
- }
- break;
- }
- }
- translator.buildReturnRecordSize(func.ctx, selfCursor);
- }
- }
- void IndexGroupAggregateBuilder::buildTransform(IHqlExpression * expr)
- {
- MemberFunction func(translator, instance->startctx, "void doProcessRow(const byte * left, IHThorGroupAggregateCallback * callback)");
- translator.associateBlobHelper(func.ctx, tableExpr, "fpp");
- buildGroupAggregateTransformBody(func.ctx, expr, isNormalize || transformAccessesCallback, true);
- }
- //---------------------------------------------------------------------------
- ABoundActivity * HqlCppTranslator::doBuildActivityIndexGroupAggregate(BuildCtx & ctx, IHqlExpression * expr)
- {
- OwnedHqlExpr transformed = buildIndexFromPhysical(expr);
- OwnedHqlExpr optimized = optimizeHqlExpression(queryErrorProcessor(), transformed, getSourceAggregateOptimizeFlags());
- IHqlExpression *tableExpr = queryPhysicalRootTable(optimized);
- if (!tableExpr)
- return buildNullIndexActivity(*this, ctx, optimized);
- ensureDiskAccessAllowed(tableExpr);
- IHqlExpression * aggregate = expr->queryChild(0);
- assertex(aggregate->getOperator() == no_newaggregate || aggregate->getOperator() == no_aggregate);
- ThorActivityKind tak = TAKindexgroupaggregate;
- assertex(tableExpr->getOperator() == no_newkeyindex);
- IndexGroupAggregateBuilder info(*this, tableExpr, tableExpr->queryChild(3));
- info.gatherVirtualFields(true, true);
- return info.buildActivity(ctx, optimized, tak, "IndexGroupAggregate", NULL);
- }
- //---------------------------------------------------------------------------
- void associateVirtualCallbacks(HqlCppTranslator & translator, BuildCtx & ctx, IHqlExpression * dataset)
- {
- OwnedHqlExpr fpos = getFilepos(dataset, false);
- OwnedHqlExpr lfpos = getFilepos(dataset, true);
- Owned<IHqlExpression> fposExpr = createFileposCall(translator, getFilePositionId, "fpp", "crSelf.row()");
- Owned<IHqlExpression> lfposExpr = createFileposCall(translator, getLocalFilePositionId, "fpp", "crSelf.row()");
- ctx.associateExpr(fpos, fposExpr);
- ctx.associateExpr(lfpos, lfposExpr);
- }
- void HqlCppTranslator::buildXmlReadTransform(IHqlExpression * dataset, StringBuffer & factoryName, bool & usesContents)
- {
- OwnedHqlExpr xmlMarker = createAttribute(xmlReadMarkerAtom, LINK(dataset->queryRecord()));
- BuildCtx declarectx(*code, declareAtom);
- HqlExprAssociation * match = declarectx.queryMatchExpr(xmlMarker);
- if (match)
- {
- IHqlExpression * matchExpr = match->queryExpr();
- matchExpr->queryChild(0)->queryValue()->getStringValue(factoryName);
- usesContents = matchExpr->queryChild(1)->queryValue()->getBoolValue();
- return;
- }
- StringBuffer s, id, className;
- getUniqueId(id);
- className.append("cx2r").append(id);
- const char * interfaceName = "IXmlToRowTransformer";
- StringBuffer prolog, epilog;
- prolog.append("struct ").append(className).append(" : public RtlCInterface, implements ").append(interfaceName);
- epilog.append(";");
- GlobalClassBuilder builder(*this, declarectx, className, "CXmlToRowTransformer", interfaceName);
- builder.buildClass(XmlTransformerPrio);
- builder.setIncomplete(true);
- BuildCtx & classctx = builder.classctx;
- s.clear().append("inline ").append(className).append("(unsigned _activityId) : CXmlToRowTransformer(_activityId) {}");
- classctx.addQuoted(s);
- {
- MemberFunction func(*this, classctx, "virtual size32_t transform(ARowBuilder & crSelf, IColumnProvider * row, IThorDiskCallback * fpp) override");
- ensureRowAllocated(func.ctx, "crSelf");
- xmlUsesContents = false;
- //MORE: If this becomes a compound activity
- BoundRow * rootSelfRow = bindSelf(func.ctx, dataset, "crSelf");
- bindXmlTableCursor(func.ctx, dataset, "row", no_none, NULL, true);
- OwnedHqlExpr activityId = createVariable("activityId", LINK(sizetType));
- func.ctx.associateExpr(queryActivityIdMarker(), activityId);
- associateVirtualCallbacks(*this, func.ctx, dataset);
- OwnedHqlExpr active = ensureActiveRow(dataset);
- buildAssign(func.ctx, rootSelfRow->querySelector(), active);
- buildReturnRecordSize(func.ctx, rootSelfRow);
- usesContents = xmlUsesContents;
- rootSelfRow = NULL;
- }
- buildMetaMember(classctx, dataset, false, "queryRecordSize");
- builder.setIncomplete(false);
- builder.completeClass(XmlTransformerPrio);
- factoryName.append(builder.accessorName);
- OwnedHqlExpr matchedValue = createAttribute(internalAtom, createConstant(factoryName.str()), createConstant(usesContents));
- declarectx.associateExpr(xmlMarker, matchedValue);
- }
- //---------------------------------------------------------------------------
- unsigned HqlCppTranslator::buildCsvReadTransform(BuildCtx & subctx, IHqlExpression * dataset, bool newInterface, IHqlExpression * csvAttr)
- {
- MemberFunction func(*this, subctx);
- if (newInterface)
- func.start("virtual size32_t transform(ARowBuilder & crSelf, unsigned * lenSrc, const char * * dataSrc) override");
- else
- func.start("virtual size32_t transform(ARowBuilder & crSelf, unsigned * lenSrc, const char * * dataSrc, unsigned __int64 _fpos) override");
- //MORE: If this becomes a compound activity
- BoundRow * rootSelfRow = bindSelf(func.ctx, dataset, "crSelf");
- bindCsvTableCursor(func.ctx, dataset, "Src", no_none, NULL, true, queryCsvEncoding(csvAttr));
- ensureRowAllocated(func.ctx, rootSelfRow);
- if (newInterface)
- {
- associateVirtualCallbacks(*this, func.ctx, dataset);
- }
- else
- {
- OwnedHqlExpr fpos = getFilepos(dataset, false);
- OwnedHqlExpr fposVar = createVariable("_fpos", fpos->getType());
- func.ctx.associateExpr(fpos, fposVar);
- }
- OwnedHqlExpr active = ensureActiveRow(dataset);
- buildAssign(func.ctx, rootSelfRow->querySelector(), active);
- buildReturnRecordSize(func.ctx, rootSelfRow);
- rootSelfRow = NULL;
- return countTotalFields(dataset->queryRecord(), false);
- }
- void HqlCppTranslator::buildCsvReadTransformer(IHqlExpression * dataset, StringBuffer & instanceName, IHqlExpression * optCsvAttr)
- {
- OwnedHqlExpr csvMarker = createAttribute(csvReadMarkerAtom, LINK(dataset->queryRecord()), LINK(optCsvAttr));
- BuildCtx declarectx(*code, declareAtom);
- HqlExprAssociation * match = declarectx.queryMatchExpr(csvMarker);
- if (match)
- {
- IHqlExpression * matchExpr = match->queryExpr();
- matchExpr->queryChild(0)->queryValue()->getStringValue(instanceName);
- return;
- }
- StringBuffer id, className;
- getUniqueId(id);
- instanceName.append("c2r").append(id);
- className.append("cc2r").append(id);
- StringBuffer prolog, epilog;
- prolog.append("struct ").append(className).append(" : public RtlCInterface, implements ICsvToRowTransformer");
- epilog.append(" ").append(instanceName).append(";");
- BuildCtx classctx(declarectx);
- classctx.setNextPriority(XmlTransformerPrio);
- IHqlStmt * transformClass = classctx.addQuotedCompound(prolog, epilog);
- transformClass->setIncomplete(true);
- transformClass->setIncluded(false); // if can't generate csv for this record, then don't generate an invalid class.
- classctx.addQuotedLiteral("virtual void Link() const override { RtlCInterface::Link(); }");
- classctx.addQuotedLiteral("virtual bool Release() const override { return RtlCInterface::Release(); }");
- unsigned maxColumns = buildCsvReadTransform(classctx, dataset, false, optCsvAttr);
- doBuildUnsignedFunction(classctx, "getMaxColumns", maxColumns);
- buildMetaMember(classctx, dataset, false, "queryRecordSize");
- buildCsvParameters(classctx, optCsvAttr, NULL, true);
- transformClass->setIncomplete(false);
- transformClass->setIncluded(true);
- if (options.spanMultipleCpp)
- {
- StringBuffer helperFunc;
- createAccessFunctions(helperFunc, declarectx, XmlTransformerPrio, "ICsvToRowTransformer", instanceName.str());
- instanceName.clear().append(helperFunc).append("()");
- }
- OwnedHqlExpr matchedValue = createAttribute(internalAtom, createConstant(instanceName.str()));
- declarectx.associateExpr(csvMarker, matchedValue);
- }
- ABoundActivity * HqlCppTranslator::doBuildActivityXmlRead(BuildCtx & ctx, IHqlExpression * expr)
- {
- IHqlExpression * tableExpr = expr;
- ensureDiskAccessAllowed(tableExpr);
- IHqlExpression * filename = tableExpr->queryChild(0);
- IHqlExpression * mode = tableExpr->queryChild(2);
- node_operator modeType = mode->getOperator();
- StringBuffer s;
- ThorActivityKind kind = (modeType == no_json) ? TAKjsonread : TAKxmlread;
- Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, kind, expr, "XmlRead");
- buildActivityFramework(instance);
- buildInstancePrefix(instance);
- //MORE: Improve when we support projecting xml instead of reading all
- SourceFieldUsage * fieldUsage = querySourceFieldUsage(tableExpr);
- if (fieldUsage && !fieldUsage->seenAll())
- fieldUsage->noteAll();
- //---- virtual const char * getFileName() { return "x.d00"; } ----
- buildFilenameFunction(*instance, instance->startctx, "getFileName", filename, hasDynamicFilename(tableExpr));
- buildEncryptHelper(instance->startctx, tableExpr->queryAttribute(encryptAtom));
- bool usesContents = false;
- doBuildXmlReadMember(*instance, tableExpr, "queryTransformer", usesContents);
- doBuildVarStringFunction(instance->classctx, "getXmlIteratorPath", queryRealChild(mode, 0));
- buildMetaMember(instance->classctx, tableExpr, false, "queryDiskRecordSize"); // A lie, but I don't care....
- buildMetaMember(instance->classctx, tableExpr, false, "queryProjectedDiskRecordSize"); // A lie, but I don't care....
- //virtual unsigned getFlags() = 0;
- StringBuffer flags;
- if (expr->hasAttribute(_spill_Atom)) flags.append("|TDXtemporary");
- if (expr->hasAttribute(unsortedAtom)) flags.append("|TDRunsorted");
- if (expr->hasAttribute(optAtom)) flags.append("|TDRoptional");
- if (usesContents) flags.append("|TDRusexmlcontents");
- if (mode->hasAttribute(noRootAtom)) flags.append("|TDRxmlnoroot");
- if (!filename->isConstant()) flags.append("|TDXvarfilename");
- if (hasDynamicFilename(expr)) flags.append("|TDXdynamicfilename");
- if (flags.length())
- doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);
- //Note the helper base class contains code like the following
- //IThorDiskCallback * fpp;");
- //virtual void setCallback(IThorDiskCallback * _tc) { fpp = _tc; }");
- buildInstanceSuffix(instance);
- addFileDependency(filename, instance->queryBoundActivity());
- return instance->getBoundActivity();
- }
- //---------------------------------------------------------------------------
- ABoundActivity * HqlCppTranslator::doBuildActivityTable(BuildCtx & ctx, IHqlExpression * expr)
- {
- node_operator mode = expr->queryChild(2)->getOperator();
- switch (mode)
- {
- case no_thor:
- case no_flat:
- case no_pipe:
- case no_csv:
- return doBuildActivityDiskRead(ctx, expr);
- case no_xml:
- case no_json:
- return doBuildActivityXmlRead(ctx, expr);
- default:
- UNIMPLEMENTED;
- }
- }
- //---------------------------------------------------------------------------
- class FetchBuilder : public SourceBuilder
- {
- public:
- FetchBuilder(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr, IHqlExpression * _fetchExpr)
- : SourceBuilder(_translator, _tableExpr, _nameExpr)
- {
- compoundExpr.set(_fetchExpr);
- fetchExpr.set(queryFetch(_fetchExpr));
- selSeq.set(querySelSeq(fetchExpr));
- fetchRhs = fetchExpr->queryChild(1);
- memoryRhsRecord = fetchRhs->queryRecord();
- serializedRhsRecord.setown(getSerializedForm(memoryRhsRecord, diskAtom));
- }
- virtual void buildMembers(IHqlExpression * expr);
- virtual void buildTransform(IHqlExpression * expr);
- virtual void buildTransformFpos(BuildCtx & transformCtx);
- protected:
- HqlExprAttr compoundExpr;
- HqlExprAttr fetchExpr;
- HqlExprAttr selSeq;
- HqlExprAttr serializedRhsRecord;
- IHqlExpression * fetchRhs;
- IHqlExpression * memoryRhsRecord;
- };
- void FetchBuilder::buildMembers(IHqlExpression * expr)
- {
- buildFilenameMember();
- IHqlExpression * fetch = queryFetch(expr);
- {
- MemberFunction func(translator, instance->startctx, "virtual unsigned __int64 extractPosition(const void * _right) override");
- func.ctx.addQuotedLiteral("const unsigned char * right = (const unsigned char *) _right;");
- translator.bindTableCursor(func.ctx, fetch->queryChild(1), "right", no_right, selSeq);
- translator.buildReturn(func.ctx, fetch->queryChild(2));
- }
-
- translator.buildEncryptHelper(instance->startctx, tableExpr->queryAttribute(encryptAtom), "getFileEncryptKey");
- //Fetch flags
- StringBuffer flags;
- if (tableExpr->hasAttribute(optAtom))
- flags.append("|FFdatafileoptional");
- if (!nameExpr->isConstant())
- flags.append("|FFvarfilename");
- if (translator.hasDynamicFilename(tableExpr))
- flags.append("|FFdynamicfilename");
- if (flags.length())
- translator.doBuildUnsignedFunction(instance->classctx, "getFetchFlags", flags.str()+1);
- if (tableExpr->hasAttribute(optAtom) && translator.targetRoxie())
- instance->addAttributeBool("_isOpt", true);
- buildLimits(instance->startctx, expr, instance->activityId);
- switch (getDatasetKind(tableExpr))
- {
- case no_csv:
- {
- translator.buildCsvParameters(instance->nestedctx, tableExpr->queryChild(2), NULL, true);
- unsigned maxColumns = getFieldCount(tableExpr->queryRecord());
- StringBuffer s;
- s.clear().append("virtual unsigned getMaxColumns() override { return ").append(maxColumns).append("; }");
- instance->classctx.addQuoted(s);
- break;
- }
- case no_xml:
- case no_json:
- break;
- default:
- translator.buildFormatCrcFunction(instance->classctx, "getDiskFormatCrc", false, physicalRecord, NULL, 0);
- break;
- }
- if (!containsOnlyLeft(fetch->queryChild(3), true))
- {
- //MORE: Need to change following if we optimize it to only extract the relevant fields.
- instance->classctx.addQuotedLiteral("virtual bool extractAllJoinFields() override { return true; }");
- {
- MemberFunction func(translator, instance->startctx, "virtual size32_t extractJoinFields(ARowBuilder & crSelf, const void * _left) override");
- translator.ensureRowAllocated(func.ctx, "crSelf");
- translator.buildRecordSerializeExtract(func.ctx, memoryRhsRecord);
- }
- StringBuffer s;
- MetaInstance meta(translator, serializedRhsRecord, false);
- translator.buildMetaInfo(meta);
- instance->classctx.addQuoted(s.clear().append("virtual IOutputMetaData * queryExtractedSize() override { return &").append(meta.queryInstanceObject()).append("; }"));
- }
- }
- void FetchBuilder::buildTransform(IHqlExpression * expr)
- {
- translator.xmlUsesContents = false;
- MemberFunction func(translator, instance->startctx);
- switch (getDatasetKind(tableExpr))
- {
- case no_csv:
- func.start("virtual size32_t transform(ARowBuilder & crSelf, unsigned * lenLeft, const char * * dataLeft, const void * _right, unsigned __int64 _fpos) override");
- func.ctx.addQuotedLiteral("unsigned char * right = (unsigned char *)_right;");
- break;
- case no_xml:
- case no_json:
- func.start("virtual size32_t transform(ARowBuilder & crSelf, IColumnProvider * xmlLeft, const void * _right, unsigned __int64 _fpos) override");
- func.ctx.addQuotedLiteral("unsigned char * right = (unsigned char *)_right;");
- break;
- default:
- func.start("virtual size32_t transform(ARowBuilder & crSelf, const void * _left, const void * _right, unsigned __int64 _fpos) override");
- func.ctx.addQuotedLiteral("unsigned char * left = (unsigned char *)_left;");
- func.ctx.addQuotedLiteral("unsigned char * right = (unsigned char *)_right;");
- break;
- }
- translator.ensureRowAllocated(func.ctx, "crSelf");
- buildTransformBody(func.ctx, expr, true, false, true);
- if (translator.xmlUsesContents)
- instance->classctx.addQuotedLiteral("virtual bool requiresContents() override { return true; }");
- }
- void FetchBuilder::buildTransformFpos(BuildCtx & transformCtx)
- {
- fpos.setown(createVariable("_fpos", LINK(fposType)));
- //NB: Because the fetch gets merged with the usertable used to project the dataset, the
- //transform contains filepos(LEFT) not filepos(tableExpr)
- OwnedHqlExpr leftSelect = createSelector(no_left, fetchExpr->queryChild(0), selSeq);
- OwnedHqlExpr fposField = getFilepos(leftSelect, false);
- transformCtx.associateExpr(fposField, fpos);
- //MORE: Could possibly support virtual(filename) here
- }
- static HqlTransformerInfo fetchInputReplacerInfo("FetchInputReplacer");
- class FetchInputReplacer : public NewHqlTransformer
- {
- public:
- FetchInputReplacer(IHqlExpression * _newDataset, node_operator side)
- : NewHqlTransformer(fetchInputReplacerInfo)
- {
- newDataset = _newDataset;
- child = (side == no_left) ? 0 : 1;
- }
- virtual IHqlExpression * createTransformed(IHqlExpression * expr)
- {
- if (expr->getOperator() == no_fetch)
- return replaceChild(expr, child, newDataset);
- return NewHqlTransformer::createTransformed(expr);
- }
- protected:
- IHqlExpression * newDataset;
- unsigned child;
- };
- IHqlExpression * replaceFetchInput(IHqlExpression * expr, IHqlExpression * newDataset, node_operator side)
- {
- FetchInputReplacer replacer(newDataset, side);
- return replacer.transformRoot(expr);
- }
- ABoundActivity * HqlCppTranslator::doBuildActivityFetch(BuildCtx & ctx, IHqlExpression * expr)
- {
- IHqlExpression *fetch = queryFetch(expr);
- IHqlExpression *tableExpr = queryPhysicalRootTable(fetch->queryChild(0));
- if (!tableExpr)
- throwError(HQLERR_FetchNonDiskfile);
- FetchBuilder info(*this, tableExpr, tableExpr->queryChild(0), expr);
- info.gatherVirtualFields(false, true);//?needToSerializeRecord(mode)
- unsigned optFlags = (options.foldOptimized ? HOOfold : 0);
- if (info.recordHasVirtualsOrDeserialize())
- {
- OwnedHqlExpr projected = createTableWithoutVirtuals(info.fieldInfo, tableExpr);
- //Nasty: We don't want to optimize the rhs, otherwise references get changed!
- //so optimize everything except the rhs, and then add the rhs back in again.
- IHqlExpression * fetchRhs = fetch->queryChild(1);
- OwnedHqlExpr null = createDataset(no_anon, LINK(fetchRhs->queryRecord()));
- OwnedHqlExpr simple = replaceFetchInput(expr, null, no_right);
- OwnedHqlExpr transformed = replaceExpression(simple, tableExpr, projected);
- OwnedHqlExpr optSimple = optimizeHqlExpression(queryErrorProcessor(), transformed, optFlags);
- OwnedHqlExpr optimized = replaceFetchInput(optSimple, fetchRhs, no_right);
- return doBuildActivityFetch(ctx, optimized);
- }
- if (getProjectCount(expr) > 1)
- {
- OwnedHqlExpr optimized = optimizeHqlExpression(queryErrorProcessor(), expr, optFlags);
- return doBuildActivityFetch(ctx, optimized);
- }
- Owned<ABoundActivity> childActivity = buildCachedActivity(ctx, fetch->queryChild(1));
- node_operator kind = getDatasetKind(tableExpr);
- switch (kind)
- {
- case no_csv:
- return info.buildActivity(ctx, expr, TAKcsvfetch, "CsvFetch", childActivity);
- case no_xml:
- return info.buildActivity(ctx, expr, TAKxmlfetch, "XmlFetch", childActivity);
- case no_json:
- //Note use of "XmlFetch" because we want the code generator to leverage existing xml classes
- return info.buildActivity(ctx, expr, TAKjsonfetch, "XmlFetch", childActivity);
- case no_flat:
- case no_thor:
- return info.buildActivity(ctx, expr, TAKfetch, "Fetch", childActivity);
- }
- throwError1(HQLERR_FetchNotSupportMode, getOpString(kind));
- return NULL;
- }
|