hqlttcpp.cpp 496 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853685468556856685768586859686068616862686368646865686668676868686968706871687268736874687568766877687868796880688168826883688468856886688768886889689068916892689368946895689668976898689969006901690269036904690569066907690869096910691169126913691469156916691769186919692069216922692369246925692669276928692969306931693269336934693569366937693869396940694169426943694469456946694769486949695069516952695369546955695669576958695969606961696269636964696569666967696869696970697169726973697469756976697769786979698069816982698369846985698669876988698969906991699269936994699569966997699869997000700170027003700470057006700770087009701070117012701370147015701670177018701970207021702270237024702570267027702870297030703170327033703470357036703770387039704070417042704370447045704670477048704970507051705270537054705570567057705870597060706170627063706470657066706770687069707070717072707370747075707670777078707970807081708270837084708570867087708870897090709170927093709470957096709770987099710071017102710371047105710671077108710971107111711271137114711571167117711871197120712171227123712471257126712771287129713071317132713371347135713671377138713971407141714271437144714571467147714871497150715171527153715471557156715771587159716071617162716371647165716671677168716971707171717271737174717571767177717871797180718171827183718471857186718771887189719071917192719371947195719671977198719972007201720272037204720572067207720872097210721172127213721472157216721772187219722072217222722372247225722672277228722972307231723272337234723572367237723872397240724172427243724472457246724772487249725072517252725372547255725672577258725972607261726272637264726572667267726872697270727172727273727472757276727772787279728072817282728372847285728672877288728972907291729272937294729572967297729872997300730173027303730473057306730773087309731073117312731373147315731673177318731973207321732273237324732573267327732873297330733173327333733473357336733773387339734073417342734373447345734673477348734973507351735273537354735573567357735873597360736173627363736473657366736773687369737073717372737373747375737673777378737973807381738273837384738573867387738873897390739173927393739473957396739773987399740074017402740374047405740674077408740974107411741274137414741574167417741874197420742174227423742474257426742774287429743074317432743374347435743674377438743974407441744274437444744574467447744874497450745174527453745474557456745774587459746074617462746374647465746674677468746974707471747274737474747574767477747874797480748174827483748474857486748774887489749074917492749374947495749674977498749975007501750275037504750575067507750875097510751175127513751475157516751775187519752075217522752375247525752675277528752975307531753275337534753575367537753875397540754175427543754475457546754775487549755075517552755375547555755675577558755975607561756275637564756575667567756875697570757175727573757475757576757775787579758075817582758375847585758675877588758975907591759275937594759575967597759875997600760176027603760476057606760776087609761076117612761376147615761676177618761976207621762276237624762576267627762876297630763176327633763476357636763776387639764076417642764376447645764676477648764976507651765276537654765576567657765876597660766176627663766476657666766776687669767076717672767376747675767676777678767976807681768276837684768576867687768876897690769176927693769476957696769776987699770077017702770377047705770677077708770977107711771277137714771577167717771877197720772177227723772477257726772777287729773077317732773377347735773677377738773977407741774277437744774577467747774877497750775177527753775477557756775777587759776077617762776377647765776677677768776977707771777277737774777577767777777877797780778177827783778477857786778777887789779077917792779377947795779677977798779978007801780278037804780578067807780878097810781178127813781478157816781778187819782078217822782378247825782678277828782978307831783278337834783578367837783878397840784178427843784478457846784778487849785078517852785378547855785678577858785978607861786278637864786578667867786878697870787178727873787478757876787778787879788078817882788378847885788678877888788978907891789278937894789578967897789878997900790179027903790479057906790779087909791079117912791379147915791679177918791979207921792279237924792579267927792879297930793179327933793479357936793779387939794079417942794379447945794679477948794979507951795279537954795579567957795879597960796179627963796479657966796779687969797079717972797379747975797679777978797979807981798279837984798579867987798879897990799179927993799479957996799779987999800080018002800380048005800680078008800980108011801280138014801580168017801880198020802180228023802480258026802780288029803080318032803380348035803680378038803980408041804280438044804580468047804880498050805180528053805480558056805780588059806080618062806380648065806680678068806980708071807280738074807580768077807880798080808180828083808480858086808780888089809080918092809380948095809680978098809981008101810281038104810581068107810881098110811181128113811481158116811781188119812081218122812381248125812681278128812981308131813281338134813581368137813881398140814181428143814481458146814781488149815081518152815381548155815681578158815981608161816281638164816581668167816881698170817181728173817481758176817781788179818081818182818381848185818681878188818981908191819281938194819581968197819881998200820182028203820482058206820782088209821082118212821382148215821682178218821982208221822282238224822582268227822882298230823182328233823482358236823782388239824082418242824382448245824682478248824982508251825282538254825582568257825882598260826182628263826482658266826782688269827082718272827382748275827682778278827982808281828282838284828582868287828882898290829182928293829482958296829782988299830083018302830383048305830683078308830983108311831283138314831583168317831883198320832183228323832483258326832783288329833083318332833383348335833683378338833983408341834283438344834583468347834883498350835183528353835483558356835783588359836083618362836383648365836683678368836983708371837283738374837583768377837883798380838183828383838483858386838783888389839083918392839383948395839683978398839984008401840284038404840584068407840884098410841184128413841484158416841784188419842084218422842384248425842684278428842984308431843284338434843584368437843884398440844184428443844484458446844784488449845084518452845384548455845684578458845984608461846284638464846584668467846884698470847184728473847484758476847784788479848084818482848384848485848684878488848984908491849284938494849584968497849884998500850185028503850485058506850785088509851085118512851385148515851685178518851985208521852285238524852585268527852885298530853185328533853485358536853785388539854085418542854385448545854685478548854985508551855285538554855585568557855885598560856185628563856485658566856785688569857085718572857385748575857685778578857985808581858285838584858585868587858885898590859185928593859485958596859785988599860086018602860386048605860686078608860986108611861286138614861586168617861886198620862186228623862486258626862786288629863086318632863386348635863686378638863986408641864286438644864586468647864886498650865186528653865486558656865786588659866086618662866386648665866686678668866986708671867286738674867586768677867886798680868186828683868486858686868786888689869086918692869386948695869686978698869987008701870287038704870587068707870887098710871187128713871487158716871787188719872087218722872387248725872687278728872987308731873287338734873587368737873887398740874187428743874487458746874787488749875087518752875387548755875687578758875987608761876287638764876587668767876887698770877187728773877487758776877787788779878087818782878387848785878687878788878987908791879287938794879587968797879887998800880188028803880488058806880788088809881088118812881388148815881688178818881988208821882288238824882588268827882888298830883188328833883488358836883788388839884088418842884388448845884688478848884988508851885288538854885588568857885888598860886188628863886488658866886788688869887088718872887388748875887688778878887988808881888288838884888588868887888888898890889188928893889488958896889788988899890089018902890389048905890689078908890989108911891289138914891589168917891889198920892189228923892489258926892789288929893089318932893389348935893689378938893989408941894289438944894589468947894889498950895189528953895489558956895789588959896089618962896389648965896689678968896989708971897289738974897589768977897889798980898189828983898489858986898789888989899089918992899389948995899689978998899990009001900290039004900590069007900890099010901190129013901490159016901790189019902090219022902390249025902690279028902990309031903290339034903590369037903890399040904190429043904490459046904790489049905090519052905390549055905690579058905990609061906290639064906590669067906890699070907190729073907490759076907790789079908090819082908390849085908690879088908990909091909290939094909590969097909890999100910191029103910491059106910791089109911091119112911391149115911691179118911991209121912291239124912591269127912891299130913191329133913491359136913791389139914091419142914391449145914691479148914991509151915291539154915591569157915891599160916191629163916491659166916791689169917091719172917391749175917691779178917991809181918291839184918591869187918891899190919191929193919491959196919791989199920092019202920392049205920692079208920992109211921292139214921592169217921892199220922192229223922492259226922792289229923092319232923392349235923692379238923992409241924292439244924592469247924892499250925192529253925492559256925792589259926092619262926392649265926692679268926992709271927292739274927592769277927892799280928192829283928492859286928792889289929092919292929392949295929692979298929993009301930293039304930593069307930893099310931193129313931493159316931793189319932093219322932393249325932693279328932993309331933293339334933593369337933893399340934193429343934493459346934793489349935093519352935393549355935693579358935993609361936293639364936593669367936893699370937193729373937493759376937793789379938093819382938393849385938693879388938993909391939293939394939593969397939893999400940194029403940494059406940794089409941094119412941394149415941694179418941994209421942294239424942594269427942894299430943194329433943494359436943794389439944094419442944394449445944694479448944994509451945294539454945594569457945894599460946194629463946494659466946794689469947094719472947394749475947694779478947994809481948294839484948594869487948894899490949194929493949494959496949794989499950095019502950395049505950695079508950995109511951295139514951595169517951895199520952195229523952495259526952795289529953095319532953395349535953695379538953995409541954295439544954595469547954895499550955195529553955495559556955795589559956095619562956395649565956695679568956995709571957295739574957595769577957895799580958195829583958495859586958795889589959095919592959395949595959695979598959996009601960296039604960596069607960896099610961196129613961496159616961796189619962096219622962396249625962696279628962996309631963296339634963596369637963896399640964196429643964496459646964796489649965096519652965396549655965696579658965996609661966296639664966596669667966896699670967196729673967496759676967796789679968096819682968396849685968696879688968996909691969296939694969596969697969896999700970197029703970497059706970797089709971097119712971397149715971697179718971997209721972297239724972597269727972897299730973197329733973497359736973797389739974097419742974397449745974697479748974997509751975297539754975597569757975897599760976197629763976497659766976797689769977097719772977397749775977697779778977997809781978297839784978597869787978897899790979197929793979497959796979797989799980098019802980398049805980698079808980998109811981298139814981598169817981898199820982198229823982498259826982798289829983098319832983398349835983698379838983998409841984298439844984598469847984898499850985198529853985498559856985798589859986098619862986398649865986698679868986998709871987298739874987598769877987898799880988198829883988498859886988798889889989098919892989398949895989698979898989999009901990299039904990599069907990899099910991199129913991499159916991799189919992099219922992399249925992699279928992999309931993299339934993599369937993899399940994199429943994499459946994799489949995099519952995399549955995699579958995999609961996299639964996599669967996899699970997199729973997499759976997799789979998099819982998399849985998699879988998999909991999299939994999599969997999899991000010001100021000310004100051000610007100081000910010100111001210013100141001510016100171001810019100201002110022100231002410025100261002710028100291003010031100321003310034100351003610037100381003910040100411004210043100441004510046100471004810049100501005110052100531005410055100561005710058100591006010061100621006310064100651006610067100681006910070100711007210073100741007510076100771007810079100801008110082100831008410085100861008710088100891009010091100921009310094100951009610097100981009910100101011010210103101041010510106101071010810109101101011110112101131011410115101161011710118101191012010121101221012310124101251012610127101281012910130101311013210133101341013510136101371013810139101401014110142101431014410145101461014710148101491015010151101521015310154101551015610157101581015910160101611016210163101641016510166101671016810169101701017110172101731017410175101761017710178101791018010181101821018310184101851018610187101881018910190101911019210193101941019510196101971019810199102001020110202102031020410205102061020710208102091021010211102121021310214102151021610217102181021910220102211022210223102241022510226102271022810229102301023110232102331023410235102361023710238102391024010241102421024310244102451024610247102481024910250102511025210253102541025510256102571025810259102601026110262102631026410265102661026710268102691027010271102721027310274102751027610277102781027910280102811028210283102841028510286102871028810289102901029110292102931029410295102961029710298102991030010301103021030310304103051030610307103081030910310103111031210313103141031510316103171031810319103201032110322103231032410325103261032710328103291033010331103321033310334103351033610337103381033910340103411034210343103441034510346103471034810349103501035110352103531035410355103561035710358103591036010361103621036310364103651036610367103681036910370103711037210373103741037510376103771037810379103801038110382103831038410385103861038710388103891039010391103921039310394103951039610397103981039910400104011040210403104041040510406104071040810409104101041110412104131041410415104161041710418104191042010421104221042310424104251042610427104281042910430104311043210433104341043510436104371043810439104401044110442104431044410445104461044710448104491045010451104521045310454104551045610457104581045910460104611046210463104641046510466104671046810469104701047110472104731047410475104761047710478104791048010481104821048310484104851048610487104881048910490104911049210493104941049510496104971049810499105001050110502105031050410505105061050710508105091051010511105121051310514105151051610517105181051910520105211052210523105241052510526105271052810529105301053110532105331053410535105361053710538105391054010541105421054310544105451054610547105481054910550105511055210553105541055510556105571055810559105601056110562105631056410565105661056710568105691057010571105721057310574105751057610577105781057910580105811058210583105841058510586105871058810589105901059110592105931059410595105961059710598105991060010601106021060310604106051060610607106081060910610106111061210613106141061510616106171061810619106201062110622106231062410625106261062710628106291063010631106321063310634106351063610637106381063910640106411064210643106441064510646106471064810649106501065110652106531065410655106561065710658106591066010661106621066310664106651066610667106681066910670106711067210673106741067510676106771067810679106801068110682106831068410685106861068710688106891069010691106921069310694106951069610697106981069910700107011070210703107041070510706107071070810709107101071110712107131071410715107161071710718107191072010721107221072310724107251072610727107281072910730107311073210733107341073510736107371073810739107401074110742107431074410745107461074710748107491075010751107521075310754107551075610757107581075910760107611076210763107641076510766107671076810769107701077110772107731077410775107761077710778107791078010781107821078310784107851078610787107881078910790107911079210793107941079510796107971079810799108001080110802108031080410805108061080710808108091081010811108121081310814108151081610817108181081910820108211082210823108241082510826108271082810829108301083110832108331083410835108361083710838108391084010841108421084310844108451084610847108481084910850108511085210853108541085510856108571085810859108601086110862108631086410865108661086710868108691087010871108721087310874108751087610877108781087910880108811088210883108841088510886108871088810889108901089110892108931089410895108961089710898108991090010901109021090310904109051090610907109081090910910109111091210913109141091510916109171091810919109201092110922109231092410925109261092710928109291093010931109321093310934109351093610937109381093910940109411094210943109441094510946109471094810949109501095110952109531095410955109561095710958109591096010961109621096310964109651096610967109681096910970109711097210973109741097510976109771097810979109801098110982109831098410985109861098710988109891099010991109921099310994109951099610997109981099911000110011100211003110041100511006110071100811009110101101111012110131101411015110161101711018110191102011021110221102311024110251102611027110281102911030110311103211033110341103511036110371103811039110401104111042110431104411045110461104711048110491105011051110521105311054110551105611057110581105911060110611106211063110641106511066110671106811069110701107111072110731107411075110761107711078110791108011081110821108311084110851108611087110881108911090110911109211093110941109511096110971109811099111001110111102111031110411105111061110711108111091111011111111121111311114111151111611117111181111911120111211112211123111241112511126111271112811129111301113111132111331113411135111361113711138111391114011141111421114311144111451114611147111481114911150111511115211153111541115511156111571115811159111601116111162111631116411165111661116711168111691117011171111721117311174111751117611177111781117911180111811118211183111841118511186111871118811189111901119111192111931119411195111961119711198111991120011201112021120311204112051120611207112081120911210112111121211213112141121511216112171121811219112201122111222112231122411225112261122711228112291123011231112321123311234112351123611237112381123911240112411124211243112441124511246112471124811249112501125111252112531125411255112561125711258112591126011261112621126311264112651126611267112681126911270112711127211273112741127511276112771127811279112801128111282112831128411285112861128711288112891129011291112921129311294112951129611297112981129911300113011130211303113041130511306113071130811309113101131111312113131131411315113161131711318113191132011321113221132311324113251132611327113281132911330113311133211333113341133511336113371133811339113401134111342113431134411345113461134711348113491135011351113521135311354113551135611357113581135911360113611136211363113641136511366113671136811369113701137111372113731137411375113761137711378113791138011381113821138311384113851138611387113881138911390113911139211393113941139511396113971139811399114001140111402114031140411405114061140711408114091141011411114121141311414114151141611417114181141911420114211142211423114241142511426114271142811429114301143111432114331143411435114361143711438114391144011441114421144311444114451144611447114481144911450114511145211453114541145511456114571145811459114601146111462114631146411465114661146711468114691147011471114721147311474114751147611477114781147911480114811148211483114841148511486114871148811489114901149111492114931149411495114961149711498114991150011501115021150311504115051150611507115081150911510115111151211513115141151511516115171151811519115201152111522115231152411525115261152711528115291153011531115321153311534115351153611537115381153911540115411154211543115441154511546115471154811549115501155111552115531155411555115561155711558115591156011561115621156311564115651156611567115681156911570115711157211573115741157511576115771157811579115801158111582115831158411585115861158711588115891159011591115921159311594115951159611597115981159911600116011160211603116041160511606116071160811609116101161111612116131161411615116161161711618116191162011621116221162311624116251162611627116281162911630116311163211633116341163511636116371163811639116401164111642116431164411645116461164711648116491165011651116521165311654116551165611657116581165911660116611166211663116641166511666116671166811669116701167111672116731167411675116761167711678116791168011681116821168311684116851168611687116881168911690116911169211693116941169511696116971169811699117001170111702117031170411705117061170711708117091171011711117121171311714117151171611717117181171911720117211172211723117241172511726117271172811729117301173111732117331173411735117361173711738117391174011741117421174311744117451174611747117481174911750117511175211753117541175511756117571175811759117601176111762117631176411765117661176711768117691177011771117721177311774117751177611777117781177911780117811178211783117841178511786117871178811789117901179111792117931179411795117961179711798117991180011801118021180311804118051180611807118081180911810118111181211813118141181511816118171181811819118201182111822118231182411825118261182711828118291183011831118321183311834118351183611837118381183911840118411184211843118441184511846118471184811849118501185111852118531185411855118561185711858118591186011861118621186311864118651186611867118681186911870118711187211873118741187511876118771187811879118801188111882118831188411885118861188711888118891189011891118921189311894118951189611897118981189911900119011190211903119041190511906119071190811909119101191111912119131191411915119161191711918119191192011921119221192311924119251192611927119281192911930119311193211933119341193511936119371193811939119401194111942119431194411945119461194711948119491195011951119521195311954119551195611957119581195911960119611196211963119641196511966119671196811969119701197111972119731197411975119761197711978119791198011981119821198311984119851198611987119881198911990119911199211993119941199511996119971199811999120001200112002120031200412005120061200712008120091201012011120121201312014120151201612017120181201912020120211202212023120241202512026120271202812029120301203112032120331203412035120361203712038120391204012041120421204312044120451204612047120481204912050120511205212053120541205512056120571205812059120601206112062120631206412065120661206712068120691207012071120721207312074120751207612077120781207912080120811208212083120841208512086120871208812089120901209112092120931209412095120961209712098120991210012101121021210312104121051210612107121081210912110121111211212113121141211512116121171211812119121201212112122121231212412125121261212712128121291213012131121321213312134121351213612137121381213912140121411214212143121441214512146121471214812149121501215112152121531215412155121561215712158121591216012161121621216312164121651216612167121681216912170121711217212173121741217512176121771217812179121801218112182121831218412185121861218712188121891219012191121921219312194121951219612197121981219912200122011220212203122041220512206122071220812209122101221112212122131221412215122161221712218122191222012221122221222312224122251222612227122281222912230122311223212233122341223512236122371223812239122401224112242122431224412245122461224712248122491225012251122521225312254122551225612257122581225912260122611226212263122641226512266122671226812269122701227112272122731227412275122761227712278122791228012281122821228312284122851228612287122881228912290122911229212293122941229512296122971229812299123001230112302123031230412305123061230712308123091231012311123121231312314123151231612317123181231912320123211232212323123241232512326123271232812329123301233112332123331233412335123361233712338123391234012341123421234312344123451234612347123481234912350123511235212353123541235512356123571235812359123601236112362123631236412365123661236712368123691237012371123721237312374123751237612377123781237912380123811238212383123841238512386123871238812389123901239112392123931239412395123961239712398123991240012401124021240312404124051240612407124081240912410124111241212413124141241512416124171241812419124201242112422124231242412425124261242712428124291243012431124321243312434124351243612437124381243912440124411244212443124441244512446124471244812449124501245112452124531245412455124561245712458124591246012461124621246312464124651246612467124681246912470124711247212473124741247512476124771247812479124801248112482124831248412485124861248712488124891249012491124921249312494124951249612497124981249912500125011250212503125041250512506125071250812509125101251112512125131251412515125161251712518125191252012521125221252312524125251252612527125281252912530125311253212533125341253512536125371253812539125401254112542125431254412545125461254712548125491255012551125521255312554125551255612557125581255912560125611256212563125641256512566125671256812569125701257112572125731257412575125761257712578125791258012581125821258312584125851258612587125881258912590125911259212593125941259512596125971259812599126001260112602126031260412605126061260712608126091261012611126121261312614126151261612617126181261912620126211262212623126241262512626126271262812629126301263112632126331263412635126361263712638126391264012641126421264312644126451264612647126481264912650126511265212653126541265512656126571265812659126601266112662126631266412665126661266712668126691267012671126721267312674126751267612677126781267912680126811268212683126841268512686126871268812689126901269112692126931269412695126961269712698126991270012701127021270312704127051270612707127081270912710127111271212713127141271512716127171271812719127201272112722127231272412725127261272712728127291273012731127321273312734127351273612737127381273912740127411274212743127441274512746127471274812749127501275112752127531275412755127561275712758127591276012761127621276312764127651276612767127681276912770127711277212773127741277512776127771277812779127801278112782127831278412785127861278712788127891279012791127921279312794127951279612797127981279912800128011280212803128041280512806128071280812809128101281112812128131281412815128161281712818128191282012821128221282312824128251282612827128281282912830128311283212833128341283512836128371283812839128401284112842128431284412845128461284712848128491285012851128521285312854128551285612857128581285912860128611286212863128641286512866128671286812869128701287112872128731287412875128761287712878128791288012881128821288312884128851288612887128881288912890128911289212893128941289512896128971289812899129001290112902129031290412905129061290712908129091291012911129121291312914129151291612917129181291912920129211292212923129241292512926129271292812929129301293112932129331293412935129361293712938129391294012941129421294312944129451294612947129481294912950129511295212953129541295512956129571295812959129601296112962129631296412965129661296712968129691297012971129721297312974129751297612977129781297912980129811298212983129841298512986129871298812989129901299112992129931299412995129961299712998129991300013001130021300313004130051300613007130081300913010130111301213013130141301513016130171301813019130201302113022130231302413025130261302713028130291303013031130321303313034130351303613037130381303913040130411304213043130441304513046130471304813049130501305113052130531305413055130561305713058130591306013061130621306313064130651306613067130681306913070130711307213073130741307513076130771307813079130801308113082130831308413085130861308713088130891309013091130921309313094130951309613097130981309913100131011310213103131041310513106131071310813109131101311113112131131311413115131161311713118131191312013121131221312313124131251312613127131281312913130131311313213133131341313513136131371313813139131401314113142131431314413145131461314713148131491315013151131521315313154131551315613157131581315913160131611316213163131641316513166131671316813169131701317113172131731317413175131761317713178131791318013181131821318313184131851318613187131881318913190131911319213193131941319513196131971319813199132001320113202132031320413205132061320713208132091321013211132121321313214132151321613217132181321913220132211322213223132241322513226132271322813229132301323113232132331323413235132361323713238132391324013241132421324313244132451324613247132481324913250132511325213253132541325513256132571325813259132601326113262132631326413265132661326713268132691327013271132721327313274132751327613277132781327913280132811328213283132841328513286132871328813289132901329113292132931329413295132961329713298132991330013301133021330313304133051330613307133081330913310133111331213313133141331513316133171331813319133201332113322133231332413325133261332713328133291333013331133321333313334133351333613337133381333913340133411334213343133441334513346133471334813349133501335113352133531335413355133561335713358133591336013361133621336313364133651336613367133681336913370133711337213373133741337513376133771337813379133801338113382133831338413385133861338713388133891339013391133921339313394133951339613397133981339913400134011340213403134041340513406134071340813409134101341113412134131341413415134161341713418134191342013421134221342313424134251342613427134281342913430134311343213433134341343513436134371343813439134401344113442134431344413445134461344713448134491345013451134521345313454134551345613457134581345913460134611346213463134641346513466134671346813469134701347113472134731347413475134761347713478134791348013481134821348313484134851348613487134881348913490134911349213493134941349513496134971349813499135001350113502135031350413505135061350713508135091351013511135121351313514135151351613517135181351913520135211352213523135241352513526135271352813529135301353113532135331353413535135361353713538135391354013541135421354313544135451354613547135481354913550135511355213553135541355513556135571355813559135601356113562135631356413565135661356713568135691357013571135721357313574135751357613577135781357913580135811358213583135841358513586135871358813589135901359113592135931359413595135961359713598135991360013601136021360313604136051360613607136081360913610136111361213613136141361513616136171361813619136201362113622136231362413625136261362713628136291363013631136321363313634136351363613637136381363913640136411364213643136441364513646136471364813649136501365113652136531365413655136561365713658136591366013661136621366313664136651366613667136681366913670136711367213673136741367513676136771367813679136801368113682136831368413685136861368713688136891369013691136921369313694136951369613697136981369913700137011370213703137041370513706137071370813709137101371113712137131371413715137161371713718137191372013721137221372313724137251372613727137281372913730137311373213733137341373513736137371373813739137401374113742137431374413745137461374713748137491375013751137521375313754137551375613757137581375913760137611376213763137641376513766137671376813769137701377113772137731377413775137761377713778137791378013781137821378313784137851378613787137881378913790137911379213793137941379513796137971379813799138001380113802138031380413805138061380713808138091381013811138121381313814138151381613817138181381913820138211382213823138241382513826138271382813829138301383113832138331383413835138361383713838138391384013841138421384313844138451384613847138481384913850138511385213853138541385513856138571385813859138601386113862138631386413865138661386713868138691387013871138721387313874138751387613877138781387913880138811388213883138841388513886138871388813889138901389113892138931389413895138961389713898138991390013901139021390313904139051390613907139081390913910139111391213913139141391513916139171391813919139201392113922139231392413925139261392713928139291393013931139321393313934139351393613937139381393913940139411394213943139441394513946139471394813949139501395113952139531395413955
  1. /*##############################################################################
  2. HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. ############################################################################## */
  13. #include "platform.h"
  14. #include "jlib.hpp"
  15. #include "jmisc.hpp"
  16. #include "jstream.ipp"
  17. #include "eclrtl.hpp"
  18. #include "hql.hpp"
  19. #include "hqlttcpp.ipp"
  20. #include "hqlmeta.hpp"
  21. #include "hqlutil.hpp"
  22. #include "hqlcpputil.hpp"
  23. #include "hqlthql.hpp"
  24. #include "hqlhtcpp.ipp"
  25. #include "hqltcppc.ipp"
  26. #include "hqlcatom.hpp"
  27. #include "hqlfold.hpp"
  28. #include "hqlgraph.ipp"
  29. #include "hqllib.ipp"
  30. #include "hqlmeta.hpp"
  31. #include "hqlpmap.hpp"
  32. #include "hqlopt.hpp"
  33. #include "hqlcerrors.hpp"
  34. #include "hqlscope.hpp"
  35. #include "hqlsource.ipp"
  36. #include "hqlvalid.hpp"
  37. #include "hqlerror.hpp"
  38. #include "hqlalias.hpp"
  39. #include "hqlir.hpp"
  40. #define TraceExprPrintLog(x, expr) TOSTRLOG(MCdebugInfo(300), unknownJob, x, (expr)->toString);
  41. //Following are for code that currently cause problems, but are probably a good idea
  42. //#define MAP_PROJECT_TO_USERTABLE
  43. //#define REMOVE_NAMED_SCALARS
  44. //#define OPTIMIZE_IMPLICIT_CAST
  45. #define PERSIST_VERSION 1 // Increment when implementation is incompatible.
  46. #define REMOVE_GLOBAL_ANNOTATION // This should improve cse. It currently does for some, but not others...
  47. #define DEFAULT_FOLD_OPTIONS HFOfoldfilterproject
  48. //#define PICK_ENGINE_EARLY
  49. //===========================================================================
  50. static bool isWorthHoisting(IHqlExpression * expr, bool asSubQuery)
  51. {
  52. bool isFiltered = false;
  53. for (;;)
  54. {
  55. switch (expr->getOperator())
  56. {
  57. case no_newkeyindex:
  58. case no_table:
  59. case no_temptable:
  60. case no_inlinetable:
  61. case no_globalscope:
  62. case no_global:
  63. case no_independent:
  64. case no_field:
  65. case no_datasetfromrow:
  66. case no_datasetfromdictionary:
  67. case no_null:
  68. case no_workunit_dataset:
  69. case no_colon:
  70. //compound activities not in list because not created at this point.
  71. return (isFiltered && asSubQuery);
  72. case no_select:
  73. return !isTargetSelector(expr);
  74. case no_selectnth:
  75. {
  76. IHqlExpression * ds = expr->queryChild(0);
  77. if (!hasSingleRow(ds))
  78. return true;
  79. expr = ds;
  80. break;
  81. }
  82. case no_filter:
  83. expr = expr->queryChild(0);
  84. isFiltered = true;
  85. break;
  86. case no_compound_diskread:
  87. case no_compound_indexread:
  88. case no_hqlproject:
  89. case no_newusertable:
  90. case no_limit:
  91. case no_catchds:
  92. case no_keyedlimit:
  93. case no_sorted:
  94. case no_grouped:
  95. case no_stepped:
  96. case no_distributed:
  97. case no_preservemeta:
  98. case no_unordered:
  99. case no_nofold:
  100. case no_nohoist:
  101. case no_nocombine:
  102. case no_section:
  103. case no_sectioninput:
  104. case no_dataset_alias:
  105. case no_forcegraph:
  106. expr = expr->queryChild(0);
  107. break;
  108. case no_fail:
  109. return false;
  110. default:
  111. return true;
  112. }
  113. }
  114. }
  115. IHqlExpression * getDebugValueExpr(IConstWorkUnit * wu, IHqlExpression * expr)
  116. {
  117. StringBuffer name;
  118. getStringValue(name, expr->queryChild(0));
  119. SCMStringBuffer value;
  120. wu->getDebugValue(name, value);
  121. ITypeInfo * exprType = expr->queryType();
  122. if (exprType->getTypeCode() == type_boolean)
  123. return createConstant(clipStrToBool(value.length(), value.str()));
  124. return createConstant(exprType->castFrom(value.length(), value.str()));
  125. }
  126. //===========================================================================
  127. struct GlobalAttributeInfo
  128. {
  129. public:
  130. GlobalAttributeInfo(const char * _filePrefix, const char * _storedPrefix, IHqlExpression * _value) : value(_value), persistRefresh(true)
  131. {
  132. setOp = no_none;
  133. persistOp = no_none;
  134. few = false;
  135. filePrefix = _filePrefix;
  136. storedPrefix = _storedPrefix;
  137. numPersistInstances = 0;
  138. }
  139. void extractGlobal(IHqlExpression * global, ClusterType platform);
  140. void extractStoredInfo(IHqlExpression * expr, const char * id, IHqlExpression * codehash, bool isRoxie, int multiplePersistInstances);
  141. void checkFew(HqlCppTranslator & translator);
  142. void splitGlobalDefinition(ITypeInfo * type, IHqlExpression * value, IConstWorkUnit * wu, SharedHqlExpr & setOutput, OwnedHqlExpr * getOutput, bool isRoxie);
  143. IHqlExpression * getStoredKey();
  144. void preventDiskSpill() { few = true; }
  145. IHqlExpression * queryCluster() const { return cluster; }
  146. int queryMaxPersistCopies() const { return numPersistInstances; }
  147. bool queryPersistRefresh() const { return persistRefresh; }
  148. const char * queryLabel() const { return label ? label.str() : nullptr; }
  149. protected:
  150. void doSplitGlobalDefinition(ITypeInfo * type, IHqlExpression * value, IConstWorkUnit * wu, SharedHqlExpr & setOutput, OwnedHqlExpr * getOutput, bool isRoxie);
  151. IHqlExpression * createSetValue(IHqlExpression * value, IHqlExpression * aliasName);
  152. void createSmallOutput(IHqlExpression * value, SharedHqlExpr & setOutput);
  153. IHqlExpression * queryAlias(IHqlExpression * value);
  154. IHqlExpression * queryFilename(IHqlExpression * value, IConstWorkUnit * wu, bool isRoxie);
  155. void splitSmallDataset(IHqlExpression * value, SharedHqlExpr & setOutput, OwnedHqlExpr * getOutput);
  156. void setCluster(IHqlExpression * expr);
  157. public:
  158. LinkedHqlExpr value;
  159. OwnedHqlExpr storedName;
  160. OwnedHqlExpr originalLabel;
  161. OwnedHqlExpr sequence;
  162. OwnedHqlExpr fieldFormat;
  163. node_operator setOp;
  164. node_operator persistOp;
  165. protected:
  166. OwnedHqlExpr aliasName;
  167. OwnedHqlExpr cachedFilename;
  168. OwnedHqlExpr cluster;
  169. OwnedHqlExpr extraSetAttr;
  170. OwnedHqlExpr extraOutputAttr;
  171. OwnedHqlExpr codehash;
  172. StringBuffer label;
  173. const char * filePrefix;
  174. const char * storedPrefix;
  175. int numPersistInstances;
  176. bool few;
  177. bool persistRefresh;
  178. };
  179. static bool isTrivialInlineOutput(IHqlExpression * expr)
  180. {
  181. if (queryRealChild(expr, 1))
  182. return false;
  183. IHqlExpression * ds = expr->queryChild(0);
  184. if ((ds->getOperator() != no_null) || !ds->isDataset())
  185. return false;
  186. IHqlExpression * seq = querySequence(expr);
  187. if (getIntValue(seq, -1) >= 0)
  188. return false;
  189. return true;
  190. }
  191. //---------------------------------------------------------------------------
  192. IHqlExpression * createNextStringValue(IHqlExpression * value, const char * prefix = NULL)
  193. {
  194. StringBuffer valueText;
  195. valueText.append(prefix ? prefix : "a");
  196. getUniqueId(valueText);
  197. #if 0
  198. if (value)
  199. {
  200. const char * nameText = value->queryName()->str();
  201. if (nameText)
  202. valueText.append("__").appendLower(strlen(nameText), nameText);
  203. //otherwise append the operator?
  204. }
  205. #endif
  206. #if 0
  207. #ifdef _DEBUG
  208. //Following lines are here to add a break point when debugging
  209. if (stricmp(valueText.str(), "a1") == 0)
  210. valueText.length();
  211. if (stricmp(valueText.str(), "a2") == 0)
  212. valueText.length();
  213. #endif
  214. #endif
  215. return createConstant(valueText.str());
  216. }
  217. IHqlExpression * createSetResult(IHqlExpression * value)
  218. {
  219. HqlExprArray args;
  220. args.append(*LINK(value));
  221. args.append(*createAttribute(sequenceAtom, getLocalSequenceNumber()));
  222. args.append(*createAttribute(namedAtom, createNextStringValue(value)));
  223. return createSetResult(args);
  224. }
  225. static IHqlExpression * addAttrOwnToDataset(IHqlExpression * dataset, IHqlExpression * attr)
  226. {
  227. HqlExprArray args;
  228. unwindChildren(args, dataset);
  229. args.append(*attr);
  230. return dataset->clone(args);
  231. }
  232. static IHqlExpression * mergeLimitIntoDataset(IHqlExpression * dataset, IHqlExpression * limit)
  233. {
  234. return addAttrOwnToDataset(dataset, createAttribute(limitAtom, LINK(limit->queryChild(1)), LINK(limit->queryChild(2))));
  235. }
  236. //---------------------------------------------------------------------------
  237. static bool isOptionTooLate(const char * name)
  238. {
  239. if (stricmp(name, "gatherDependencies") == 0) return true;
  240. if (stricmp(name, "gatherDependenciesSelection") == 0) return true;
  241. if (stricmp(name, "archiveToCpp") == 0) return true;
  242. if (stricmp(name, "importAllModules") == 0) return true;
  243. if (stricmp(name, "importImplicitModules") == 0) return true;
  244. if (stricmp(name, "noCache") == 0) return true;
  245. return false;
  246. }
  247. static HqlTransformerInfo newThorStoredReplacerInfo("NewThorStoredReplacer");
  248. NewThorStoredReplacer::NewThorStoredReplacer(HqlCppTranslator & _translator, IWorkUnit * _wu, ICodegenContextCallback * _ctxCallback)
  249. : QuickHqlTransformer(newThorStoredReplacerInfo, NULL), translator(_translator)
  250. {
  251. wu = _wu;
  252. ctxCallback = _ctxCallback;
  253. foldStored = false;
  254. seenMeta = false;
  255. }
  256. void NewThorStoredReplacer::doAnalyse(IHqlExpression * expr)
  257. {
  258. //NOTE: This is called very early before no_assertconstant has been processed, so we need to explicitly
  259. //constant fold, and check it is constant (bug 26963)
  260. node_operator op = expr->getOperator();
  261. if (op == no_setmeta)
  262. {
  263. StringBuffer errorTemp;
  264. seenMeta = true;
  265. IAtom * kind = expr->queryChild(0)->queryName();
  266. if (kind == webserviceAtom)
  267. {
  268. Owned<IWUWebServicesInfo> wsi = wu->updateWebServicesInfo(false);
  269. if (wsi)
  270. translator.ERRORAT(expr, HQLERR_MultipleHashWebserviceCalls);
  271. wsi.setown(wu->updateWebServicesInfo(true));
  272. IHqlExpression *wsExpr = expr->queryChild(0);
  273. ForEachChild(i, wsExpr)
  274. {
  275. StringBuffer name, value;
  276. OwnedHqlExpr folded = foldHqlExpression(wsExpr->queryChild(i));
  277. getHintNameValue(folded, name, value);
  278. wsi->setText(name, value);
  279. }
  280. }
  281. else if (kind == debugAtom)
  282. {
  283. OwnedHqlExpr foldedName = foldHqlExpression(expr->queryChild(1));
  284. OwnedHqlExpr foldedValue = foldHqlExpression(expr->queryChild(2));
  285. IValue * name = foldedName->queryValue();
  286. IValue * value = foldedValue->queryValue();
  287. if (!name)
  288. translator.reportError(expr, ECODETEXT(HQLERR_ExpectedConstantDebug), getExprECL(foldedName, errorTemp).str());
  289. if (!value)
  290. translator.reportError(expr, ECODETEXT(HQLERR_ExpectedConstantDebug), getExprECL(foldedValue, errorTemp).str());
  291. StringBuffer nameText,valueText;
  292. name->getStringValue(nameText);
  293. if (isOptionTooLate(nameText.str()))
  294. translator.reportWarning(CategoryIgnored, HQLWRN_OptionSetToLate, HQLWRN_OptionSetToLate_Text, nameText.str());
  295. if (value->queryType()->getTypeCode() == type_boolean)
  296. valueText.append(value->getBoolValue() ? 1 : 0);
  297. else
  298. value->getStringValue(valueText);
  299. wu->setDebugValue(nameText.str(), valueText, true);
  300. }
  301. else if (kind == workunitAtom)
  302. {
  303. OwnedHqlExpr foldedName = foldHqlExpression(expr->queryChild(1));
  304. OwnedHqlExpr foldedValue = foldHqlExpression(expr->queryChild(2));
  305. IValue * name = foldedName->queryValue();
  306. IValue * value = foldedValue->queryValue();
  307. if (!name)
  308. translator.reportError(expr, ECODETEXT(HQLERR_ExpectedConstantWorkunit), getExprECL(foldedName, errorTemp).str());
  309. if (!value)
  310. translator.reportError(expr, ECODETEXT(HQLERR_ExpectedConstantWorkunit), getExprECL(foldedValue, errorTemp).str());
  311. StringBuffer nameText,valueText;
  312. name->getStringValue(nameText);
  313. value->getStringValue(valueText);
  314. if (stricmp(nameText.str(), "name") == 0)
  315. wu->setJobName(valueText.str());
  316. else if (stricmp(nameText.str(), "priority") == 0)
  317. {
  318. if (isStringType(value->queryType()))
  319. {
  320. WUPriorityClass prio = PriorityClassUnknown;
  321. if (stricmp(valueText.str(), "low") == 0)
  322. prio = PriorityClassLow;
  323. else if (stricmp(valueText.str(), "normal") == 0)
  324. prio = PriorityClassNormal;
  325. else if (stricmp(valueText.str(), "high") == 0)
  326. prio = PriorityClassHigh;
  327. wu->setPriority(prio);
  328. }
  329. else
  330. wu->setPriorityLevel((int)value->getIntValue());
  331. }
  332. else if (stricmp(nameText.str(), "cluster") == 0)
  333. {
  334. ctxCallback->noteCluster(valueText.str());
  335. wu->setClusterName(valueText.str());
  336. }
  337. else if (stricmp(nameText.str(), "protect") == 0)
  338. {
  339. wu->protect(value->getBoolValue());
  340. }
  341. else if (stricmp(nameText.str(), "scope") == 0)
  342. {
  343. wu->setWuScope(valueText.str());
  344. }
  345. else
  346. translator.reportError(expr, ECODETEXT(HQLERR_UnsupportedHashWorkunit), nameText.str());
  347. }
  348. else if (kind == linkAtom)
  349. {
  350. OwnedHqlExpr foldedName = foldHqlExpression(expr->queryChild(1));
  351. StringBuffer libraryText;
  352. if (getStringValue(libraryText, foldedName).length())
  353. translator.useLibrary(libraryText);
  354. }
  355. else if ((kind == constAtom) || (kind == storedAtom))
  356. {
  357. //assume there won't be many of these... otherwise we should use a hash table
  358. OwnedHqlExpr lowerName = getLowerCaseConstantExpr(expr->queryChild(1));
  359. IHqlExpression * searchExpr = lowerName->queryBody();
  360. IHqlExpression * newValue = expr->queryChild(2);
  361. //Use lowerName->queryBody() to remove named symbols/location annotations etc.
  362. unsigned match = storedNames.find(*searchExpr);
  363. if (match != NotFound)
  364. {
  365. IHqlExpression * oldValue = &storedValues.item(match);
  366. if (oldValue->queryBody() != newValue->queryBody())
  367. {
  368. if (queryLocationIndependent(oldValue) != queryLocationIndependent(newValue))
  369. {
  370. StringBuffer labelText;
  371. getExprECL(searchExpr, labelText);
  372. StringBuffer oldText;
  373. StringBuffer newText;
  374. const char * oldType = storedIsConstant.item(match) ? "CONSTANT" : "STORED";
  375. const char * newType = (kind == constAtom) ? "CONSTANT" : "STORED";
  376. translator.reportWarning(CategoryMistake, SeverityError, queryLocation(expr), ECODETEXT(HQLERR_HashStoredDuplication), oldType, labelText.str(), getExprECL(oldValue, oldText).str(), newType, labelText.str(), getExprECL(newValue, newText).str());
  377. }
  378. }
  379. }
  380. else
  381. {
  382. storedNames.append(*LINK(searchExpr));
  383. storedValues.append(*LINK(newValue));
  384. storedIsConstant.append(kind == constAtom);
  385. }
  386. }
  387. else if (kind == onWarningAtom)
  388. translator.addGlobalOnWarning(expr);
  389. }
  390. else if (op == no_colon)
  391. {
  392. if (queryAttributeInList(labeledAtom, expr->queryChild(1)))
  393. seenMeta = true;
  394. }
  395. //Walk the body rather than expr so that the no_setmeta isn't processed again.
  396. //If the code above was in doAnalyseBody() the error location information would not be available.
  397. QuickHqlTransformer::doAnalyseBody(expr->queryBody());
  398. }
  399. bool NewThorStoredReplacer::needToTransform()
  400. {
  401. //foldStored = translator.queryOptions().foldStored; // NB: options isn't initialised correctly at this point.
  402. foldStored = wu->getDebugValueBool("foldStored", false);
  403. return (foldStored || seenMeta);
  404. }
  405. // This works on unnormalized trees, so based on QuickHqlTransformer
  406. IHqlExpression * NewThorStoredReplacer::createTransformed(IHqlExpression * expr)
  407. {
  408. switch (expr->getOperator())
  409. {
  410. case no_colon:
  411. {
  412. HqlExprArray actions;
  413. expr->queryChild(1)->unwindList(actions, no_comma);
  414. OwnedHqlExpr replacement;
  415. OwnedHqlExpr matchedName;
  416. bool onlyStored = true;
  417. bool forceConstant = foldStored;
  418. ForEachItemIn(idx, actions)
  419. {
  420. IHqlExpression & cur = actions.item(idx);
  421. switch (cur.getOperator())
  422. {
  423. case no_stored:
  424. {
  425. OwnedHqlExpr storedName = getLowerCaseConstantExpr(cur.queryChild(0));
  426. IHqlExpression * searchName = storedName->queryBody();
  427. unsigned match = storedNames.find(*searchName);
  428. if (match != NotFound)
  429. {
  430. if (storedIsConstant.item(match))
  431. forceConstant = true;
  432. matchedName.set(searchName);
  433. replacement.set(&storedValues.item(match));
  434. }
  435. break;
  436. }
  437. case no_attr:
  438. case no_attr_expr:
  439. if (cur.queryName() == labeledAtom)
  440. {
  441. OwnedHqlExpr storedName = getLowerCaseConstantExpr(cur.queryChild(0));
  442. IHqlExpression * searchName = storedName->queryBody();
  443. unsigned match = storedNames.find(*searchName);
  444. if (match != NotFound)
  445. {
  446. matchedName.set(searchName);
  447. replacement.set(&storedValues.item(match));
  448. }
  449. else
  450. replacement.set(expr->queryChild(0));
  451. forceConstant = true;
  452. break;
  453. }
  454. default:
  455. onlyStored = false;
  456. break;
  457. }
  458. }
  459. if (matchedName)
  460. {
  461. unsigned activeMatch = activeReplacements.find(*matchedName);
  462. if (activeMatch != NotFound)
  463. {
  464. StringBuffer nameText;
  465. getExprECL(matchedName, nameText);
  466. if (activeMatch+1 != activeReplacements.ordinality())
  467. {
  468. StringBuffer othersText;
  469. for (unsigned i=activeMatch+1; i < activeReplacements.ordinality(); i++)
  470. {
  471. othersText.append(",");
  472. getExprECL(&activeReplacements.item(i), othersText);
  473. }
  474. translator.reportError(expr, ECODETEXT(HQLERR_RecursiveStoredOther), forceConstant ? "CONSTANT" : "STORED", nameText.str(), othersText.str()+1);
  475. }
  476. else
  477. translator.reportError(expr, ECODETEXT(HQLERR_RecursiveStored), forceConstant ? "CONSTANT" : "STORED", nameText.str());
  478. }
  479. ITypeInfo * exprType = expr->queryType();
  480. ITypeInfo * replacementType = replacement->queryType();
  481. type_t etc = exprType->getTypeCode();
  482. type_t rtc = replacementType->getTypeCode();
  483. StringBuffer nameText, exprTypeText, replacementTypeText;
  484. switch (etc)
  485. {
  486. case type_groupedtable:
  487. case type_table:
  488. case type_row:
  489. case type_record:
  490. case type_transform:
  491. case type_void:
  492. {
  493. if (etc != rtc)
  494. {
  495. getExprECL(matchedName, nameText);
  496. getFriendlyTypeStr(exprType, exprTypeText);
  497. getFriendlyTypeStr(replacementType, replacementTypeText);
  498. translator.reportError(expr, ECODETEXT(HQLERR_HashStoredTypeMismatch), nameText.str(), exprTypeText.str(), replacementTypeText.str());
  499. }
  500. else if (expr->queryRecord() != replacement->queryRecord())
  501. {
  502. StringBuffer s;
  503. translator.reportError(expr, ECODETEXT(HQLERR_HashStoredRecordMismatch), getExprECL(matchedName, s).str());
  504. }
  505. }
  506. break;
  507. case type_set:
  508. case type_array:
  509. {
  510. if ((rtc != type_set) && (rtc != type_array))
  511. {
  512. getExprECL(matchedName, nameText);
  513. getFriendlyTypeStr(exprType, exprTypeText);
  514. getFriendlyTypeStr(replacementType, replacementTypeText);
  515. translator.reportError(expr, ECODETEXT(HQLERR_HashStoredTypeMismatch), nameText.str(), exprTypeText.str(), replacementTypeText.str());
  516. }
  517. replacement.setown(ensureExprType(replacement, exprType));
  518. break;
  519. }
  520. default:
  521. {
  522. switch (rtc)
  523. {
  524. case type_groupedtable:
  525. case type_table:
  526. case type_row:
  527. case type_record:
  528. case type_transform:
  529. case type_void:
  530. {
  531. getExprECL(matchedName, nameText);
  532. getFriendlyTypeStr(exprType, exprTypeText);
  533. getFriendlyTypeStr(replacementType, replacementTypeText);
  534. translator.reportError(expr, ECODETEXT(HQLERR_HashStoredTypeMismatch), nameText.str(), exprTypeText.str(), replacementTypeText.str());
  535. }
  536. default:
  537. replacement.setown(ensureExprType(replacement, exprType));
  538. }
  539. }
  540. break;
  541. }
  542. }
  543. LinkedHqlExpr result;
  544. if (matchedName)
  545. activeReplacements.append(*matchedName);
  546. if (onlyStored)
  547. {
  548. if (forceConstant && replacement)
  549. result.setown(transform(replacement));
  550. else if (foldStored)
  551. result.setown(transform(expr->queryChild(0)));
  552. }
  553. if (replacement && !result)
  554. {
  555. HqlExprArray args;
  556. args.append(*transform(replacement));
  557. result.setown(completeTransform(expr, args));
  558. }
  559. if (matchedName)
  560. activeReplacements.pop();
  561. if (result)
  562. return result.getClear();
  563. break;
  564. }
  565. case no_comma:
  566. case no_compound:
  567. if (expr->queryChild(0)->getOperator() == no_setmeta)
  568. return transform(expr->queryChild(1));
  569. if (expr->queryChild(1)->getOperator() == no_setmeta)
  570. return transform(expr->queryChild(0));
  571. break;
  572. case no_actionlist:
  573. case no_orderedactionlist:
  574. {
  575. HqlExprArray actions;
  576. ForEachChild(i, expr)
  577. {
  578. IHqlExpression * cur = expr->queryChild(i);
  579. if (cur->getOperator() != no_setmeta)
  580. actions.append(*transform(cur));
  581. }
  582. if (actions.ordinality() != 0)
  583. return expr->clone(actions);
  584. return transform(expr->queryChild(0));
  585. }
  586. }
  587. return QuickHqlTransformer::createTransformed(expr);
  588. }
  589. //---------------------------------------------------------------------------
  590. // NB: This is called after no_setresults are added, but before any normalization.
  591. static HqlTransformerInfo hqlThorBoundaryTransformerInfo("HqlThorBoundaryTransformer");
  592. HqlThorBoundaryTransformer::HqlThorBoundaryTransformer(IConstWorkUnit * _wu, bool _isRoxie, unsigned _maxRootMaybes, bool _resourceConditionalActions, bool _resourceSequential)
  593. : NewHqlTransformer(hqlThorBoundaryTransformerInfo)
  594. {
  595. wu = _wu;
  596. isRoxie = _isRoxie;
  597. maxRootMaybes = _maxRootMaybes;
  598. resourceConditionalActions = _resourceConditionalActions;
  599. resourceSequential = _resourceSequential;
  600. }
  601. void HqlThorBoundaryTransformer::transformCompound(HqlExprArray & result, node_operator compoundOp, const HqlExprArray & args, unsigned MaxMaybes)
  602. {
  603. UnsignedArray normalizeOptions;
  604. ForEachItemIn(i, args)
  605. {
  606. IHqlExpression & cur = args.item(i);
  607. normalizeOptions.append(normalizeThor(&cur));
  608. }
  609. //If any "yes" or "some" values are separated by maybes then convert the maybes (and "somes") into yes
  610. unsigned lastYes = NotFound;
  611. unsigned numMaybes = 0;
  612. ForEachItemIn(i2, args)
  613. {
  614. switch (normalizeOptions.item(i2))
  615. {
  616. case OptionYes:
  617. case OptionSome:
  618. if (lastYes != NotFound)
  619. {
  620. if (numMaybes <= MaxMaybes)
  621. {
  622. for (unsigned j = lastYes; j <= i2; j++)
  623. normalizeOptions.replace(OptionYes, j);
  624. }
  625. }
  626. numMaybes = 0;
  627. lastYes = i2;
  628. break;
  629. case OptionMaybe:
  630. numMaybes++;
  631. break;
  632. case OptionNo:
  633. lastYes = NotFound;
  634. break;
  635. }
  636. }
  637. //Do thor and non-thor parallel actions independently
  638. HqlExprArray thor;
  639. ForEachItemIn(idx, args)
  640. {
  641. IHqlExpression * cur = &args.item(idx);
  642. if (normalizeOptions.item(idx) == OptionYes)
  643. thor.append(*LINK(cur));
  644. else
  645. {
  646. if (thor.ordinality())
  647. {
  648. result.append(*createWrapper(no_thor, createCompound(compoundOp, thor)));
  649. thor.kill();
  650. }
  651. result.append(*createTransformed(cur));
  652. }
  653. }
  654. if (thor.ordinality())
  655. result.append(*createWrapper(no_thor, createCompound(compoundOp, thor)));
  656. }
  657. IHqlExpression * HqlThorBoundaryTransformer::createTransformed(IHqlExpression * expr)
  658. {
  659. node_operator op = expr->getOperator();
  660. switch (op)
  661. {
  662. case no_field:
  663. case no_constant:
  664. case no_attr:
  665. case no_attr_link:
  666. case no_getresult:
  667. case no_left:
  668. case no_right:
  669. return LINK(expr);
  670. case no_sizeof:
  671. case no_offsetof:
  672. return getTransformedChildren(expr);
  673. }
  674. //Unusually, wrap the expression in a thor node before processing annotations.
  675. //This ensures that the location/named symbol stays with the action.
  676. //MORE: If this is a dataset then it needs to turn it into a setResult()/getResult() pair.
  677. if (normalizeThor(expr) == OptionYes)
  678. {
  679. if (!expr->isTransform())
  680. return createWrapper(no_thor, LINK(expr));
  681. }
  682. IHqlExpression * ret = queryTransformAnnotation(expr);
  683. if (ret)
  684. return ret;
  685. switch (op)
  686. {
  687. case no_actionlist:
  688. case no_orderedactionlist:
  689. {
  690. HqlExprArray nonThor, args;
  691. expr->unwindList(args, op);
  692. transformCompound(nonThor, op, args, (unsigned)-1);
  693. return createCompound(op, nonThor);
  694. }
  695. case no_parallel:
  696. {
  697. HqlExprArray expanded;
  698. expr->unwindList(expanded, no_parallel);
  699. //Similar to compound, but possible to reorder branches...
  700. unsigned numThor = 0;
  701. UnsignedArray normalizeOptions;
  702. ForEachItemIn(idx, expanded)
  703. {
  704. YesNoOption option = normalizeThor(&expanded.item(idx));
  705. normalizeOptions.append(option);
  706. if ((option == OptionYes) || (option == OptionSome))
  707. numThor++;
  708. }
  709. if (numThor > 1)
  710. {
  711. HqlExprArray thor, nonThor;
  712. ForEachItemIn(idx, expanded)
  713. {
  714. IHqlExpression * cur = &expanded.item(idx);
  715. switch (normalizeOptions.item(idx))
  716. {
  717. case OptionYes:
  718. case OptionSome:
  719. thor.append(*LINK(cur));
  720. break;
  721. default:
  722. nonThor.append(*createTransformed(cur));
  723. break;
  724. }
  725. }
  726. if (nonThor.ordinality() == 0)
  727. {
  728. //can happen if inputs are a mixture of yes and some.
  729. return createWrapper(no_thor, LINK(expr));
  730. }
  731. nonThor.append(*createWrapper(no_thor, createValue(no_parallel, makeVoidType(), thor)));
  732. return expr->clone(nonThor);
  733. }
  734. break;
  735. }
  736. case no_nothor:
  737. return LINK(expr);
  738. }
  739. return NewHqlTransformer::createTransformed(expr);
  740. }
  741. static YesNoOption combine(YesNoOption left, YesNoOption right, bool isUnion)
  742. {
  743. if ((left == OptionNo) || (right == OptionNo))
  744. return OptionNo;
  745. if (left == OptionUnknown)
  746. return right;
  747. if (right == OptionUnknown)
  748. return left;
  749. //Yes,Some,Maybe
  750. if (isUnion)
  751. {
  752. //return definite if both branches may benefit.
  753. switch (left)
  754. {
  755. case OptionYes:
  756. return (right != OptionMaybe) ? OptionYes : OptionSome;
  757. case OptionSome:
  758. return (right != OptionMaybe) ? OptionYes : OptionSome;
  759. case OptionMaybe:
  760. return (right != OptionMaybe) ? OptionSome : OptionMaybe;
  761. }
  762. }
  763. else
  764. {
  765. //Intersection, return definite
  766. switch (left)
  767. {
  768. case OptionYes:
  769. return (right == OptionYes) ? OptionYes : OptionSome;
  770. case OptionSome:
  771. return OptionSome;
  772. case OptionMaybe:
  773. return (right == OptionMaybe) ? OptionMaybe : OptionSome;
  774. }
  775. }
  776. throwUnexpected();
  777. }
  778. //MORE: Needs to be yes, no, possibly.
  779. YesNoOption HqlThorBoundaryTransformer::normalizeThor(IHqlExpression * expr)
  780. {
  781. HqlThorBoundaryInfo * extra = queryBodyExtra(expr);
  782. if (extra->normalize == OptionUnknown)
  783. extra->normalize = calcNormalizeThor(expr);
  784. return extra->normalize;
  785. }
  786. YesNoOption HqlThorBoundaryTransformer::calcNormalizeThor(IHqlExpression * expr)
  787. {
  788. //MORE: This should probably be cached in the extra info & recursed more correctly
  789. node_operator op = expr->getOperator();
  790. ITypeInfo * type = expr->queryType();
  791. IHqlExpression * parallel = expr->queryAttribute(parallelAtom);
  792. if (parallel && getIntValue(parallel->queryChild(0), 0) != 1)
  793. return OptionYes;
  794. switch (op)
  795. {
  796. case no_constant:
  797. case no_field:
  798. case no_record:
  799. case no_attr:
  800. case no_attr_expr:
  801. case no_attr_link:
  802. case no_getresult:
  803. case no_left:
  804. case no_right:
  805. case no_sizeof:
  806. case no_all:
  807. case no_self:
  808. case no_activerow:
  809. case no_param:
  810. return OptionMaybe;
  811. case no_evaluate:
  812. throwUnexpected();
  813. case no_select:
  814. {
  815. bool isNew;
  816. IHqlExpression * ds = querySelectorDataset(expr, isNew);
  817. switch (ds->getOperator())
  818. {
  819. case no_getresult:
  820. case no_call:
  821. case no_externalcall:
  822. return normalizeThor(ds);
  823. case no_self:
  824. case no_param:
  825. return OptionMaybe;
  826. }
  827. return isNew ? OptionYes : OptionMaybe;
  828. }
  829. case NO_AGGREGATE:
  830. case no_executewhen:
  831. return OptionYes;
  832. case NO_ACTION_REQUIRES_GRAPH:
  833. {
  834. if ((op == no_output) && isTrivialInlineOutput(expr))
  835. return OptionMaybe;
  836. return OptionYes;
  837. }
  838. case no_sequential: // do not do inside thor - stops graphs being merged.
  839. if (!resourceSequential)
  840. return OptionNo;
  841. // fallthrough
  842. case no_actionlist:
  843. case no_orderedactionlist:
  844. case no_parallel:
  845. {
  846. YesNoOption option = OptionUnknown;
  847. ForEachChild(idx, expr)
  848. {
  849. YesNoOption childOption = normalizeThor(expr->queryChild(idx));
  850. if (childOption == OptionNo)
  851. return OptionNo;
  852. bool fixedOrder = (op == no_sequential) || (op == no_orderedactionlist);
  853. option = combine(option, childOption, fixedOrder); // can reorder parallel - so intersection is better
  854. }
  855. return option;
  856. }
  857. case no_cluster:
  858. case no_nothor:
  859. return OptionNo;
  860. case no_if:
  861. if (type && (type->getTypeCode() == type_void))
  862. {
  863. if (resourceConditionalActions)
  864. {
  865. IHqlExpression * falseExpr = expr->queryChild(2);
  866. YesNoOption leftOption = normalizeThor(expr->queryChild(1));
  867. YesNoOption rightOption = falseExpr ? normalizeThor(falseExpr) : OptionMaybe;
  868. YesNoOption branchOption = combine(leftOption, rightOption, true);
  869. YesNoOption condOption = normalizeThor(expr->queryChild(0));
  870. if ((branchOption == OptionYes) && (condOption != OptionNo))
  871. return OptionYes;
  872. // if ((condOption == OptionYes) && (branchOption != OptionNo))
  873. // return OptionYes;
  874. return combine(condOption, branchOption, true);
  875. }
  876. return OptionNo; // not supported
  877. }
  878. // default action. Not completely convinced it is correct....
  879. break;
  880. case no_choose:
  881. if (type && (type->getTypeCode() == type_void))
  882. {
  883. if (resourceConditionalActions)
  884. {
  885. UNIMPLEMENTED;
  886. }
  887. }
  888. break;
  889. case no_setresult:
  890. {
  891. IHqlExpression * value = expr->queryChild(0);
  892. YesNoOption valueOption = normalizeThor(value);
  893. //Probably worth doing the whole thing in thor if some part if it needs to be - will improve commoning up if nothing else.
  894. if (valueOption == OptionSome)
  895. return OptionYes;
  896. return valueOption;
  897. }
  898. case no_extractresult:
  899. {
  900. IHqlExpression * ds = expr->queryChild(0);
  901. OwnedHqlExpr transformedDs = transform(ds);
  902. if (transformedDs != ds)
  903. return OptionYes;
  904. //Probably worth doing the whole thing in thor if some part if it needs to be - will improve commoning up if nothing else.
  905. return normalizeThor(expr->queryChild(1));
  906. }
  907. case no_compound:
  908. {
  909. YesNoOption leftOption = normalizeThor(expr->queryChild(0));
  910. YesNoOption rightOption = normalizeThor(expr->queryChild(1));
  911. return combine(leftOption, rightOption, true);
  912. }
  913. case no_call:
  914. {
  915. YesNoOption bodyOption = normalizeThor(expr->queryBody()->queryFunctionDefinition());
  916. //do Something with it
  917. break;
  918. }
  919. case no_externalcall:
  920. {
  921. IHqlExpression * func = expr->queryExternalDefinition();
  922. IHqlExpression * funcDef = func->queryChild(0);
  923. if (funcDef->hasAttribute(gctxmethodAtom) || funcDef->hasAttribute(globalContextAtom))
  924. return OptionNo;
  925. // if (funcDef->hasAttribute(graphAtom))
  926. // return OptionYes;
  927. if (!resourceConditionalActions && expr->isAction())
  928. return OptionNo;
  929. //depends on the results of the arguments..
  930. type = NULL; // don't check the return type
  931. break;
  932. }
  933. case no_evaluate_stmt:
  934. return normalizeThor(expr->queryChild(0));
  935. case no_setworkflow_cond:
  936. case no_ensureresult:
  937. return OptionNo;
  938. case no_null:
  939. return OptionMaybe;
  940. }
  941. //NB: things like NOT EXISTS we want evaluated as NOT THOR(EXISTS()), or as part of a larger context
  942. //otherwise things like klogermann14.xhql don't get EXISTS() csed between graphs.
  943. YesNoOption option = OptionMaybe;
  944. ForEachChild(idx, expr)
  945. {
  946. YesNoOption childOption = normalizeThor(expr->queryChild(idx));
  947. if (childOption == OptionNo)
  948. return OptionNo;
  949. option = combine(option, childOption, true);
  950. }
  951. if (type)
  952. {
  953. switch (type->getTypeCode())
  954. {
  955. case type_row:
  956. switch (op)
  957. {
  958. case no_fromxml:
  959. case no_fromjson:
  960. return option;
  961. case no_createrow:
  962. //MORE: There are more cases that could be evaluated outside of thor, but playing safe.
  963. if (expr->queryChild(0)->isConstant())
  964. return option;
  965. break;
  966. }
  967. return OptionYes;
  968. case type_groupedtable:
  969. case type_table:
  970. //must be a dataset parameter to a call, or an argument to a comparison
  971. //Need to know whether it can be evaluate inline or not.
  972. //if it does require thor, then we will need to generate a setresult/get result pair to do it.
  973. return !canProcessInline(NULL, expr) ? OptionYes : OptionMaybe;
  974. }
  975. }
  976. return option;
  977. }
  978. void HqlThorBoundaryTransformer::transformRoot(const HqlExprArray & in, HqlExprArray & out)
  979. {
  980. //NewHqlTransformer::transformArray(in, out);
  981. //following theoretically might improve things, but generally just causes code to become worse,
  982. //because all the global set results are done in activities, and there isn't cse between them
  983. transformCompound(out, no_actionlist, in, maxRootMaybes);
  984. }
  985. void HqlCppTranslator::markThorBoundaries(WorkflowItem & curWorkflow)
  986. {
  987. HqlExprArray & exprs = curWorkflow.queryExprs();
  988. HqlExprArray bounded;
  989. HqlThorBoundaryTransformer thorTransformer(wu(), targetRoxie(), options.maxRootMaybeThorActions, options.resourceConditionalActions, options.resourceSequential);
  990. thorTransformer.transformRoot(exprs, bounded);
  991. replaceArray(exprs, bounded);
  992. }
  993. //---------------------------------------------------------------------------
  994. // NB: This is called after no_setresults are added, but before any normalization.
  995. IHqlExpression * ThorScalarTransformer::queryAlreadyTransformed(IHqlExpression * expr)
  996. {
  997. bool conditional = isConditional();
  998. if (conditional)
  999. {
  1000. IHqlExpression * ret = queryExtra(expr)->transformed[false];
  1001. if (ret)
  1002. return ret;
  1003. }
  1004. return queryExtra(expr)->transformed[isConditional()];
  1005. }
  1006. IHqlExpression * ThorScalarTransformer::queryAlreadyTransformedSelector(IHqlExpression * expr)
  1007. {
  1008. return queryExtra(expr)->transformedSelector[isConditional()];
  1009. }
  1010. void ThorScalarTransformer::setTransformed(IHqlExpression * expr, IHqlExpression * transformed)
  1011. {
  1012. queryExtra(expr)->transformed[isConditional()].set(transformed);
  1013. }
  1014. void ThorScalarTransformer::setTransformedSelector(IHqlExpression * expr, IHqlExpression * transformed)
  1015. {
  1016. queryExtra(expr)->transformedSelector[isConditional()].set(transformed);
  1017. }
  1018. static HqlTransformerInfo ThorScalarTransformerInfo("ThorScalarTransformer");
  1019. ThorScalarTransformer::ThorScalarTransformer(const HqlCppOptions & _options) : HoistingHqlTransformer(ThorScalarTransformerInfo, CTFnoteifactions), options(_options)
  1020. {
  1021. isConditionalDepth = 0;
  1022. seenCandidate = false;
  1023. }
  1024. void ThorScalarTransformer::doAnalyseExpr(IHqlExpression * expr)
  1025. {
  1026. switch (expr->getOperator())
  1027. {
  1028. case no_thor:
  1029. {
  1030. ITypeInfo * type = expr->queryType();
  1031. if (type && (type->isScalar() || type->getTypeCode() == type_row))
  1032. seenCandidate = true;
  1033. //No point looking further than a no_thor they don't (currently) get nested within each other.
  1034. return;
  1035. }
  1036. }
  1037. HoistingHqlTransformer::doAnalyseExpr(expr);
  1038. }
  1039. void ThorScalarTransformer::createHoisted(IHqlExpression * expr, SharedHqlExpr & setResultStmt, SharedHqlExpr & getResult, bool addWrapper)
  1040. {
  1041. IHqlExpression * value = expr;
  1042. HqlExprArray actions;
  1043. while (value->getOperator() == no_compound)
  1044. {
  1045. unwindCommaCompound(actions, value->queryChild(0));
  1046. value = value->queryChild(1);
  1047. }
  1048. IHqlExpression * setResult = createSetResult(value);
  1049. getResult.setown(createGetResultFromSetResult(setResult));
  1050. actions.append(*setResult);
  1051. setResultStmt.setown(createActionList(actions));
  1052. if (addWrapper)
  1053. setResultStmt.setown(createValue(no_thor, makeVoidType(), setResultStmt.getClear()));
  1054. }
  1055. IHqlExpression * ThorScalarTransformer::createTransformed(IHqlExpression * expr)
  1056. {
  1057. if (expr->isConstant())
  1058. {
  1059. //THOR(NULL) is marked as constant (probably incorrectly), but if it is constant can be evaluated inline
  1060. if ((expr->getOperator() == no_thor) && (expr->numChildren() != 0))
  1061. return LINK(expr->queryChild(0));
  1062. return LINK(expr);
  1063. }
  1064. IHqlExpression * ret = queryTransformAnnotation(expr);
  1065. if (ret)
  1066. return ret;
  1067. node_operator op = expr->getOperator();
  1068. switch (op)
  1069. {
  1070. case no_if:
  1071. case no_chooseds:
  1072. {
  1073. bool isGuard = isFailureGuard(expr);
  1074. HqlExprArray children;
  1075. children.append(*transform(expr->queryChild(0)));
  1076. if (!isGuard)
  1077. isConditionalDepth++;
  1078. transformChildren(expr, children);
  1079. if (!isGuard)
  1080. isConditionalDepth--;
  1081. return cloneOrLink(expr, children);
  1082. }
  1083. case no_thor:
  1084. {
  1085. ITypeInfo * type = expr->queryType();
  1086. // if (isUsedUnconditionally(expr) && type && (type->isScalar() || type->getTypeCode() == type_row))
  1087. if ((type->isScalar() || type->getTypeCode() == type_row))
  1088. {
  1089. //only other solution is to have some kind of graph result which is returned.
  1090. assertex(options.workunitTemporaries);
  1091. OwnedHqlExpr getResult, setResult;
  1092. OwnedHqlExpr transformedChild = transform(expr->queryChild(0));
  1093. createHoisted(transformedChild, setResult, getResult, true);
  1094. //Note sure if this condition is needed any more - I suspect better without.
  1095. if (isConditional())
  1096. return createCompound(setResult.getClear(), getResult.getClear());
  1097. else
  1098. {
  1099. appendToTarget(*setResult.getClear());
  1100. return getResult.getClear();
  1101. }
  1102. }
  1103. return LINK(expr);
  1104. }
  1105. break;
  1106. case no_mapto:
  1107. if (isConditional())
  1108. {
  1109. HqlExprArray children;
  1110. isConditionalDepth--;
  1111. children.append(*transform(expr->queryChild(0)));
  1112. isConditionalDepth++;
  1113. children.append(*transform(expr->queryChild(1)));
  1114. return cloneOrLink(expr, children);
  1115. }
  1116. break;
  1117. case no_case:
  1118. case no_map:
  1119. {
  1120. HqlExprArray children;
  1121. unsigned firstArg = 0;
  1122. unsigned numChildren = expr->numChildren();
  1123. children.ensure(numChildren);
  1124. if (op != no_map)
  1125. {
  1126. firstArg = 1;
  1127. children.append(*transform(expr->queryChild(0)));
  1128. }
  1129. isConditionalDepth++;
  1130. for (unsigned idx=firstArg; idx < numChildren; idx++)
  1131. children.append(*transform(expr->queryChild(idx)));
  1132. isConditionalDepth--;
  1133. return cloneOrLink(expr, children);
  1134. }
  1135. case no_table:
  1136. //Don't look at the default values for fields in the table's record
  1137. return LINK(expr);
  1138. }
  1139. return HoistingHqlTransformer::createTransformed(expr);
  1140. }
  1141. //---------------------------------------------------------------------------
  1142. /*
  1143. Look for expressions like count(x) and abc[1].x inside a condition and convert to
  1144. setresult(<complicated>, 'x')
  1145. getresult('x')
  1146. Problems:
  1147. o it tries to keep get/set results within the conditional branches that need them,
  1148. but that means if it occurs in more than one then it will only get added once.
  1149. o It (and all transformers) should possibly always generate a setresult,getresult locally,
  1150. and then have another pass that moves all the setresults to the optimal place. E.g. to the highest shared location.
  1151. [Alternatively can a global, but conditional, root graph be generated for it?]
  1152. Also converts
  1153. setresult(thor(x)) to thor(setresult(x))
  1154. thor(scalar) to setresult(scalar,'x'),getresult('x')
  1155. */
  1156. static void normalizeResultFormat(WorkflowItem & curWorkflow, const HqlCppOptions & options)
  1157. {
  1158. //Until thor has a way of calling a graph and returning a result we need to call this transformer, so that
  1159. //scalars that need to be evaluated in thor are correctly hoisted.
  1160. ThorScalarTransformer transformer(options);
  1161. HqlExprArray & exprs = curWorkflow.queryExprs();
  1162. transformer.analyseArray(exprs, 0);
  1163. if (transformer.needToTransform())
  1164. {
  1165. HqlExprArray transformed;
  1166. transformer.transformRoot(exprs, transformed);
  1167. replaceArray(exprs, transformed);
  1168. }
  1169. }
  1170. //---------------------------------------------------------------------------
  1171. //Try and get the HOLe queries at the start and the Thor queries at the end.
  1172. //Keep track of the dependencies of the statements, so that they don't get reordered
  1173. //too aggressively.
  1174. //---------------------------------------------------------------------------
  1175. static HqlTransformerInfo sequenceNumberAllocatorInfo("SequenceNumberAllocator");
  1176. SequenceNumberAllocator::SequenceNumberAllocator(HqlCppTranslator & _translator) : NewHqlTransformer(sequenceNumberAllocatorInfo), translator(_translator)
  1177. {
  1178. applyDepth = 0;
  1179. sequence = 0;
  1180. }
  1181. void SequenceNumberAllocator::nextSequence(HqlExprArray & args, IHqlExpression * name, IAtom * overwriteAction, IHqlExpression * value, bool needAttr, bool * duplicate)
  1182. {
  1183. IHqlExpression * seq = NULL;
  1184. if (duplicate)
  1185. *duplicate = false;
  1186. if (name)
  1187. {
  1188. SharedHqlExpr * matched = namedMap.getValue(name);
  1189. if (matched)
  1190. {
  1191. StringBuffer nameText;
  1192. name->toString(nameText);
  1193. IHqlExpression * prev = matched->get();
  1194. if (prev->isAttribute())
  1195. {
  1196. IAtom * prevName = prev->queryName();
  1197. if (!overwriteAction)
  1198. {
  1199. if (prevName == extendAtom)
  1200. throwError1(HQLERR_ExtendMismatch, nameText.str());
  1201. else
  1202. throwError1(HQLERR_OverwriteMismatch, nameText.str());
  1203. }
  1204. else if (prevName != overwriteAction)
  1205. throwError1(HQLERR_ExtendOverwriteMismatch, nameText.str());
  1206. IHqlExpression * prevValue = prev->queryChild(1);
  1207. if (!recordTypesMatch(prevValue->queryType(), value->queryType()))
  1208. throwError1(HQLERR_ExtendTypeMismatch, nameText.str());
  1209. seq = LINK(prev->queryChild(0));
  1210. assertex(duplicate);
  1211. *duplicate = true;
  1212. }
  1213. else
  1214. {
  1215. if (overwriteAction)
  1216. {
  1217. if (overwriteAction == extendAtom)
  1218. throwError1(HQLERR_ExtendMismatch, nameText.str());
  1219. else
  1220. throwError1(HQLERR_OverwriteMismatch, nameText.str());
  1221. }
  1222. else
  1223. throwError1(HQLERR_DuplicateNameOutput, nameText.str());
  1224. }
  1225. }
  1226. if (!seq)
  1227. {
  1228. seq = createConstant(signedType->castFrom(true, (__int64)sequence++));
  1229. OwnedHqlExpr saveValue = overwriteAction ? createAttribute(overwriteAction, LINK(seq), LINK(value)) : LINK(seq);
  1230. namedMap.setValue(name, saveValue);
  1231. }
  1232. }
  1233. else
  1234. seq = createConstant(signedType->castFrom(true, (__int64)sequence++));
  1235. if (needAttr)
  1236. args.append(*createAttribute(sequenceAtom, seq));
  1237. else
  1238. args.append(*seq);
  1239. }
  1240. IHqlExpression * SequenceNumberAllocator::doTransformRootExpr(IHqlExpression * expr)
  1241. {
  1242. node_operator op = expr->getOperator();
  1243. switch(op)
  1244. {
  1245. case no_compound:
  1246. case no_comma:
  1247. case no_parallel:
  1248. case no_sequential:
  1249. case no_actionlist:
  1250. case no_orderedactionlist:
  1251. {
  1252. HqlExprArray args;
  1253. ForEachChild(idx, expr)
  1254. args.append(*doTransformRootExpr(expr->queryChild(idx)));
  1255. return cloneOrLink(expr, args);
  1256. }
  1257. case no_buildindex:
  1258. case no_output:
  1259. case no_apply:
  1260. case no_distribution:
  1261. case no_keydiff:
  1262. case no_keypatch:
  1263. case no_outputscalar:
  1264. return createTransformed(expr); //NB: Do not common up!!!
  1265. case no_setmeta:
  1266. return LINK(expr);
  1267. default:
  1268. {
  1269. OwnedHqlExpr transformed = transform(expr);
  1270. ITypeInfo * type = transformed->queryType();
  1271. if (type && type->getTypeCode() != type_void)
  1272. {
  1273. HqlExprArray args;
  1274. bool isOuterWorkflow = (op == no_colon) && workflowContainsSchedule(transformed);
  1275. assertex(!isOuterWorkflow || !workflowContainsNonSchedule(transformed));
  1276. if (isOuterWorkflow)
  1277. args.append(*LINK(transformed->queryChild(0)));
  1278. else
  1279. args.append(*LINK(transformed));
  1280. nextSequence(args, NULL, NULL, NULL, true, NULL);
  1281. IHqlExpression * ret = createSetResult(args);
  1282. if (isOuterWorkflow)
  1283. {
  1284. args.kill();
  1285. args.append(*ret);
  1286. unwindChildren(args, transformed, 1);
  1287. ret = transformed->clone(args);
  1288. }
  1289. return ret;
  1290. }
  1291. return LINK(transformed);
  1292. }
  1293. }
  1294. }
  1295. IHqlExpression * SequenceNumberAllocator::createTransformed(IHqlExpression * expr)
  1296. {
  1297. switch (expr->getOperator())
  1298. {
  1299. case no_actionlist:
  1300. return doTransformRootExpr(expr);
  1301. case no_apply:
  1302. {
  1303. HqlExprArray args;
  1304. args.append(*transform(expr->queryChild(0)));
  1305. applyDepth++;
  1306. args.append(*transform(expr->queryChild(1)));
  1307. applyDepth--;
  1308. OwnedHqlExpr ret = completeTransform(expr, args);
  1309. return attachSequenceNumber(ret);
  1310. }
  1311. case no_outputscalar:
  1312. if (applyDepth)
  1313. {
  1314. translator.WARNINGAT(CategoryUnexpected, expr, HQLERR_ScalarOutputWithinApply);
  1315. }
  1316. break;
  1317. case no_parallel:
  1318. case no_sequential:
  1319. case no_orderedactionlist:
  1320. {
  1321. HqlExprArray args;
  1322. ForEachChild(i, expr)
  1323. {
  1324. OwnedHqlExpr next = transform(expr->queryChild(i));
  1325. if (!next->isAction())
  1326. {
  1327. if (!next->isConstant())
  1328. next.setown(createValue(no_evaluate_stmt, makeVoidType(), next.getClear()));
  1329. else
  1330. next.clear();
  1331. }
  1332. if (next)
  1333. args.append(*next.getClear());
  1334. }
  1335. return expr->clone(args);
  1336. }
  1337. }
  1338. Owned<IHqlExpression> transformed = NewHqlTransformer::createTransformed(expr);
  1339. return attachSequenceNumber(transformed.get());
  1340. }
  1341. static IAtom * queryOverwriteAction(IHqlExpression * expr)
  1342. {
  1343. if (expr->hasAttribute(extendAtom))
  1344. return extendAtom;
  1345. if (expr->hasAttribute(overwriteAtom))
  1346. return overwriteAtom;
  1347. if (expr->hasAttribute(noOverwriteAtom))
  1348. return noOverwriteAtom;
  1349. return NULL;
  1350. }
  1351. IHqlExpression * SequenceNumberAllocator::attachSequenceNumber(IHqlExpression * expr)
  1352. {
  1353. switch (expr->getOperator())
  1354. {
  1355. case no_buildindex:
  1356. case no_output:
  1357. case no_apply:
  1358. case no_distribution:
  1359. case no_keydiff:
  1360. case no_keypatch:
  1361. {
  1362. queryExtra(expr)->setGetsSequence();
  1363. HqlExprArray args;
  1364. unwindChildren(args, expr);
  1365. bool duplicate = false;
  1366. nextSequence(args, queryResultName(expr), queryOverwriteAction(expr), expr->queryChild(0), true, &duplicate);
  1367. args.append(*createUniqueId());
  1368. return expr->clone(args);
  1369. }
  1370. break;
  1371. case no_outputscalar:
  1372. {
  1373. IHqlExpression * name = queryResultName(expr);
  1374. queryExtra(expr)->setGetsSequence();
  1375. HqlExprArray args;
  1376. args.append(*LINK(expr->queryChild(0)));
  1377. bool duplicate = false;
  1378. nextSequence(args, name, queryOverwriteAction(expr), expr->queryChild(0), true, &duplicate);
  1379. if (name)
  1380. args.append(*createAttribute(namedAtom, LINK(name)));
  1381. args.append(*createAttribute(outputAtom));
  1382. args.append(*createUniqueId());
  1383. IHqlExpression *noXpath = expr->queryAttribute(noXpathAtom);
  1384. if (noXpath)
  1385. args.append(*LINK(noXpath));
  1386. gatherAttributes(args, xmlnsAtom, expr);
  1387. return createSetResult(args);
  1388. }
  1389. default:
  1390. return LINK(expr);
  1391. }
  1392. }
  1393. void HqlCppTranslator::allocateSequenceNumbers(HqlExprArray & exprs)
  1394. {
  1395. HqlExprArray sequenced;
  1396. SequenceNumberAllocator transformer(*this);
  1397. transformer.transformRoot(exprs, sequenced);
  1398. replaceArray(exprs, sequenced);
  1399. maxSequence = transformer.getMaxSequence();
  1400. }
  1401. //---------------------------------------------------------------------------
  1402. static void replaceAssignSelector(HqlExprArray & assigns, IHqlExpression * newSelector)
  1403. {
  1404. ForEachItemIn(idx, assigns)
  1405. {
  1406. IHqlExpression & cur = assigns.item(idx);
  1407. IHqlExpression * lhs = cur.queryChild(0);
  1408. IHqlExpression * rhs = cur.queryChild(1);
  1409. assigns.replace(*createAssign(replaceSelector(lhs, queryActiveTableSelector(), newSelector), replaceSelector(rhs, queryActiveTableSelector(), newSelector)), idx);
  1410. }
  1411. }
  1412. static IHqlExpression * createArith(node_operator op, ITypeInfo * type, IHqlExpression * numerator, IHqlExpression * denominator)
  1413. {
  1414. return createValue(op, LINK(type), ensureExprType(numerator, type), ensureExprType(denominator, type));
  1415. }
  1416. class AggregateNormalizer
  1417. {
  1418. public:
  1419. AggregateNormalizer(IHqlExpression * _rootSelector, HqlExprArray & _fields, HqlExprArray & _assigns, bool & _extraSelectNeeded)
  1420. : rootSelector(_rootSelector), fields(_fields), assigns(_assigns), extraSelectNeeded(_extraSelectNeeded)
  1421. {
  1422. }
  1423. IHqlExpression * normalizeAggregateExpr(IHqlExpression * selector, IHqlExpression * expr, bool canOptimizeCasts)
  1424. {
  1425. IHqlExpression * match = static_cast<IHqlExpression *>(expr->queryTransformExtra());
  1426. if (match)
  1427. return LINK(match);
  1428. IHqlExpression * ret = evalNormalizeAggregateExpr(selector, expr, canOptimizeCasts);
  1429. expr->setTransformExtra(ret);
  1430. return ret;
  1431. }
  1432. protected:
  1433. IHqlExpression * evalNormalizeAggregateExpr(IHqlExpression * selector, IHqlExpression * expr, bool canOptimizeCasts)
  1434. {
  1435. if (!expr->isGroupAggregateFunction() && !expr->usesSelector(rootSelector))
  1436. return LINK(expr);
  1437. switch (expr->getOperator())
  1438. {
  1439. case no_avegroup:
  1440. //Map this to sum(x)/count(x)
  1441. {
  1442. IHqlExpression * arg = expr->queryChild(0);
  1443. IHqlExpression * cond = expr->queryChild(1);
  1444. Owned<ITypeInfo> sumType = getSumAggType(arg);
  1445. ITypeInfo * exprType = expr->queryType();
  1446. OwnedHqlExpr sum = createValue(no_sumgroup, LINK(sumType), LINK(arg), LINK(cond));
  1447. OwnedHqlExpr count = createValue(no_countgroup, LINK(defaultIntegralType), LINK(cond));
  1448. //average should be done as a real operation I think, possibly decimal, if argument is decimal
  1449. OwnedHqlExpr avg = createArith(no_div, exprType, sum, count);
  1450. return normalizeAggregateExpr(selector, avg, false);
  1451. }
  1452. case no_vargroup:
  1453. //Map this to (sum(x^2)-sum(x)^2/count())/count()
  1454. {
  1455. IHqlExpression * arg = expr->queryChild(0);
  1456. IHqlExpression * cond = expr->queryChild(1);
  1457. ITypeInfo * exprType = expr->queryType();
  1458. OwnedHqlExpr xx = createArith(no_mul, exprType, arg, arg);
  1459. OwnedHqlExpr sumxx = createValue(no_sumgroup, LINK(exprType), LINK(xx), LINK(cond));
  1460. OwnedHqlExpr sumx = createValue(no_sumgroup, LINK(exprType), LINK(arg), LINK(cond));
  1461. OwnedHqlExpr count = createValue(no_countgroup, LINK(defaultIntegralType), LINK(cond));
  1462. //average should be done as a real operation I think, possibly decimal, if argument is decimal
  1463. OwnedHqlExpr n1 = createArith(no_mul, exprType, sumx, sumx);
  1464. OwnedHqlExpr n2 = createArith(no_div, exprType, n1, count);
  1465. OwnedHqlExpr n3 = createArith(no_sub, exprType, sumxx, n2);
  1466. OwnedHqlExpr n4 = createArith(no_div, exprType, n3, count);
  1467. return normalizeAggregateExpr(selector, n4, false);
  1468. }
  1469. case no_covargroup:
  1470. //Map this to (sum(x.y)-sum(x).sum(y)/count())/count()
  1471. {
  1472. IHqlExpression * argX = expr->queryChild(0);
  1473. IHqlExpression * argY = expr->queryChild(1);
  1474. IHqlExpression * cond = expr->queryChild(2);
  1475. ITypeInfo * exprType = expr->queryType();
  1476. OwnedHqlExpr xy = createArith(no_mul, exprType, argX, argY);
  1477. OwnedHqlExpr sumxy = createValue(no_sumgroup, LINK(exprType), LINK(xy), LINK(cond));
  1478. OwnedHqlExpr sumx = createValue(no_sumgroup, LINK(exprType), LINK(argX), LINK(cond));
  1479. OwnedHqlExpr sumy = createValue(no_sumgroup, LINK(exprType), LINK(argY), LINK(cond));
  1480. OwnedHqlExpr count = createValue(no_countgroup, LINK(defaultIntegralType), LINK(cond));
  1481. //average should be done as a real operation I think, possibly decimal, if argument is decimal
  1482. OwnedHqlExpr n1 = createArith(no_mul, exprType, sumx, sumy);
  1483. OwnedHqlExpr n2 = createArith(no_div, exprType, n1, count);
  1484. OwnedHqlExpr n3 = createArith(no_sub, exprType, sumxy, n2);
  1485. OwnedHqlExpr n4 = createArith(no_div, exprType, n3, count);
  1486. return normalizeAggregateExpr(selector, n4, false);
  1487. }
  1488. case no_corrgroup:
  1489. //Map this to (covar(x,y)/(var(x).var(y)))
  1490. //== (sum(x.y)*count() - sum(x).sum(y))/sqrt((sum(x.x)*count()-sum(x)^2) * (sum(y.y)*count()-sum(y)^2))
  1491. {
  1492. IHqlExpression * argX = expr->queryChild(0);
  1493. IHqlExpression * argY = expr->queryChild(1);
  1494. IHqlExpression * cond = expr->queryChild(2);
  1495. ITypeInfo * exprType = expr->queryType();
  1496. OwnedHqlExpr xx = createArith(no_mul, exprType, argX, argX);
  1497. OwnedHqlExpr sumxx = createValue(no_sumgroup, LINK(exprType), LINK(xx), LINK(cond));
  1498. OwnedHqlExpr xy = createArith(no_mul, exprType, argX, argY);
  1499. OwnedHqlExpr sumxy = createValue(no_sumgroup, LINK(exprType), LINK(xy), LINK(cond));
  1500. OwnedHqlExpr yy = createArith(no_mul, exprType, argY, argY);
  1501. OwnedHqlExpr sumyy = createValue(no_sumgroup, LINK(exprType), LINK(yy), LINK(cond));
  1502. OwnedHqlExpr sumx = createValue(no_sumgroup, LINK(exprType), LINK(argX), LINK(cond));
  1503. OwnedHqlExpr sumy = createValue(no_sumgroup, LINK(exprType), LINK(argY), LINK(cond));
  1504. OwnedHqlExpr count = createValue(no_countgroup, LINK(defaultIntegralType), LINK(cond));
  1505. OwnedHqlExpr n1 = createArith(no_mul, exprType, sumxy, count);
  1506. OwnedHqlExpr n2 = createArith(no_mul, exprType, sumx, sumy);
  1507. OwnedHqlExpr n3 = createArith(no_sub, exprType, n1, n2);
  1508. OwnedHqlExpr n4 = createArith(no_mul, exprType, sumxx, count);
  1509. OwnedHqlExpr n5 = createArith(no_mul, exprType, sumx, sumx);
  1510. OwnedHqlExpr n6 = createArith(no_sub, exprType, n4, n5);
  1511. OwnedHqlExpr n7 = createArith(no_mul, exprType, sumyy, count);
  1512. OwnedHqlExpr n8 = createArith(no_mul, exprType, sumy, sumy);
  1513. OwnedHqlExpr n9 = createArith(no_sub, exprType, n7, n8);
  1514. OwnedHqlExpr n10 = createArith(no_mul, exprType, n6, n9);
  1515. OwnedHqlExpr n11 = createValue(no_sqrt, LINK(exprType), LINK(n10));
  1516. OwnedHqlExpr n12 = createArith(no_div, exprType, n3, n11);
  1517. return normalizeAggregateExpr(selector, n12, false);
  1518. }
  1519. throwUnexpected();
  1520. case no_variance:
  1521. case no_covariance:
  1522. case no_correlation:
  1523. throwUnexpectedOp(expr->getOperator());
  1524. case no_select:
  1525. case no_count:
  1526. case no_sum:
  1527. case no_max:
  1528. case no_min:
  1529. case no_ave:
  1530. case no_exists:
  1531. //Fall through - and add the expression as a field in the resulting aggregate which is later projected
  1532. case no_countgroup:
  1533. case no_sumgroup:
  1534. case no_maxgroup:
  1535. case no_mingroup:
  1536. case no_existsgroup:
  1537. {
  1538. ForEachItemIn(idx, assigns)
  1539. {
  1540. IHqlExpression & cur = assigns.item(idx);
  1541. if (cur.queryChild(1) == expr)
  1542. {
  1543. extraSelectNeeded = true;
  1544. return LINK(cur.queryChild(0)); //replaceSelector(cur.queryChild(0), querySelf(), queryActiveTableSelector());
  1545. }
  1546. }
  1547. IHqlExpression * targetField;
  1548. if (selector)
  1549. {
  1550. targetField = LINK(selector->queryChild(1));
  1551. }
  1552. else
  1553. {
  1554. StringBuffer temp;
  1555. temp.append("_agg_").append(assigns.ordinality());
  1556. targetField = createFieldFromValue(createIdAtom(temp.str()), expr);
  1557. extraSelectNeeded = true;
  1558. }
  1559. fields.append(*targetField);
  1560. assigns.append(*createAssign(createSelectExpr(getActiveTableSelector(), LINK(targetField)), LINK(expr)));
  1561. return createSelectExpr(getActiveTableSelector(), LINK(targetField));
  1562. }
  1563. case no_cast:
  1564. case no_implicitcast:
  1565. if (selector && canOptimizeCasts)
  1566. {
  1567. IHqlExpression * child = expr->queryChild(0);
  1568. if (expr->queryType()->getTypeCode() == child->queryType()->getTypeCode())
  1569. {
  1570. IHqlExpression * ret = normalizeAggregateExpr(selector, child, false);
  1571. //This should be ret==child
  1572. if (ret == selector)
  1573. return ret;
  1574. HqlExprArray args;
  1575. args.append(*ret);
  1576. return expr->clone(args);
  1577. }
  1578. }
  1579. //fallthrough...
  1580. default:
  1581. {
  1582. HqlExprArray args;
  1583. unsigned max = expr->numChildren();
  1584. unsigned idx;
  1585. bool diff = false;
  1586. args.ensure(max);
  1587. for (idx = 0; idx < max; idx++)
  1588. {
  1589. IHqlExpression * child = expr->queryChild(idx);
  1590. IHqlExpression * changed = normalizeAggregateExpr(NULL, child, false);
  1591. args.append(*changed);
  1592. if (child != changed)
  1593. diff = true;
  1594. }
  1595. if (diff)
  1596. return expr->clone(args);
  1597. return LINK(expr);
  1598. }
  1599. }
  1600. }
  1601. protected:
  1602. TransformMutexBlock block;
  1603. IHqlExpression * rootSelector;
  1604. HqlExprArray & fields;
  1605. HqlExprArray & assigns;
  1606. bool & extraSelectNeeded;
  1607. };
  1608. IHqlExpression * normalizeAggregateExpr(IHqlExpression * tableSelector, IHqlExpression * selector, IHqlExpression * expr, HqlExprArray & fields, HqlExprArray & assigns, bool & extraSelectNeeded, bool canOptimizeCasts)
  1609. {
  1610. AggregateNormalizer normalizer(tableSelector, fields, assigns, extraSelectNeeded);
  1611. return normalizer.normalizeAggregateExpr(selector, expr, canOptimizeCasts);
  1612. }
  1613. //---------------------------------------------------------------------------
  1614. class SortListSimplifier
  1615. {
  1616. public:
  1617. IHqlExpression * simplify(IHqlExpression * expr);
  1618. protected:
  1619. void appendComponent(bool invert, IHqlExpression * expr);
  1620. void expandRowComponents(bool invert, IHqlExpression * select, IHqlExpression * record);
  1621. protected:
  1622. TransformMutexBlock block;
  1623. HqlExprArray cpts;
  1624. };
  1625. void SortListSimplifier::appendComponent(bool invert, IHqlExpression * expr)
  1626. {
  1627. //If already sorted by x or -x then no point sorting again by the same expression.
  1628. if (!expr->queryTransformExtra())
  1629. {
  1630. expr->setTransformExtraUnlinked(expr);
  1631. if (invert)
  1632. cpts.append(*createValue(no_negate, expr->getType(), LINK(expr)));
  1633. else
  1634. cpts.append(*LINK(expr));
  1635. }
  1636. }
  1637. void SortListSimplifier::expandRowComponents(bool invert, IHqlExpression * select, IHqlExpression * record)
  1638. {
  1639. ForEachChild(i, record)
  1640. {
  1641. IHqlExpression * cur = record->queryChild(i);
  1642. switch (cur->getOperator())
  1643. {
  1644. case no_record:
  1645. expandRowComponents(invert, select, cur);
  1646. break;
  1647. case no_ifblock:
  1648. expandRowComponents(invert, select, cur->queryChild(1));
  1649. break;
  1650. case no_field:
  1651. {
  1652. OwnedHqlExpr childSelect = createSelectExpr(LINK(select), LINK(cur));
  1653. if (!childSelect->isDatarow())
  1654. appendComponent(invert, childSelect);
  1655. else
  1656. expandRowComponents(invert, childSelect, childSelect->queryRecord());
  1657. break;
  1658. }
  1659. }
  1660. }
  1661. }
  1662. IHqlExpression * SortListSimplifier::simplify(IHqlExpression * sortlist)
  1663. {
  1664. //convert concat on fixed width strings to a list of fields.
  1665. bool same = true;
  1666. ForEachChild(idx, sortlist)
  1667. {
  1668. IHqlExpression * cpt = sortlist->queryChild(idx);
  1669. IHqlExpression * cur = cpt;
  1670. bool expand = false;
  1671. bool invert = false;
  1672. if (cpt->getOperator() == no_negate)
  1673. {
  1674. invert = true;
  1675. cur = cur->queryChild(0);
  1676. }
  1677. if (cur->getOperator() == no_concat)
  1678. {
  1679. HqlExprArray concats;
  1680. cur->unwindList(concats, no_concat);
  1681. expand = true;
  1682. ForEachItemIn(idxc, concats)
  1683. {
  1684. ITypeInfo * type = concats.item(idxc).queryType();
  1685. unsigned tc = type->getTypeCode();
  1686. if (!((tc == type_string || tc == type_data) && (type->getSize() != UNKNOWN_LENGTH)))
  1687. expand = false;
  1688. }
  1689. if (expand)
  1690. {
  1691. ForEachItemIn(idxc, concats)
  1692. appendComponent(invert, &concats.item(idxc));
  1693. }
  1694. }
  1695. else if (cur->getOperator() == no_trim)
  1696. {
  1697. //Strings are always compared as if they are trimmed (or padded with arbitrary spaces)
  1698. //=> sort by TRIM(string) can just sort by the string instead. (Don't match LEFT/RIGHT versions.)
  1699. if (!cur->queryChild(1))
  1700. {
  1701. IHqlExpression * arg = cur->queryChild(0);
  1702. ITypeInfo * argType = arg->queryType();
  1703. if (cur->queryType()->getTypeCode() == argType->getTypeCode())
  1704. {
  1705. expand = true;
  1706. appendComponent(invert, arg);
  1707. }
  1708. }
  1709. }
  1710. else if (isCast(cur) && castPreservesValueAndOrder(cur))
  1711. {
  1712. expand = true;
  1713. appendComponent(invert, cur->queryChild(0));
  1714. }
  1715. else
  1716. {
  1717. #if 0
  1718. if (cur->getOperator() == no_select && cur->isDatarow() && !cur->hasAttribute(newAtom))
  1719. {
  1720. expand = true;
  1721. expandRowComponents(invert, cur, cur->queryRecord());
  1722. }
  1723. #endif
  1724. if (cur->queryTransformExtra())
  1725. expand = true;
  1726. if (cur->isConstant() && cpts.ordinality())
  1727. expand = true;
  1728. }
  1729. if (!expand)
  1730. {
  1731. cur->setTransformExtraUnlinked(cur);
  1732. cpts.append(*LINK(cpt));
  1733. }
  1734. else
  1735. same = false;
  1736. }
  1737. if (!same)
  1738. return createSortList(cpts);
  1739. return NULL;
  1740. }
  1741. static IHqlExpression * simplifySortlistComplexity(IHqlExpression * sortlist)
  1742. {
  1743. if (!sortlist)
  1744. return NULL;
  1745. SortListSimplifier transformer;
  1746. return transformer.simplify(sortlist);
  1747. }
  1748. static IHqlExpression * normalizeIndexBuild(IHqlExpression * expr, bool sortIndexPayload, bool alwaysLocal, bool allowImplicitSubSort)
  1749. {
  1750. LinkedHqlExpr dataset = expr->queryChild(0);
  1751. IHqlExpression * buildRecord = dataset->queryRecord();
  1752. // If any field types collate differently before and after translation to their hozed
  1753. // format, then we need to do the translation here, otherwise this
  1754. // sort may not be in the correct order. (ebcdic->ascii? integers are ok; unicode isn't!)
  1755. // First build the sort order we need....
  1756. HqlExprArray sorts;
  1757. gatherIndexBuildSortOrder(sorts, expr, sortIndexPayload);
  1758. OwnedHqlExpr sortOrder = createSortList(sorts);
  1759. OwnedHqlExpr newsort = simplifySortlistComplexity(sortOrder);
  1760. if (!newsort)
  1761. newsort.set(sortOrder);
  1762. ForEachChild(i1, expr)
  1763. {
  1764. IHqlExpression * cur = expr->queryChild(i1);
  1765. if (cur->getOperator() == no_distributer)
  1766. {
  1767. LinkedHqlExpr ds = dataset;
  1768. IHqlExpression * index = cur->queryChild(0);
  1769. if (!expr->hasAttribute(sortedAtom))
  1770. {
  1771. if (!expr->hasAttribute(localAtom))
  1772. {
  1773. HqlExprArray joinCondition;
  1774. IHqlExpression * indexRecord = index->queryChild(1);
  1775. assertex(indexRecord->numChildren() == buildRecord->numChildren());
  1776. unsigned numFields = firstPayloadField(index);
  1777. OwnedHqlExpr seq = createSelectorSequence();
  1778. OwnedHqlExpr left = createSelector(no_left, dataset, seq);
  1779. OwnedHqlExpr right = createSelector(no_right, index, seq);
  1780. OwnedHqlExpr cond;
  1781. unsigned idxLhs = 0;
  1782. unsigned idxRhs = 0;
  1783. for (unsigned i2=0; i2 < numFields; i2++)
  1784. {
  1785. IHqlExpression * lhs = createSelectExpr(LINK(left), LINK(queryNextRecordField(buildRecord, idxLhs)));
  1786. IHqlExpression * rhs = createSelectExpr(LINK(right), LINK(queryNextRecordField(indexRecord, idxRhs)));
  1787. IHqlExpression * test = createBoolExpr(no_eq, lhs, rhs);
  1788. extendConditionOwn(cond, no_and, test);
  1789. }
  1790. HqlExprArray args;
  1791. args.append(*ds.getClear());
  1792. args.append(*LINK(index));
  1793. args.append(*cond.getClear());
  1794. args.append(*LINK(seq));
  1795. ds.setown(createDataset(no_keyeddistribute, args));
  1796. ds.setown(cloneInheritedAnnotations(expr, ds));
  1797. }
  1798. ds.setown(createDataset(no_sort, ds.getClear(), createComma(LINK(newsort), createLocalAttribute())));
  1799. ds.setown(cloneInheritedAnnotations(expr, ds));
  1800. }
  1801. if (expr->hasAttribute(mergeAtom))
  1802. {
  1803. LinkedHqlExpr sortedIndex = index;
  1804. if (!index->hasAttribute(sortedAtom))
  1805. {
  1806. HqlExprArray args;
  1807. unwindChildren(args, index);
  1808. args.append(*createAttribute(sortedAtom));
  1809. sortedIndex.setown(index->clone(args));
  1810. }
  1811. HqlExprArray sorts;
  1812. unwindChildren(sorts, newsort);
  1813. OwnedHqlExpr sortAttr = createExprAttribute(sortedAtom, sorts);
  1814. HqlExprArray args;
  1815. args.append(*LINK(ds));
  1816. args.append(*sortedIndex.getClear());
  1817. args.append(*createLocalAttribute());
  1818. args.append(*replaceSelector(sortAttr, ds->queryNormalizedSelector(), queryActiveTableSelector()));
  1819. ds.setown(createDataset(no_merge, args));
  1820. ds.setown(cloneInheritedAnnotations(expr, ds));
  1821. }
  1822. HqlExprArray args;
  1823. unwindChildren(args, expr);
  1824. args.replace(*ds.getClear(), 0);
  1825. args.remove(i1);
  1826. args.append(*createAttribute(sortedAtom));
  1827. args.append(*createLocalAttribute());
  1828. args.append(*createAttribute(indexAtom, LINK(index->queryChild(3))));
  1829. return expr->clone(args);
  1830. }
  1831. }
  1832. IHqlExpression * distributed = expr->queryAttribute(distributedAtom);
  1833. if (distributed && distributed->queryChild(0))
  1834. {
  1835. OwnedHqlExpr distribute = createDataset(no_distribute, LINK(dataset), LINK(distributed->queryChild(0)));
  1836. distribute.setown(cloneInheritedAnnotations(expr, distribute));
  1837. HqlExprArray args;
  1838. args.append(*distribute.getClear());
  1839. unwindChildren(args, expr, 1);
  1840. args.zap(*distributed);
  1841. return expr->clone(args);
  1842. }
  1843. if (!expr->hasAttribute(sortedAtom))
  1844. {
  1845. if (dataset->queryType()->getTypeCode() == type_groupedtable)
  1846. {
  1847. while (dataset->getOperator() == no_group)
  1848. dataset.set(dataset->queryChild(0));
  1849. if (dataset->queryType()->getTypeCode() == type_groupedtable)
  1850. {
  1851. dataset.setown(createDataset(no_group, LINK(dataset), NULL));
  1852. dataset.setown(cloneInheritedAnnotations(expr, dataset));
  1853. }
  1854. }
  1855. OwnedHqlExpr sorted = ensureSorted(dataset, newsort, expr, expr->hasAttribute(localAtom), true, alwaysLocal, allowImplicitSubSort, true);
  1856. if (sorted == dataset)
  1857. return NULL;
  1858. sorted.setown(inheritAttribute(sorted, expr, skewAtom));
  1859. sorted.setown(inheritAttribute(sorted, expr, thresholdAtom));
  1860. HqlExprArray args;
  1861. args.append(*LINK(sorted));
  1862. unwindChildren(args, expr, 1);
  1863. args.append(*createAttribute(sortedAtom));
  1864. return expr->clone(args);
  1865. }
  1866. if (expr->hasAttribute(dedupAtom))
  1867. {
  1868. IHqlExpression * ds = expr->queryChild(0);
  1869. OwnedHqlExpr seq = createSelectorSequence();
  1870. OwnedHqlExpr mappedSortList = replaceSelector(newsort, queryActiveTableSelector(), ds);
  1871. HqlExprArray dedupArgs;
  1872. dedupArgs.append(*LINK(expr->queryChild(0)));
  1873. unwindChildren(dedupArgs, mappedSortList);
  1874. dedupArgs.append(*createLocalAttribute());
  1875. dedupArgs.append(*LINK(seq));
  1876. OwnedHqlExpr dedup = createDataset(no_dedup, dedupArgs);
  1877. HqlExprArray buildArgs;
  1878. buildArgs.append(*cloneInheritedAnnotations(expr, dedup));
  1879. unwindChildren(buildArgs, expr, 1);
  1880. removeAttribute(buildArgs, dedupAtom);
  1881. return expr->clone(buildArgs);
  1882. }
  1883. return NULL;
  1884. }
  1885. static IHqlExpression *getEmbedOptionString(IHqlExpression *funcdef)
  1886. {
  1887. IHqlExpression * outofline = funcdef->queryChild(0);
  1888. IHqlExpression * bodyCode = outofline->queryChild(0);
  1889. HqlExprArray attrArgs;
  1890. ForEachChild(idx, bodyCode)
  1891. {
  1892. IHqlExpression *child = bodyCode->queryChild(idx);
  1893. if (child->isAttribute() && !isInternalEmbedAttr(child->queryName()))
  1894. {
  1895. StringBuffer attrParam;
  1896. if (attrArgs.ordinality())
  1897. attrParam.append(",");
  1898. attrParam.append(child->queryName());
  1899. IHqlExpression * value = child->queryChild(0);
  1900. if (value)
  1901. attrParam.append("=");
  1902. attrArgs.append(*createConstant(attrParam));
  1903. if (value)
  1904. attrArgs.append(*ensureExprType(value, unknownStringType));
  1905. }
  1906. }
  1907. OwnedHqlExpr folded;
  1908. if (attrArgs.length())
  1909. {
  1910. OwnedHqlExpr concat = createUnbalanced(no_concat, unknownStringType, attrArgs);
  1911. OwnedHqlExpr cast = ensureExprType(concat, unknownVarStringType);
  1912. folded.setown(foldHqlExpression(cast));
  1913. }
  1914. else
  1915. folded.setown(createBlankString());
  1916. return folded.getClear();
  1917. }
  1918. static HqlTransformerInfo thorHqlTransformerInfo("ThorHqlTransformer");
  1919. ThorHqlTransformer::ThorHqlTransformer(HqlCppTranslator & _translator, ClusterType _targetClusterType, IConstWorkUnit * wu, unsigned & _implicitFunctionId)
  1920. : NewHqlTransformer(thorHqlTransformerInfo), translator(_translator), options(_translator.queryOptions()), implicitFunctionId(_implicitFunctionId)
  1921. {
  1922. targetClusterType = _targetClusterType;
  1923. topNlimit = options.topnLimit;
  1924. groupAllDistribute = isThorCluster(targetClusterType) && options.groupAllDistribute;
  1925. }
  1926. IHqlExpression * ThorHqlTransformer::createTransformed(IHqlExpression * expr)
  1927. {
  1928. OwnedHqlExpr transformed = PARENT::createTransformed(expr);
  1929. updateOrphanedSelectors(transformed, expr);
  1930. IHqlExpression * normalized = NULL;
  1931. switch (transformed->getOperator())
  1932. {
  1933. case no_group:
  1934. normalized = normalizeGroup(transformed);
  1935. break;
  1936. case no_join:
  1937. case no_selfjoin:
  1938. case no_denormalize:
  1939. case no_denormalizegroup:
  1940. if (transformed->hasAttribute(groupAtom))
  1941. normalized = normalizeJoinAndGroup(transformed);
  1942. else
  1943. normalized = normalizeJoinOrDenormalize(transformed);
  1944. break;
  1945. case no_cosort:
  1946. case no_sort:
  1947. case no_sorted:
  1948. case no_assertsorted:
  1949. normalized = normalizeSort(transformed);
  1950. break;
  1951. case no_subsort:
  1952. normalized = normalizeSubSort(transformed);
  1953. break;
  1954. case no_cogroup:
  1955. normalized = normalizeCoGroup(transformed);
  1956. break;
  1957. case no_choosen:
  1958. normalized = normalizeChooseN(transformed);
  1959. break;
  1960. case no_aggregate:
  1961. normalized = normalizeTableGrouping(transformed);
  1962. break;
  1963. case no_newusertable:
  1964. normalized = normalizeTableGrouping(transformed);
  1965. break;
  1966. case no_newaggregate:
  1967. normalized = normalizeTableGrouping(transformed);
  1968. if (!normalized)
  1969. normalized = normalizeTableToAggregate(transformed, true);
  1970. if (!normalized || (normalized == transformed))
  1971. {
  1972. ::Release(normalized);
  1973. normalized = normalizePrefetchAggregate(transformed);
  1974. }
  1975. break;
  1976. case no_dedup:
  1977. normalized = normalizeDedup(transformed);
  1978. break;
  1979. case no_rollup:
  1980. normalized = normalizeRollup(transformed);
  1981. break;
  1982. case no_select:
  1983. normalized = normalizeSelect(transformed);
  1984. break;
  1985. case no_temptable:
  1986. normalized = normalizeTempTable(transformed);
  1987. break;
  1988. //MORE should do whole aggregate expression e.g., max(x)-min(x)
  1989. case NO_AGGREGATE:
  1990. normalized = normalizeScalarAggregate(transformed);
  1991. break;
  1992. case no_projectrow:
  1993. {
  1994. IHqlExpression * ds = transformed->queryChild(0);
  1995. if (isAlwaysActiveRow(ds))
  1996. {
  1997. //Transform PROJECT(row, transform) to a ROW(transform') since more efficient
  1998. OwnedHqlExpr myLeft = createSelector(no_left, ds, querySelSeq(transformed));
  1999. OwnedHqlExpr replaced = replaceSelector(transformed->queryChild(1), myLeft, ds);
  2000. normalized = createRow(no_createrow, LINK(replaced));
  2001. }
  2002. break;
  2003. }
  2004. case no_debug_option_value:
  2005. //pick best engine etc. definitely done by now, so substitute any options that haven't been processed already
  2006. return getDebugValueExpr(translator.wu(), expr);
  2007. case no_embedbody:
  2008. {
  2009. //Convert all definitions of non functional embeds to implicit functions
  2010. StringBuffer funcname;
  2011. funcname.append("userx").append(++implicitFunctionId);
  2012. HqlExprArray args;
  2013. args.append(*LINK(expr));
  2014. if (expr->hasAttribute(languageAtom))
  2015. {
  2016. args.append(*createAttribute(contextAtom));
  2017. if (expr->isDatarow())
  2018. args.append(*createAttribute(allocatorAtom));
  2019. }
  2020. OwnedHqlExpr body = createWrapper(no_outofline, expr->queryType(), args);
  2021. HqlExprArray newFormals;
  2022. if (expr->hasAttribute(languageAtom))
  2023. {
  2024. HqlExprArray attrs;
  2025. attrs.append(*createAttribute(_hidden_Atom));
  2026. newFormals.append(*createParameter(__optionsId, 0, LINK(unknownVarStringType), attrs));
  2027. }
  2028. IHqlExpression * formals = createValue(no_sortlist, makeSortListType(NULL), newFormals);
  2029. OwnedHqlExpr funcdef = createFunctionDefinition(createIdAtom(funcname), body.getClear(), formals, NULL, NULL);
  2030. HqlExprArray actuals;
  2031. if (expr->hasAttribute(languageAtom))
  2032. actuals.append(*getEmbedOptionString(funcdef));
  2033. return createBoundFunction(NULL, funcdef, actuals, nullptr, true);
  2034. }
  2035. }
  2036. if (normalized && (normalized != transformed))
  2037. {
  2038. transformed.setown(transform(normalized));
  2039. normalized->Release();
  2040. }
  2041. /*
  2042. //Has a minor impact on unnecessary local attributes
  2043. if (!translator.targetThor() && transformed->hasAttribute(localAtom) && localChangesActivityAction(transformed))
  2044. return removeAttribute(transformed, localAtom);
  2045. */
  2046. return transformed.getClear();
  2047. }
  2048. static IHqlExpression * convertDedupToGroupedDedup(IHqlExpression * expr, IHqlExpression * grouping, bool compareAll)
  2049. {
  2050. IHqlExpression * localAttr = expr->queryAttribute(localAtom);
  2051. HqlExprArray groupArgs;
  2052. groupArgs.append(*LINK(expr->queryChild(0)));
  2053. groupArgs.append(*LINK(grouping));
  2054. if (compareAll)
  2055. groupArgs.append(*createAttribute(allAtom));
  2056. if (localAttr)
  2057. groupArgs.append(*LINK(localAttr));
  2058. //Ideally this would remove the equality conditions from the dedup, but ok since they are ignored later when generating
  2059. OwnedHqlExpr group = createDataset(no_group, groupArgs);
  2060. group.setown(cloneInheritedAnnotations(expr, group));
  2061. HqlExprArray dedupArgs;
  2062. dedupArgs.append(*LINK(group));
  2063. unwindChildren(dedupArgs, expr, 1);
  2064. removeAttribute(dedupArgs, localAtom); //(since now a grouped dedup)
  2065. OwnedHqlExpr ungroup = createDataset(no_group, expr->clone(dedupArgs), NULL);
  2066. return cloneInheritedAnnotations(expr, ungroup);
  2067. }
  2068. IHqlExpression * ThorHqlTransformer::normalizeDedup(IHqlExpression * expr)
  2069. {
  2070. if (isGroupedActivity(expr))
  2071. {
  2072. //MORE: It should be possible to remove ,ALL if no conditions and
  2073. //the equalities (ignoring any grouping conditions) match the group sort order
  2074. return NULL;
  2075. }
  2076. // DEDUP, ALL, local: - pre sort the data, group, dedup and ungroup
  2077. // DEDUP, ALL, global - if just had a sort by any of the criteria then do it local
  2078. // DEDUP, not all, not grouped, group by criteria, dedup, degroup
  2079. DedupInfoExtractor info(expr);
  2080. if (info.equalities.ordinality() == 0)
  2081. return NULL;
  2082. IHqlExpression * dataset = expr->queryChild(0);
  2083. bool hasLocal = isLocalActivity(expr);
  2084. bool isLocal = hasLocal || !translator.targetThor();
  2085. bool isHashDedup = expr->hasAttribute(hashAtom);
  2086. if (info.compareAllRows)
  2087. {
  2088. IHqlExpression * manyProp = expr->queryAttribute(manyAtom);
  2089. if (!isLocal && manyProp)
  2090. {
  2091. //If lots of duplicates, then dedup all locally and then dedup all globally.
  2092. HqlExprArray localArgs;
  2093. unwindChildren(localArgs, expr);
  2094. localArgs.zap(*manyProp);
  2095. localArgs.append(*createLocalAttribute());
  2096. OwnedHqlExpr localDedup = expr->clone(localArgs);
  2097. HqlExprArray globalArgs;
  2098. globalArgs.append(*localDedup.getClear());
  2099. unwindChildren(globalArgs, expr, 1);
  2100. globalArgs.zap(*manyProp);
  2101. return expr->clone(globalArgs);
  2102. }
  2103. }
  2104. //If a dedup can be done locally then force it to be local
  2105. if (!isLocal)
  2106. {
  2107. OwnedHqlExpr newSort = createValueSafe(no_sortlist, makeSortListType(NULL), info.equalities);
  2108. if (isPartitionedForGroup(dataset, newSort, info.compareAllRows))
  2109. {
  2110. OwnedHqlExpr ret = appendOwnedOperand(expr, createLocalAttribute());
  2111. //A global all join implies hash (historically) so preserve that semantic
  2112. if (info.compareAllRows && !isHashDedup)
  2113. return appendOwnedOperand(ret, createAttribute(hashAtom));
  2114. return ret.getClear();
  2115. }
  2116. }
  2117. //DEDUP,ALL
  2118. if (info.compareAllRows)
  2119. {
  2120. OwnedHqlExpr groupOrder = createValueSafe(no_sortlist, makeSortListType(NULL), info.equalities);
  2121. bool checkLocal = isLocal || (options.supportsMergeDistribute && !isHashDedup);
  2122. //If the dataset is already sorted for deduping, (and no extra tests) then
  2123. //if local can just remove the ALL attribute, since the records are already adjacent.
  2124. //if global remove the all, but enclose the dedup in a group to avoid serial processing
  2125. //Ignore HASH if specified since this has to be more efficient.
  2126. if (info.conds.ordinality() == 0)
  2127. {
  2128. bool alreadySorted = isSortedForGroup(dataset, groupOrder, checkLocal);
  2129. if (alreadySorted)
  2130. {
  2131. OwnedHqlExpr noHash = removeAttribute(expr, hashAtom);
  2132. OwnedHqlExpr noAll = removeAttribute(noHash, allAtom);
  2133. if (isLocal)
  2134. return noAll.getClear();
  2135. return convertDedupToGroupedDedup(noAll, groupOrder, checkLocal && !isLocal);
  2136. }
  2137. }
  2138. if (!isHashDedup)
  2139. {
  2140. //If has post non equality condition, change it to a group all->dedup->ungroup
  2141. if (info.conds.ordinality())
  2142. return convertDedupToGroupedDedup(expr, groupOrder, true);
  2143. //If local and thor (since hash dedup may overflow) convert to sort, dedup(not all)
  2144. //Otherwise a hashdedup is likely to be more efficient - since it will be linear cf O(NlnN) for the sort
  2145. if (hasLocal && translator.targetThor())
  2146. {
  2147. HqlExprArray dedupArgs;
  2148. dedupArgs.append(*ensureSortedForGroup(dataset, groupOrder, true, false, options.implicitGroupSubSort));
  2149. unwindChildren(dedupArgs, expr, 1);
  2150. removeAttribute(dedupArgs, allAtom);
  2151. return expr->clone(dedupArgs);
  2152. }
  2153. else
  2154. {
  2155. if (matchesConstantValue(info.numToKeep, 1) && !info.keepLeft)
  2156. return appendOwnedOperand(expr, createAttribute(hashAtom));
  2157. }
  2158. }
  2159. }
  2160. else
  2161. {
  2162. //Convert dedup(ds, exprs) to group(dedup(group(ds, exprs), exprs))
  2163. //To ensure that the activity isn't executed serially.
  2164. if (!isLocal && !areConstant(info.equalities))
  2165. {
  2166. OwnedHqlExpr groupOrder = createValueSafe(no_sortlist, makeSortListType(NULL), info.equalities);
  2167. return convertDedupToGroupedDedup(expr, groupOrder, false);
  2168. }
  2169. }
  2170. return NULL;
  2171. }
  2172. IHqlExpression * ThorHqlTransformer::normalizeRollup(IHqlExpression * expr)
  2173. {
  2174. if (isGroupedActivity(expr))
  2175. return NULL;
  2176. IHqlExpression * dataset = expr->queryChild(0);
  2177. IHqlExpression * cond = expr->queryChild(1);
  2178. if (isThorCluster(targetClusterType) && !expr->queryAttribute(localAtom) && isIndependentOfScope(expr))
  2179. {
  2180. HqlExprArray equalities;
  2181. OwnedHqlExpr extra;
  2182. if (cond->getOperator() == no_sortlist)
  2183. cond->unwindList(equalities, no_sortlist);
  2184. else if (!cond->isBoolean())
  2185. equalities.append(*LINK(cond));
  2186. else
  2187. {
  2188. HqlExprArray terms;
  2189. cond->unwindList(terms, no_and);
  2190. OwnedHqlExpr left = createSelector(no_left, dataset, querySelSeq(expr));
  2191. OwnedHqlExpr right = createSelector(no_right, dataset, querySelSeq(expr));
  2192. ForEachItemIn(i, terms)
  2193. {
  2194. IHqlExpression & cur = terms.item(i);
  2195. bool matched = false;
  2196. if (cur.getOperator() == no_eq)
  2197. {
  2198. OwnedHqlExpr mappedLeft = replaceSelector(cur.queryChild(0), left, dataset);
  2199. OwnedHqlExpr mappedRight = replaceSelector(cur.queryChild(1), right, dataset);
  2200. if (mappedLeft == mappedRight)
  2201. {
  2202. equalities.append(*LINK(mappedLeft));
  2203. matched = true;
  2204. }
  2205. }
  2206. if (!matched)
  2207. extendConditionOwn(extra, no_and, LINK(&cur));
  2208. }
  2209. }
  2210. //Don't create a group by constant - it will kill thor!
  2211. ForEachItemInRev(ie, equalities)
  2212. if (equalities.item(ie).isConstant())
  2213. equalities.remove(ie);
  2214. if (equalities.ordinality())
  2215. {
  2216. OwnedHqlExpr left = createSelector(no_left, dataset, querySelSeq(expr));
  2217. //If anything in the join condition references LEFT then the whole condition is currently passed the modified row
  2218. //so remove any fields that are modified in the transform
  2219. HqlExprArray ambiguousSelects;
  2220. if (cond->usesSelector(left))
  2221. filterAmbiguousRollupCondition(ambiguousSelects, equalities, expr);
  2222. if (equalities.ordinality() == 0)
  2223. {
  2224. translator.reportWarning(CategoryEfficiency, SeverityUnknown, queryLocation(expr), ECODETEXT(HQLWRN_AmbiguousRollupNoGroup));
  2225. }
  2226. else
  2227. {
  2228. OwnedHqlExpr groupOrder = createValueSafe(no_sortlist, makeSortListType(NULL), equalities);
  2229. if (isPartitionedForGroup(dataset, groupOrder, false))
  2230. return appendOwnedOperand(expr, createLocalAttribute());
  2231. //This list can only contain items if the filter is using left/right => expand to an equality
  2232. IHqlExpression * selector = dataset->queryNormalizedSelector();
  2233. OwnedHqlExpr right = createSelector(no_right, dataset, querySelSeq(expr));
  2234. ForEachItemIn(i, ambiguousSelects)
  2235. {
  2236. IHqlExpression * select = &ambiguousSelects.item(i);
  2237. OwnedHqlExpr leftSelect = replaceSelector(select, selector, left);
  2238. OwnedHqlExpr rightSelect = replaceSelector(select, selector, right);
  2239. IHqlExpression * eq = createBoolExpr(no_eq, leftSelect.getClear(), rightSelect.getClear());
  2240. extendConditionOwn(extra, no_and, eq);
  2241. }
  2242. HqlExprArray groupArgs, rollupArgs;
  2243. groupArgs.append(*LINK(dataset));
  2244. groupArgs.append(*LINK(groupOrder));
  2245. OwnedHqlExpr group = createDataset(no_group, groupArgs);
  2246. group.setown(cloneInheritedAnnotations(expr, group));
  2247. rollupArgs.append(*LINK(group));
  2248. if (extra)
  2249. rollupArgs.append(*extra.getClear());
  2250. else
  2251. rollupArgs.append(*createConstant(true));
  2252. unwindChildren(rollupArgs, expr, 2);
  2253. OwnedHqlExpr ungroup = createDataset(no_group, expr->clone(rollupArgs), NULL);
  2254. return cloneInheritedAnnotations(expr, ungroup);
  2255. }
  2256. }
  2257. }
  2258. return NULL;
  2259. }
  2260. IHqlExpression * ThorHqlTransformer::skipOverGroups(IHqlExpression * dataset, bool isLocal)
  2261. {
  2262. //if grouping a group, remove the initial group.
  2263. //Not completely sure about this - it may potentially cause extra splitters.
  2264. IHqlExpression * newDataset = dataset;
  2265. while (newDataset->getOperator() == no_group)
  2266. {
  2267. if (newDataset->hasAttribute(allAtom))
  2268. break;
  2269. if (isLocal && queryRealChild(newDataset, 1))
  2270. {
  2271. //NOTE: local groups should not remove preceding non-local groups.
  2272. if (translator.targetThor() && !newDataset->hasAttribute(localAtom))
  2273. break;
  2274. }
  2275. newDataset = newDataset->queryChild(0);
  2276. }
  2277. return newDataset;
  2278. }
  2279. IHqlExpression * ThorHqlTransformer::skipGroupsWithinGroup(IHqlExpression * expr, bool isLocal)
  2280. {
  2281. //if grouping a group, remove the initial group.
  2282. //Not completely sure about this - it may potentially cause extra splitters.
  2283. IHqlExpression * dataset = expr->queryChild(0);
  2284. if (dataset->getOperator() == no_group)
  2285. {
  2286. IHqlExpression * newDataset = skipOverGroups(dataset, isLocal);
  2287. if (newDataset == dataset)
  2288. return NULL;
  2289. //if we end up with the original grouping then probably have ungroup(group(x,y))
  2290. //so no need to do this group either
  2291. if (queryGrouping(newDataset) == queryGrouping(expr))
  2292. return LINK(newDataset);
  2293. return replaceChild(expr, 0, newDataset);
  2294. }
  2295. return NULL;
  2296. }
  2297. IHqlExpression * ThorHqlTransformer::normalizeGroup(IHqlExpression * expr)
  2298. {
  2299. assertex(expr->getOperator() == no_group);
  2300. IHqlExpression * sortlist = queryRealChild(expr, 1);
  2301. IHqlExpression * dataset = expr->queryChild(0);
  2302. if (!sortlist)
  2303. return skipGroupsWithinGroup(expr, false);
  2304. OwnedHqlExpr newsort = simplifySortlistComplexity(sortlist);
  2305. if (newsort)
  2306. return replaceChild(expr, 1, newsort);
  2307. bool hasLocal = expr->hasAttribute(localAtom);
  2308. bool isLocal = hasLocal || !translator.targetThor();
  2309. bool wantSorted = expr->hasAttribute(sortedAtom);
  2310. bool hasAll = expr->hasAttribute(allAtom);
  2311. // First check if a global group can be done locally - applicable to all and non-all versions.
  2312. if (!isLocal)
  2313. {
  2314. if (!wantSorted && isPartitionedForGroup(dataset, sortlist, hasAll))
  2315. return appendLocalAttribute(expr);
  2316. }
  2317. if (!hasAll)
  2318. {
  2319. //if grouping a group, remove the initial group.
  2320. //Not completely sure about this - it may potentially cause extra splitters.
  2321. return skipGroupsWithinGroup(expr, isLocal);
  2322. }
  2323. //First check to see if the dataset is already sorted by the group criteria, or more.
  2324. //The the data could be globally sorted, but not distributed, and this is likely to be more efficient than redistributing...
  2325. OwnedHqlExpr sorted = ensureSortedForGroup(dataset, sortlist, hasLocal, !translator.targetThor(), options.implicitGroupSubSort);
  2326. if (sorted == dataset)
  2327. return removeAttribute(expr, allAtom);
  2328. sorted.setown(cloneInheritedAnnotations(expr, sorted));
  2329. sorted.setown(inheritAttribute(sorted, expr, skewAtom));
  2330. sorted.setown(inheritAttribute(sorted, expr, thresholdAtom));
  2331. if (!isLocal)
  2332. {
  2333. //Options for ensuring distributed and locally sorted (in order)
  2334. // DISTRIBUTE,MERGE - since lightweight and streaming.
  2335. // DISTRIBUTE,LOCAL SORT
  2336. // SORT
  2337. if (!wantSorted)
  2338. {
  2339. //is it best to hash on all the grouping fields, or just some of them? Do all for the moment.
  2340. OwnedHqlExpr hashed = createValue(no_hash32, LINK(unsignedType), LINK(sortlist), createAttribute(internalAtom));
  2341. if (options.supportsMergeDistribute && isSortedForGroup(dataset, sortlist, true))
  2342. {
  2343. //Dataset is locally sorted, so can use the merge distribute to remove the subsequent local sort.
  2344. //changing a heavyweight global sort into a lightweight distribute,merge
  2345. OwnedHqlExpr sortOrder = getExistingSortOrder(dataset, true, true);
  2346. OwnedHqlExpr mergeAttr = createExprAttribute(mergeAtom, replaceSelector(sortOrder, queryActiveTableSelector(), dataset));
  2347. sorted.setown(createDatasetF(no_distribute, LINK(dataset), LINK(hashed), mergeAttr.getClear(), NULL));
  2348. sorted.setown(cloneInheritedAnnotations(expr, sorted));
  2349. }
  2350. else
  2351. {
  2352. if (groupAllDistribute || expr->hasAttribute(unsortedAtom))
  2353. {
  2354. OwnedHqlExpr distributed = createDataset(no_distribute, LINK(dataset), LINK(hashed));
  2355. distributed.setown(cloneInheritedAnnotations(expr, distributed));
  2356. sorted.setown(createDataset(no_sort, LINK(distributed), createComma(LINK(sortlist), createLocalAttribute())));
  2357. sorted.setown(cloneInheritedAnnotations(expr, sorted));
  2358. }
  2359. }
  2360. }
  2361. #ifdef _DEBUG
  2362. assertex(!sortlist->isPure() || isPartitionedForGroup(sorted, sortlist, true)); // sanity check
  2363. #endif
  2364. }
  2365. //Do a local group after the sort because we know they can't overlap...
  2366. OwnedHqlExpr ret = createDatasetF(no_group, sorted.getClear(), LINK(sortlist), createLocalAttribute(), NULL);
  2367. return expr->cloneAllAnnotations(ret);
  2368. }
  2369. IHqlExpression * ThorHqlTransformer::normalizeCoGroup(IHqlExpression * expr)
  2370. {
  2371. IHqlExpression * grouping = queryAttributeChild(expr, groupAtom, 0);
  2372. OwnedHqlExpr newsort = simplifySortlistComplexity(grouping);
  2373. if (newsort)
  2374. {
  2375. OwnedHqlExpr newGroup = createExprAttribute(groupAtom, newsort.getClear());
  2376. return replaceOwnedAttribute(expr, newGroup.getClear());
  2377. }
  2378. HqlExprArray inputs;
  2379. //Gather the inputs and ensure they aren't grouped.
  2380. ForEachChild(i, expr)
  2381. {
  2382. IHqlExpression * cur = expr->queryChild(i);
  2383. if (!cur->isAttribute())
  2384. {
  2385. if (isGrouped(cur))
  2386. {
  2387. OwnedHqlExpr ungroup = createDataset(no_group, LINK(cur));
  2388. inputs.append(*cloneInheritedAnnotations(expr, ungroup));
  2389. }
  2390. else
  2391. inputs.append(*LINK(cur));
  2392. }
  2393. }
  2394. bool hasLocal = expr->hasAttribute(localAtom);
  2395. bool alwaysLocal = !translator.targetThor();
  2396. bool isLocal = hasLocal || alwaysLocal;
  2397. OwnedHqlExpr localFlag = !alwaysLocal ? createLocalAttribute() : NULL;
  2398. OwnedHqlExpr bestSortOrder;
  2399. //Choose the best existing sort order (for the moment assume the shortest - although
  2400. //even better would be to pick the shortest most frequent
  2401. ForEachItemIn(iBest, inputs)
  2402. {
  2403. if (isSortedForGroup(&inputs.item(iBest), grouping, true))
  2404. {
  2405. OwnedHqlExpr localOrder = getExistingSortOrder(&inputs.item(iBest), true, true);
  2406. if (!bestSortOrder || (localOrder->numChildren() < bestSortOrder->numChildren()))
  2407. bestSortOrder.set(localOrder);
  2408. }
  2409. }
  2410. if (!isLocal)
  2411. {
  2412. //Ensure all the inputs are co-distributed (use an existing distribution if possible)
  2413. //Even better would be to pick the most frequent
  2414. OwnedHqlExpr distribution;
  2415. ForEachItemIn(i, inputs)
  2416. {
  2417. IHqlExpression & cur = inputs.item(i);
  2418. if (isPartitionedForGroup(&cur, grouping, true))
  2419. {
  2420. IHqlExpression * curDistribution = queryDistribution(&cur);
  2421. if (!isSortDistribution(curDistribution))
  2422. {
  2423. distribution.set(curDistribution);
  2424. break;
  2425. }
  2426. }
  2427. }
  2428. if (!distribution)
  2429. distribution.setown(createValue(no_hash32, LINK(unsignedType), LINK(grouping), createAttribute(internalAtom)));
  2430. ForEachItemIn(iReplace, inputs)
  2431. {
  2432. IHqlExpression & cur = inputs.item(iReplace);
  2433. if (queryDistribution(&cur) != distribution)
  2434. {
  2435. OwnedHqlExpr mappedDistribution = replaceSelector(distribution, queryActiveTableSelector(), &cur);
  2436. OwnedHqlExpr mergeAttr;
  2437. if (bestSortOrder && isAlreadySorted(&cur, bestSortOrder, true, true, false))
  2438. mergeAttr.setown(createExprAttribute(mergeAtom, replaceSelector(bestSortOrder, queryActiveTableSelector(), &cur)));
  2439. OwnedHqlExpr distributedInput = createDatasetF(no_distribute, LINK(&cur), LINK(mappedDistribution), mergeAttr.getClear(), NULL);
  2440. distributedInput.setown(cloneInheritedAnnotations(expr, distributedInput));
  2441. inputs.replace(*distributedInput.getClear(), iReplace);
  2442. }
  2443. }
  2444. }
  2445. OwnedHqlExpr merged;
  2446. if (bestSortOrder)
  2447. {
  2448. //If some of the datasets are sorted then sort the remaining inputs by the same order and merge
  2449. HqlExprArray sortedInputs;
  2450. ForEachItemIn(i, inputs)
  2451. {
  2452. IHqlExpression & cur = inputs.item(i);
  2453. OwnedHqlExpr mappedOrder = replaceSelector(bestSortOrder, queryActiveTableSelector(), &cur);
  2454. sortedInputs.append(*ensureSorted(&cur, mappedOrder, expr, true, true, alwaysLocal, options.implicitSubSort, false));
  2455. }
  2456. HqlExprArray sortedArgs;
  2457. unwindChildren(sortedArgs, bestSortOrder);
  2458. sortedInputs.append(*createExprAttribute(sortedAtom, sortedArgs));
  2459. if (localFlag)
  2460. sortedInputs.append(*LINK(localFlag));
  2461. merged.setown(createDataset(no_merge, sortedInputs));
  2462. }
  2463. else
  2464. {
  2465. inputs.append(*getOrderedAttribute(false));
  2466. //otherwise append the datasets and then sort them all
  2467. OwnedHqlExpr appended = createDataset(no_addfiles, inputs);
  2468. appended.setown(cloneInheritedAnnotations(expr, appended));
  2469. OwnedHqlExpr mappedOrder = replaceSelector(grouping, queryActiveTableSelector(), appended);
  2470. merged.setown(createDatasetF(no_sort, LINK(appended), mappedOrder.getClear(), LINK(localFlag), NULL));
  2471. }
  2472. //Now group by the grouping condition
  2473. merged.setown(cloneInheritedAnnotations(expr, merged));
  2474. OwnedHqlExpr mappedGrouping = replaceSelector(grouping, queryActiveTableSelector(), merged);
  2475. OwnedHqlExpr grouped = createDataset(no_group, LINK(merged), mappedGrouping.getClear());
  2476. return expr->cloneAllAnnotations(grouped);
  2477. }
  2478. static bool canReorderMatchExistingLocalSort(HqlExprArray & newElements1, HqlExprArray & newElements2, IHqlExpression * ds1, Shared<IHqlExpression> & ds2, const HqlExprArray & elements1, const HqlExprArray & elements2, bool canSubSort, bool isLocal, bool alwaysLocal)
  2479. {
  2480. newElements1.kill();
  2481. newElements2.kill();
  2482. if (reorderMatchExistingLocalSort(newElements1, newElements2, ds1, elements1, elements2))
  2483. {
  2484. if (isAlreadySorted(ds2, newElements2, isLocal||alwaysLocal, true, true))
  2485. return true;
  2486. if (canSubSort && isWorthShuffling(ds2, newElements2, isLocal||alwaysLocal, true))
  2487. {
  2488. OwnedHqlExpr subsorted = getSubSort(ds2, newElements2, isLocal, true, alwaysLocal);
  2489. if (subsorted)
  2490. {
  2491. ds2.swap(subsorted);
  2492. return true;
  2493. }
  2494. }
  2495. }
  2496. return false;
  2497. }
  2498. bool ThorHqlTransformer::isLightweightJoinCandidate(IHqlExpression * expr, bool isLocal, bool isLimitedSubstringJoin)
  2499. {
  2500. //This is equally applicable to hthor and roxie. However non lookup joins currently generate group activities on
  2501. //the inputs which look less efficient. It may still be better to enable it though.
  2502. if (!translator.targetThor())
  2503. return false;
  2504. if (!options.spotLocalMerge || isLimitedSubstringJoin || !isLocal)
  2505. return false;
  2506. if (expr->hasAttribute(_lightweight_Atom))
  2507. return false;
  2508. switch (expr->getOperator())
  2509. {
  2510. case no_join:
  2511. case no_selfjoin:
  2512. case no_denormalizegroup:
  2513. case no_denormalize:
  2514. return true;
  2515. }
  2516. return false;
  2517. }
  2518. static IHqlExpression * createDistributedInput(IHqlExpression * ds, IHqlExpression * sortlist, bool internal)
  2519. {
  2520. //could use a more optimal hash function since comparing against self, so fields are same type
  2521. IHqlExpression * internalExpr = internal ? createAttribute(internalAtom) : NULL;
  2522. OwnedHqlExpr activeDist = createValue(no_hash32, LINK(unsignedType), LINK(sortlist), internalExpr);
  2523. OwnedHqlExpr dist = replaceSelector(activeDist, queryActiveTableSelector(), ds);
  2524. return createDataset(no_distribute, LINK(ds), LINK(dist));
  2525. }
  2526. static IHqlExpression * createDistributedInput(IHqlExpression * ds, const HqlExprArray & sorts, bool internal)
  2527. {
  2528. OwnedHqlExpr sortlist = createValueSafe(no_sortlist, makeSortListType(NULL), sorts);
  2529. return createDistributedInput(ds, sortlist, internal);
  2530. }
  2531. /*
  2532. Perform the following transformation:
  2533. R := JOIN(l, r, LEFT.key = RIGHT.key AND fuzzy(LEFT,RIGHT), t(LEFT,RIGHT), GROUP(LEFT.id1, LEFT.id2), ATMOST(optional))
  2534. DL := DISTRIBUTE(L, HASH(key));
  2535. DR := DISTRIBUTE(R, HASH(key));
  2536. SL := SORT(DL, id, LOCAL); // Later replace this with a LEFTSORT() attribute on the join (so can optimize self join)
  2537. //If it is a self join, SR == SL
  2538. JR := JOIN(SL, DR, LEFT.key = RIGHT.key, t(LEFT,RIGHT), LOOKUP MANY, LOCAL);
  2539. DJ := DISTRIBUTE(J, HASH(leftid1, leftid2), MERGE(leftid1, leftid2));
  2540. R := GROUP(DJ, leftid1, leftid2, LOCAL);
  2541. */
  2542. IHqlExpression * ThorHqlTransformer::normalizeJoinAndGroup(IHqlExpression * expr)
  2543. {
  2544. IHqlExpression * oldLeft = expr->queryChild(0);
  2545. IHqlExpression * oldRight = expr->queryChild(1);
  2546. LinkedHqlExpr newLeft = oldLeft;
  2547. LinkedHqlExpr newRight = oldRight;
  2548. IHqlExpression * groupOrder = queryAttributeChild(expr, groupAtom, 0);
  2549. node_operator op = expr->getOperator();
  2550. bool hasLocal = isLocalActivity(expr);
  2551. bool alwaysLocal = !translator.targetThor();
  2552. if (!hasLocal && !alwaysLocal)
  2553. {
  2554. JoinSortInfo joinInfo(expr);
  2555. joinInfo.findJoinSortOrders(false);
  2556. OwnedHqlExpr leftList = createValueSafe(no_sortlist, makeSortListType(NULL), joinInfo.queryLeftReq());
  2557. OwnedHqlExpr mappedLeftList = replaceSelector(leftList, queryActiveTableSelector(), newLeft->queryNormalizedSelector());
  2558. OwnedHqlExpr hashLeft = createValue(no_hash32, makeIntType(4, false), mappedLeftList.getClear());
  2559. newLeft.setown(createDataset(no_distribute, LINK(newLeft), LINK(hashLeft)));
  2560. if (oldRight == oldLeft)
  2561. newRight.set(newLeft);
  2562. else if (op != no_selfjoin)
  2563. {
  2564. OwnedHqlExpr rightList = createValueSafe(no_sortlist, makeSortListType(NULL), joinInfo.queryRightReq());
  2565. OwnedHqlExpr mappedRightList = replaceSelector(rightList, queryActiveTableSelector(), newRight->queryNormalizedSelector());
  2566. OwnedHqlExpr hashRight = createValue(no_hash32, makeIntType(4, false), mappedRightList.getClear());
  2567. newRight.setown(createDataset(no_distribute, LINK(newRight), LINK(hashRight)));
  2568. }
  2569. }
  2570. OwnedHqlExpr newLocalAttr = alwaysLocal ? NULL : createLocalAttribute();
  2571. //Sort the left hand dataset into grouping order.
  2572. assertex(groupOrder);
  2573. OwnedHqlExpr left = createSelector(no_left, expr->queryChild(0), querySelSeq(expr));
  2574. OwnedHqlExpr leftSortOrder = replaceSelector(groupOrder, left, newLeft);
  2575. newLeft.setown(createDatasetF(no_sort, newLeft.getClear(), LINK(leftSortOrder), LINK(newLocalAttr), NULL));
  2576. if (oldRight == oldLeft)
  2577. newRight.set(newLeft);
  2578. //Now create the modified join
  2579. HqlExprArray joinArgs;
  2580. joinArgs.append(*LINK(newLeft));
  2581. joinArgs.append(*LINK(newRight));
  2582. unwindChildren(joinArgs, expr, 2);
  2583. removeAttribute(joinArgs, groupAtom);
  2584. if (!hasLocal && !alwaysLocal)
  2585. joinArgs.append(*createLocalAttribute());
  2586. OwnedHqlExpr newJoin = expr->clone(joinArgs);
  2587. //Now need to map the fields from the input dataset to the join output
  2588. NewProjectMapper2 mapper;
  2589. mapper.setMapping(newJoin->queryChild(3));
  2590. bool matchedAll = true;
  2591. OwnedHqlExpr mappedOrder = mapper.collapseFields(groupOrder, left, newJoin->queryNormalizedSelector(), left, &matchedAll);
  2592. assertex(matchedAll); // This is checked in the parser, so shouldn't be triggered here.
  2593. //Distribute the result
  2594. LinkedHqlExpr distributed = newJoin;
  2595. if (!hasLocal && !alwaysLocal)
  2596. {
  2597. OwnedHqlExpr hashOut = createValue(no_hash32, makeIntType(4, false), LINK(mappedOrder));
  2598. OwnedHqlExpr mergeOut = createExprAttribute(mergeAtom, LINK(mappedOrder));
  2599. distributed.setown(createDatasetF(no_distribute, LINK(newJoin), hashOut.getClear(), mergeOut.getClear(), NULL));
  2600. }
  2601. //And finally group it.
  2602. return createDatasetF(no_group, LINK(distributed), LINK(mappedOrder), LINK(newLocalAttr), NULL);
  2603. }
  2604. static IHqlExpression * queryDistributionKey(IHqlExpression * rhs)
  2605. {
  2606. for (;;)
  2607. {
  2608. switch (rhs->getOperator())
  2609. {
  2610. case no_filter:
  2611. case no_metaactivity:
  2612. case no_sorted:
  2613. rhs = rhs->queryChild(0);
  2614. break;
  2615. case no_newkeyindex:
  2616. return rhs;
  2617. default:
  2618. return NULL;
  2619. }
  2620. }
  2621. }
  2622. IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression * expr)
  2623. {
  2624. IHqlExpression * leftDs = expr->queryChild(0);
  2625. IHqlExpression * rightDs = queryJoinRhs(expr);
  2626. IHqlExpression * seq = querySelSeq(expr);
  2627. node_operator op = expr->getOperator();
  2628. if (op == no_join)
  2629. {
  2630. if (isSelfJoin(expr))
  2631. {
  2632. HqlExprArray children;
  2633. unwindChildren(children, expr);
  2634. children.replace(*createAttribute(_selfJoinPlaceholder_Atom), 1); // replace the 1st dataset with an attribute so parameters are still in the same place.
  2635. OwnedHqlExpr ret = createDataset(no_selfjoin, children);
  2636. return expr->cloneAllAnnotations(ret);
  2637. }
  2638. }
  2639. bool hasLocal = isLocalActivity(expr);
  2640. bool alwaysLocal = !translator.targetThor();
  2641. if (alwaysLocal)
  2642. {
  2643. if (expr->hasAttribute(hashAtom))
  2644. return removeAttribute(expr, hashAtom);
  2645. //Hthor and roxie don't currently support smart joins since the normal join requires the input to be sorted
  2646. if (expr->hasAttribute(smartAtom))
  2647. return removeAttribute(expr, smartAtom);
  2648. }
  2649. bool isLocal = hasLocal || alwaysLocal;
  2650. //hash,local doesn't make sense (hash is only used for distribution) => remove hash
  2651. //but also prevent it being converted to a lookup join??
  2652. if (isLocal && expr->hasAttribute(hashAtom))
  2653. return removeAttribute(expr, hashAtom);
  2654. //Check to see if this join should be done as a keyed join...
  2655. if (!expr->hasAttribute(lookupAtom) && !expr->hasAttribute(smartAtom) &&
  2656. !expr->hasAttribute(allAtom) && !expr->hasAttribute(streamedAtom))
  2657. {
  2658. if (rightDs->getOperator() == no_filter)
  2659. {
  2660. bool moveRhsFilter = false;
  2661. if (expr->hasAttribute(keyedAtom) && queryAttributeChild(expr, keyedAtom, 0))
  2662. {
  2663. //Full keyed join - ensure the filter is moved from the rhs to the condition.
  2664. moveRhsFilter = true;
  2665. }
  2666. else if (options.spotPotentialKeyedJoins && (rightDs != leftDs))
  2667. {
  2668. //This can turn some non keyed joins into keyed joins
  2669. IHqlExpression * cur = rightDs;
  2670. while (cur->getOperator() == no_filter)
  2671. cur = cur->queryChild(0);
  2672. if (cur->getOperator() == no_newkeyindex)
  2673. moveRhsFilter = true;
  2674. }
  2675. if (moveRhsFilter)
  2676. {
  2677. //Transform join(a, b(x), c) to join(a, b, c and evaluate(right, x))
  2678. HqlExprAttr extraFilter;
  2679. OwnedHqlExpr right = createSelector(no_right, rightDs, seq);
  2680. IHqlExpression * cur = rightDs;
  2681. while (cur->getOperator() == no_filter)
  2682. {
  2683. unsigned max = cur->numChildren();
  2684. for (unsigned i = 1; i < max; i++)
  2685. {
  2686. IHqlExpression * filter = cur->queryChild(i);
  2687. if (!filter->isAttribute())
  2688. {
  2689. IHqlExpression * newFilter = replaceSelector(filter, rightDs, right);
  2690. extendConditionOwn(extraFilter, no_and, newFilter);
  2691. }
  2692. }
  2693. cur = cur->queryChild(0);
  2694. }
  2695. HqlExprArray args;
  2696. unwindChildren(args, expr);
  2697. args.replace(*LINK(cur), 1);
  2698. args.replace(*createValue(no_and, makeBoolType(), LINK(expr->queryChild(2)), extraFilter.getClear()), 2);
  2699. return expr->clone(args);
  2700. }
  2701. }
  2702. }
  2703. //Tag a keyed join as ordered in the platforms that ensure it does remain ordered. Extend if the others do.
  2704. if (isKeyedJoin(expr))
  2705. {
  2706. if ((translator.targetRoxie() || options.keyedJoinPreservesOrder) && !hasOrderedAttribute(expr))
  2707. return appendOwnedOperand(expr, createAttribute(_ordered_Atom));
  2708. return NULL;
  2709. }
  2710. JoinSortInfo joinInfo(expr);
  2711. joinInfo.findJoinSortOrders(canBeSlidingJoin(expr));
  2712. //If the data is already distributed so the data is on the correct machines then perform the join locally.
  2713. //Should be equally applicable to lookup, hash, all and normal joins.
  2714. if (!isLocal && !joinInfo.hasOptionalEqualities() && joinInfo.queryLeftReq().ordinality())
  2715. {
  2716. if (isDistributedCoLocally(leftDs, rightDs, joinInfo.queryLeftReq(), joinInfo.queryRightReq()))
  2717. return appendOwnedOperand(expr, createLocalAttribute());
  2718. if (options.createImplicitKeyedDistributeForJoin && !expr->hasAttribute(lookupAtom))
  2719. {
  2720. IHqlExpression * rhsKey = queryDistributionKey(rightDs);
  2721. if (rhsKey)
  2722. {
  2723. IHqlExpression * indexRecord = rightDs->queryRecord();
  2724. unsigned numUnsortedFields = numPayloadFields(rightDs);
  2725. unsigned numKeyedFields = getFlatFieldCount(indexRecord)-numUnsortedFields;
  2726. //To create a keyed distribute, all the keyed fields in the rhs key must have equalities
  2727. //in the join condition. Need to extract a join condition which just contains that subset.
  2728. if (joinInfo.numRequiredEqualities() >= numKeyedFields)
  2729. {
  2730. OwnedHqlExpr keyedCondition = joinInfo.getContiguousJoinCondition(numKeyedFields);
  2731. if (keyedCondition)
  2732. {
  2733. OwnedHqlExpr distribute = createDataset(no_keyeddistribute, LINK(leftDs), createComma(LINK(rhsKey), keyedCondition.getClear(), LINK(seq)));
  2734. HqlExprArray args;
  2735. args.append(*distribute.getClear());
  2736. unwindChildren(args, expr, 1);
  2737. args.append(*createLocalAttribute());
  2738. return expr->clone(args);
  2739. }
  2740. }
  2741. }
  2742. }
  2743. if (options.matchExistingDistributionForJoin)
  2744. {
  2745. //Should this exclude lookup joins??
  2746. //On balance it is probably worthwhile since it means that only 1/clustersize data is on each node.
  2747. //If left side (assumed to be the largest) is already distributed, it would be more efficient
  2748. //to redistribute the rhs by a matching hash function (or use cosort), and then join locally.
  2749. //Be careful about the persist scaling factors though.
  2750. //SORT partitions should be supported once they are persisted by the system
  2751. IHqlExpression * leftDistribution = queryDistribution(leftDs);
  2752. if (matchesAnyDistribution(leftDistribution))
  2753. return appendOwnedOperand(expr, createLocalAttribute());
  2754. if (!isPersistDistribution(leftDistribution) && !isSortedDistribution(leftDistribution) && isPartitionedForGroup(leftDs, joinInfo.queryLeftReq(), true))
  2755. {
  2756. //MORE: May need a flag to stop this - to prevent issues with skew.
  2757. OwnedHqlExpr newHash = createMatchingDistribution(leftDistribution, joinInfo.queryLeftReq(), joinInfo.queryRightReq());
  2758. if (newHash)
  2759. {
  2760. OwnedHqlExpr dist = replaceSelector(newHash, queryActiveTableSelector(), rightDs);
  2761. OwnedHqlExpr newRhs = createDataset(no_distribute, LINK(rightDs), LINK(dist));
  2762. OwnedHqlExpr newJoin = replaceChild(expr, 1, newRhs);
  2763. return appendOwnedOperand(newJoin, createLocalAttribute());
  2764. }
  2765. }
  2766. }
  2767. else
  2768. {
  2769. IHqlExpression * leftDistribution = queryDistribution(leftDs);
  2770. if (!isPersistDistribution(leftDistribution) && !isSortedDistribution(leftDistribution) && isPartitionedForGroup(leftDs, joinInfo.queryLeftReq(), true))
  2771. DBGLOG("MORE: Potential for distributed join optimization");
  2772. }
  2773. }
  2774. if (joinInfo.hasOptionalEqualities() && !isLocal && !expr->hasAttribute(hashAtom) && !expr->hasAttribute(allAtom))
  2775. {
  2776. if (joinInfo.hasRequiredEqualities())
  2777. return appendAttribute(expr, hashAtom);
  2778. throwError(HQLERR_PrefixJoinRequiresEquality);
  2779. }
  2780. if (expr->hasAttribute(allAtom))
  2781. return NULL;
  2782. if (expr->hasAttribute(lookupAtom))
  2783. return NULL;
  2784. //Try and convert local joins to a lightweight join that doesn't require any sorting of the inputs.
  2785. //Improves resourcing for thor, and prevents lookup conversion for hthor/roxie
  2786. //Worthwhile even for lookup joins
  2787. if (isLightweightJoinCandidate(expr, isLocal, joinInfo.hasOptionalEqualities()))
  2788. {
  2789. if (isAlreadySorted(leftDs, joinInfo.queryLeftSort(), true, true, false) &&
  2790. isAlreadySorted(rightDs, joinInfo.queryRightSort(), true, true, false))
  2791. {
  2792. //If this is a lookup join without a many then we need to make sure only the first match is retained.
  2793. return appendOwnedOperand(expr, createAttribute(_lightweight_Atom));
  2794. }
  2795. //Check for a local join where we can reorder the condition so both sides match the existing sort orders.
  2796. //could special case self-join to do less work, but probably not worth the effort.
  2797. HqlExprArray sortedLeft, sortedRight;
  2798. if (!joinInfo.hasOptionalEqualities())
  2799. {
  2800. //Since the distribution and order of global joins is not defined this could probably be used for non-local as well.
  2801. LinkedHqlExpr newLeftDs = leftDs;
  2802. LinkedHqlExpr newRightDs = rightDs;
  2803. bool canSubSort = options.subsortLocalJoinConditions;
  2804. bool reordered = canReorderMatchExistingLocalSort(sortedLeft, sortedRight, newLeftDs, newRightDs,
  2805. joinInfo.queryLeftSort(), joinInfo.queryRightSort(), canSubSort, isLocal, alwaysLocal);
  2806. //If allowed to subsort then try the otherway around
  2807. if (!reordered && canSubSort)
  2808. reordered = canReorderMatchExistingLocalSort(sortedRight, sortedLeft, newRightDs, newLeftDs,
  2809. joinInfo.queryRightSort(), joinInfo.queryLeftSort(), canSubSort, isLocal, alwaysLocal);
  2810. if (reordered)
  2811. {
  2812. //Recreate the join condition in the correct order to match the existing sorts...
  2813. HqlExprAttr newcond;
  2814. OwnedHqlExpr leftSelector = createSelector(no_left, newLeftDs, seq);
  2815. OwnedHqlExpr rightSelector = createSelector(no_right, newRightDs, seq);
  2816. ForEachItemIn(i, sortedLeft)
  2817. {
  2818. OwnedHqlExpr lc = replaceSelector(&sortedLeft.item(i), queryActiveTableSelector(), leftSelector);
  2819. OwnedHqlExpr rc = replaceSelector(&sortedRight.item(i), queryActiveTableSelector(), rightSelector);
  2820. extendConditionOwn(newcond, no_and, createValue(no_eq, makeBoolType(), lc.getClear(), rc.getClear()));
  2821. }
  2822. extendConditionOwn(newcond, no_and, LINK(joinInfo.extraMatch));
  2823. HqlExprArray args;
  2824. args.append(*newLeftDs.getClear());
  2825. args.append(*newRightDs.getClear());
  2826. args.append(*newcond.getClear());
  2827. unwindChildren(args, expr, 3);
  2828. args.append(*createAttribute(_lightweight_Atom));
  2829. return expr->clone(args);
  2830. }
  2831. }
  2832. }
  2833. if (!isThorCluster(targetClusterType) && !expr->hasAttribute(_normalized_Atom))
  2834. {
  2835. if (!expr->hasAttribute(streamedAtom) && !expr->hasAttribute(smartAtom))
  2836. {
  2837. //Sort,Sort->join is O(NlnN) lookup join using a hash table is O(N) =>convert for hthor/roxie
  2838. bool createLookup = false;
  2839. if ((op == no_join) && options.convertJoinToLookup)
  2840. {
  2841. if ((targetClusterType == RoxieCluster) || hasFewRows(rightDs))
  2842. if (!isFullJoin(expr) && !isRightJoin(expr) && !expr->hasAttribute(partitionRightAtom))
  2843. createLookup = !expr->hasAttribute(_lightweight_Atom);
  2844. }
  2845. if (joinInfo.hasOptionalEqualities())
  2846. createLookup = false; //doesn't support it yet
  2847. else if (createLookup && joinInfo.queryLeftSort().ordinality())
  2848. {
  2849. //Check this isn't going to generate a between join - if it is that takes precedence.
  2850. if ((joinInfo.slidingMatches.ordinality() != 0) && (joinInfo.queryLeftSort().ordinality() == joinInfo.slidingMatches.ordinality()))
  2851. createLookup = false;
  2852. }
  2853. if (createLookup)
  2854. {
  2855. IHqlExpression * lhs = expr->queryChild(0);
  2856. HqlExprArray args;
  2857. if (isGrouped(lhs))
  2858. {
  2859. OwnedHqlExpr ungroup = createDataset(no_group, LINK(lhs));
  2860. args.append(*cloneInheritedAnnotations(expr, ungroup));
  2861. }
  2862. else
  2863. args.append(*LINK(lhs));
  2864. unwindChildren(args, expr, 1);
  2865. args.append(*createAttribute(manyAtom));
  2866. args.append(*createAttribute(lookupAtom));
  2867. return expr->clone(args);
  2868. }
  2869. }
  2870. }
  2871. //Convert hash selfjoin to self-join(distribute)
  2872. if ((op == no_selfjoin) && expr->hasAttribute(hashAtom))
  2873. {
  2874. assertex(!isLocal);
  2875. if (joinInfo.hasRequiredEqualities())
  2876. {
  2877. OwnedHqlExpr sortlist = createValueSafe(no_sortlist, makeSortListType(NULL), joinInfo.queryLeftReq());
  2878. OwnedHqlExpr distribute;
  2879. //Only likely to catch this partition test if isLimitedSubstringJoin true, otherwise caught above
  2880. if (!isPartitionedForGroup(leftDs, sortlist, true))
  2881. {
  2882. distribute.setown(createDistributedInput(leftDs, sortlist, true));
  2883. distribute.setown(cloneInheritedAnnotations(expr, distribute));
  2884. }
  2885. else
  2886. distribute.set(leftDs);
  2887. HqlExprArray args;
  2888. args.append(*LINK(distribute));
  2889. unwindChildren(args, expr, 1);
  2890. removeAttribute(args, hashAtom);
  2891. args.append(*createLocalAttribute());
  2892. return expr->clone(args);
  2893. }
  2894. }
  2895. if (options.expandHashJoin && isThorCluster(targetClusterType) && expr->hasAttribute(hashAtom) && !joinInfo.hasOptionalEqualities())
  2896. {
  2897. HqlExprArray args;
  2898. args.append(*createDistributedInput(leftDs, joinInfo.queryLeftReq(), false));
  2899. args.append(*createDistributedInput(rightDs, joinInfo.queryRightReq(), false));
  2900. unwindChildren(args, expr, 2);
  2901. removeAttribute(args, hashAtom);
  2902. args.append(*createLocalAttribute());
  2903. return expr->clone(args);
  2904. }
  2905. if (isThorCluster(targetClusterType) && isLocal && options.implicitJoinSubSort && !expr->hasAttribute(smartAtom))
  2906. {
  2907. IHqlExpression * noSortAttr = expr->queryAttribute(noSortAtom);
  2908. OwnedHqlExpr newLeft;
  2909. OwnedHqlExpr newRight;
  2910. if (!userPreventsSort(noSortAttr, no_left))
  2911. newLeft.setown(getSubSort(leftDs, joinInfo.queryLeftSort(), isLocal, true, alwaysLocal));
  2912. if (!userPreventsSort(noSortAttr, no_right))
  2913. newRight.setown(getSubSort(rightDs, joinInfo.queryRightSort(), isLocal, true, alwaysLocal));
  2914. if (newLeft || newRight)
  2915. {
  2916. HqlExprArray args;
  2917. if (newLeft)
  2918. args.append(*newLeft.getClear());
  2919. else
  2920. args.append(*LINK(leftDs));
  2921. if (newRight)
  2922. args.append(*newRight.getClear());
  2923. else
  2924. args.append(*LINK(rightDs));
  2925. unwindChildren(args, expr, 2);
  2926. return expr->clone(args);
  2927. }
  2928. }
  2929. return NULL;
  2930. }
  2931. IHqlExpression * ThorHqlTransformer::normalizeScalarAggregate(IHqlExpression * expr)
  2932. {
  2933. OwnedHqlExpr project = convertScalarAggregateToDataset(expr);
  2934. if (!project)
  2935. throwUnexpected();
  2936. IHqlExpression * field = project->queryRecord()->queryChild(0);
  2937. OwnedHqlExpr ret = createNewSelectExpr(project.getClear(), LINK(field));
  2938. return expr->cloneAllAnnotations(ret);
  2939. }
  2940. IHqlExpression * ThorHqlTransformer::normalizeSelect(IHqlExpression * expr)
  2941. {
  2942. return NULL;
  2943. #if 0
  2944. //Not used at the moment - for reasons see the comments below.
  2945. /*
  2946. The idea of this code is to convert a.b.c into normalize(a.b, a.b.c) if a.b is an out-of scope dataset
  2947. However the following isn't good enough since the fields from a.b also need to be accessible. We would
  2948. need to introduce a field in the result $parent$, and also assign that across. Subsequent references to
  2949. a.b.xyz would need to be converted to in.parent.xyz. It will generate very inefficient code, so not going
  2950. to go this way at the moment.
  2951. */
  2952. if (!isNewSelector(expr) || !expr->isDataset())
  2953. return NULL;
  2954. IHqlExpression * ds = expr->queryChild(0);
  2955. if (!ds->isDataset())
  2956. return NULL;
  2957. //If we are a no_select of a no_select that is also new, insert an implicit denormalized
  2958. HqlExprArray args;
  2959. args.append(*LINK(ds));
  2960. OwnedHqlExpr selSeq = createSelectorSequence();
  2961. HqlExprArray selectArgs;
  2962. unwindChildren(selectArgs, expr);
  2963. selectArgs.replace(*createSelector(no_left, ds, selSeq), 0);
  2964. removeAttribute(selectArgs, newAtom);
  2965. args.append(*expr->clone(selectArgs));
  2966. //Create a transform self := right;
  2967. OwnedHqlExpr right = createSelector(no_right, expr, selSeq);
  2968. OwnedHqlExpr assign = createAssign(getSelf(expr), LINK(right));
  2969. OwnedHqlExpr transform = createValue(no_transform, makeTransformType(LINK(expr->queryRecordType())), LINK(assign));
  2970. args.append(*LINK(transform));
  2971. args.append(*LINK(selSeq));
  2972. args.append(*createAttribute(_internal_Atom));
  2973. return createDataset(no_normalize, args);
  2974. #endif
  2975. }
  2976. IHqlExpression * ThorHqlTransformer::normalizeSort(IHqlExpression * expr)
  2977. {
  2978. IHqlExpression * dataset = expr->queryChild(0);
  2979. IHqlExpression * sortlist = expr->queryChild(1);
  2980. OwnedHqlExpr newsort = simplifySortlistComplexity(sortlist);
  2981. if (newsort)
  2982. {
  2983. if (newsort == sortlist)
  2984. {
  2985. dbglogExpr(sortlist);
  2986. throwUnexpected();
  2987. }
  2988. HqlExprArray args;
  2989. unwindChildren(args, expr);
  2990. args.replace(*newsort.getClear(), 1);
  2991. return expr->clone(args);
  2992. }
  2993. node_operator op = expr->getOperator();
  2994. if (translator.targetThor())
  2995. {
  2996. if ((op == no_sort) && !isGrouped(expr) && !expr->hasAttribute(localAtom))
  2997. {
  2998. //sort(ds, a,b,c) - check so see if there is a previous sort distribution of sort(ds,a,b,c) if so, this sort can be done locally
  2999. if (queryDistribution(expr) == queryDistribution(dataset))
  3000. return appendLocalAttribute(expr);
  3001. }
  3002. }
  3003. if (op == no_sorted)
  3004. {
  3005. IHqlExpression * normalized = normalizeSortSteppedIndex(expr, sortedAtom);
  3006. if (normalized)
  3007. return normalized;
  3008. }
  3009. bool isLocal = expr->hasAttribute(localAtom);
  3010. bool alwaysLocal = !translator.targetThor();
  3011. if ((op != no_assertsorted) && isAlreadySorted(dataset, sortlist, isLocal||alwaysLocal, false, true))
  3012. return LINK(dataset);
  3013. if (op == no_sorted)
  3014. return normalizeSortSteppedIndex(expr, sortedAtom);
  3015. //NOTE: We can't convert a global sort to a subsort because that will change the distribution
  3016. if (options.implicitSubSort && (isLocal || alwaysLocal) && (op != no_assertsorted))
  3017. {
  3018. OwnedHqlExpr subsorted = getSubSort(dataset, sortlist, isLocal, false, alwaysLocal);
  3019. if (subsorted)
  3020. return dataset->cloneAllAnnotations(subsorted);
  3021. }
  3022. return NULL;
  3023. }
  3024. IHqlExpression * ThorHqlTransformer::normalizeSubSort(IHqlExpression * expr)
  3025. {
  3026. IHqlExpression * sortlist = expr->queryChild(1);
  3027. IHqlExpression * grouping = expr->queryChild(2);
  3028. OwnedHqlExpr newsort = simplifySortlistComplexity(sortlist);
  3029. OwnedHqlExpr newgrouping = simplifySortlistComplexity(grouping);
  3030. if (newsort || newgrouping)
  3031. {
  3032. HqlExprArray args;
  3033. unwindChildren(args, expr);
  3034. if (newsort)
  3035. args.replace(*newsort.getClear(), 1);
  3036. if (newgrouping)
  3037. args.replace(*newgrouping.getClear(), 2);
  3038. return expr->clone(args);
  3039. }
  3040. if (translator.targetThor() && !expr->hasAttribute(localAtom))
  3041. return convertSubSortToGroupedSort(expr);
  3042. return NULL;
  3043. }
  3044. IHqlExpression * ThorHqlTransformer::normalizeSortSteppedIndex(IHqlExpression * expr, IAtom * attrName)
  3045. {
  3046. node_operator op = expr->getOperator();
  3047. if (op == no_assertsorted)
  3048. return NULL;
  3049. IHqlExpression * dataset = expr->queryChild(0);
  3050. node_operator datasetOp = dataset->getOperator();
  3051. if ((datasetOp == no_keyindex) || (datasetOp == no_newkeyindex))
  3052. {
  3053. if (!dataset->hasAttribute(attrName))
  3054. {
  3055. HqlExprArray selects;
  3056. IHqlExpression * sortList = expr->queryChild(1);
  3057. if (sortList)
  3058. {
  3059. OwnedHqlExpr mapped = replaceSelector(sortList, dataset->queryNormalizedSelector(), queryActiveTableSelector());
  3060. unwindChildren(selects, mapped);
  3061. }
  3062. HqlExprArray args;
  3063. unwindChildren(args, dataset);
  3064. args.append(*createExprAttribute(attrName, selects));
  3065. return dataset->clone(args);
  3066. }
  3067. }
  3068. return NULL;
  3069. }
  3070. IHqlExpression * ThorHqlTransformer::normalizeTempTable(IHqlExpression * expr)
  3071. {
  3072. #if 0
  3073. //This would be a great improvement to the generated code, but the xml storage formats are different + it doesn't cope with ALL.
  3074. IHqlExpression * values = expr->queryChild(0);
  3075. ITypeInfo * valuesType = values->queryType();
  3076. if ((values->getOperator() == no_getresult) && (valuesType->getTypeCode() == type_set))
  3077. {
  3078. IHqlExpression * record = expr->queryChild(1);
  3079. if ((record->numChildren() == 1) && (valuesType->queryChildType() == record->queryChild(0)->queryType()))
  3080. {
  3081. HqlExprArray args;
  3082. args.append(*LINK(record));
  3083. args.append(*createAttribute(sequenceAtom, LINK(values->queryChild(0))));
  3084. if (values->queryChild(1))
  3085. args.append(*createAttribute(nameAtom, LINK(values->queryChild(1))));
  3086. return createDataset(no_workunit_dataset, args);
  3087. }
  3088. }
  3089. #endif
  3090. return NULL;
  3091. }
  3092. IHqlExpression * ThorHqlTransformer::normalizeChooseN(IHqlExpression * expr)
  3093. {
  3094. OwnedHqlExpr first = foldHqlExpression(queryRealChild(expr, 2));
  3095. if (first)
  3096. {
  3097. if (matchesConstantValue(first, 1))
  3098. {
  3099. HqlExprArray args;
  3100. unwindChildren(args, expr);
  3101. args.remove(2);
  3102. return expr->clone(args);
  3103. }
  3104. }
  3105. if (!options.spotTopN) return NULL;
  3106. return queryConvertChoosenNSort(expr, topNlimit);
  3107. }
  3108. static IHqlExpression * extractPrefetchFields(HqlExprArray & fields, HqlExprArray & values, IHqlExpression * ds, IHqlExpression * expr)
  3109. {
  3110. switch (expr->getOperator())
  3111. {
  3112. case no_newtransform:
  3113. case no_transform:
  3114. case no_assignall:
  3115. case NO_AGGREGATEGROUP:
  3116. case no_sortlist:
  3117. {
  3118. HqlExprArray args;
  3119. ForEachChild(i, expr)
  3120. args.append(*extractPrefetchFields(fields, values, ds, expr->queryChild(i)));
  3121. return expr->clone(args);
  3122. }
  3123. case no_assign:
  3124. {
  3125. HqlExprArray args;
  3126. args.append(*LINK(expr->queryChild(0)));
  3127. args.append(*extractPrefetchFields(fields, values, ds, expr->queryChild(1)));
  3128. return expr->clone(args);
  3129. }
  3130. case no_attr:
  3131. case no_attr_expr:
  3132. case no_attr_link:
  3133. case no_record:
  3134. case no_field:
  3135. return LINK(expr);
  3136. }
  3137. unsigned match = values.find(*expr);
  3138. if (match == NotFound)
  3139. {
  3140. //What about preserving link counting on datasets?
  3141. match = fields.ordinality();
  3142. StringBuffer name;
  3143. name.append("_f").append(match).append("_");
  3144. IHqlExpression * field = createFieldFromValue(createIdAtom(name.str()), expr);
  3145. fields.append(*field);
  3146. values.append(*LINK(expr));
  3147. }
  3148. return createSelectExpr(getActiveTableSelector(), LINK(&fields.item(match)));
  3149. }
  3150. IHqlExpression * ThorHqlTransformer::normalizePrefetchAggregate(IHqlExpression * expr)
  3151. {
  3152. //This optimization may be worth doing even if there is no prefetch attribute if the value being summed is very complicated!
  3153. IHqlExpression * prefetch = expr->queryAttribute(prefetchAtom);
  3154. if (!prefetch)
  3155. return NULL;
  3156. //Create a prefetch project for all parameters to count/sum/grouping expressions
  3157. //and then aggregate those values.
  3158. IHqlExpression * ds = expr->queryChild(0);
  3159. HqlExprArray tempArgs, fields, values;
  3160. ForEachChildFrom(i, expr, 2)
  3161. {
  3162. IHqlExpression * cur = expr->queryChild(i);
  3163. if (cur != prefetch)
  3164. tempArgs.append(*extractPrefetchFields(fields, values, ds, cur));
  3165. }
  3166. OwnedHqlExpr newRecord = createRecord(fields);
  3167. OwnedHqlExpr self = createSelector(no_self, newRecord, NULL);
  3168. HqlExprArray assigns;
  3169. ForEachItemIn(iv, fields)
  3170. {
  3171. IHqlExpression * tgt = createSelectExpr(LINK(self), &OLINK(fields.item(iv)));
  3172. assigns.append(*createAssign(tgt, &OLINK(values.item(iv))));
  3173. }
  3174. HqlExprArray args;
  3175. args.append(*LINK(ds));
  3176. args.append(*LINK(newRecord));
  3177. args.append(*createValue(no_newtransform, makeTransformType(newRecord->getType()), assigns));
  3178. args.append(*LINK(prefetch));
  3179. OwnedHqlExpr project = createDataset(no_newusertable, args);
  3180. project.setown(cloneInheritedAnnotations(expr, project));
  3181. args.kill();
  3182. args.append(*LINK(project));
  3183. args.append(*LINK(expr->queryChild(1)));
  3184. ForEachItemIn(i2, tempArgs)
  3185. args.append(*replaceSelector(&tempArgs.item(i2), queryActiveTableSelector(), project->queryNormalizedSelector()));
  3186. return expr->clone(args);
  3187. }
  3188. static IHqlExpression * convertAggregateGroupingToGroupedAggregate(IHqlExpression * expr, IHqlExpression* groupBy)
  3189. {
  3190. IHqlExpression * dataset = expr->queryChild(0);
  3191. HqlExprArray groupArgs;
  3192. groupArgs.append(*LINK(dataset));
  3193. groupArgs.append(*LINK(groupBy));
  3194. groupArgs.append(*createAttribute(allAtom));
  3195. unwindChildren(groupArgs, expr, 4);
  3196. OwnedHqlExpr result = createDataset(no_group, groupArgs);
  3197. result.setown(cloneInheritedAnnotations(expr, result));
  3198. HqlExprArray args;
  3199. unwindChildren(args, expr);
  3200. args.replace(*result.getClear(), 0);
  3201. args.remove(3); // no longer grouped.
  3202. return expr->clone(args);
  3203. }
  3204. IHqlExpression * ThorHqlTransformer::getMergeTransform(IHqlExpression * dataset, IHqlExpression * transform)
  3205. {
  3206. HqlExprArray args;
  3207. ForEachChild(i, transform)
  3208. {
  3209. IHqlExpression * cur = transform->queryChild(i);
  3210. switch (cur->getOperator())
  3211. {
  3212. case no_assignall:
  3213. args.append(*getMergeTransform(dataset, cur));
  3214. break;
  3215. case no_assign:
  3216. {
  3217. IHqlExpression * lhs = cur->queryChild(0);
  3218. IHqlExpression * lhsField = lhs->queryChild(1);
  3219. IHqlExpression * rhs = cur->queryChild(1);
  3220. OwnedHqlExpr selected = createSelectExpr(LINK(dataset), LINK(lhsField));
  3221. OwnedHqlExpr newRhs;
  3222. node_operator rhsOp = rhs->getOperator();
  3223. switch (rhsOp)
  3224. {
  3225. case no_countgroup:
  3226. case no_sumgroup:
  3227. newRhs.setown(createValue(no_sumgroup, selected->getType(), LINK(selected)));
  3228. break;
  3229. case no_maxgroup:
  3230. case no_mingroup:
  3231. newRhs.setown(createValue(rhsOp, selected->getType(), LINK(selected)));
  3232. break;
  3233. case no_existsgroup:
  3234. newRhs.setown(createValue(no_existsgroup, selected->getType(), LINK(selected)));
  3235. break;
  3236. case no_vargroup:
  3237. case no_covargroup:
  3238. case no_corrgroup:
  3239. case no_avegroup:
  3240. throwUnexpected();
  3241. default:
  3242. newRhs.set(selected);
  3243. break;
  3244. }
  3245. args.append(*createAssign(LINK(lhs), newRhs.getClear()));
  3246. break;
  3247. }
  3248. default:
  3249. args.append(*LINK(cur));
  3250. break;
  3251. }
  3252. }
  3253. return transform->clone(args);
  3254. }
  3255. //Convert table(x { count(group), sum(group, x) }, gr) to
  3256. //sort(x, gr, local) -> group(gr) -> aggregate -> distribute(merge) -> group(local) -> aggregate'
  3257. IHqlExpression * ThorHqlTransformer::normalizeMergeAggregate(IHqlExpression * expr)
  3258. {
  3259. IHqlExpression * dataset = expr->queryChild(0);
  3260. IHqlExpression * groupBy = expr->queryChild(3);
  3261. //If locally distributed then don't do anything
  3262. OwnedHqlExpr noMerge = removeAttribute(expr, mergeAtom);
  3263. //This transformation only works for grouped aggregation
  3264. if (!groupBy || groupBy->isAttribute())
  3265. return noMerge.getClear();
  3266. if (!translator.targetThor() || expr->hasAttribute(localAtom) || isPartitionedForGroup(dataset, groupBy, true))
  3267. return noMerge.getClear();
  3268. //Convert the aggregation (so no covariance/ave and other computed fields)
  3269. OwnedHqlExpr normalized = normalizeTableToAggregate(noMerge, true);
  3270. IHqlExpression * aggregate = normalized;
  3271. if (aggregate->getOperator() != no_newaggregate)
  3272. aggregate = aggregate->queryChild(0);
  3273. assertex(aggregate->getOperator() == no_newaggregate);
  3274. HqlExprArray localAggregateArgs;
  3275. unwindChildren(localAggregateArgs, aggregate);
  3276. removeAttribute(localAggregateArgs, hashAtom);
  3277. removeAttribute(localAggregateArgs, mergeAtom);
  3278. localAggregateArgs.append(*createLocalAttribute());
  3279. localAggregateArgs.append(*createAttribute(sortedAtom));
  3280. //Local aggregate and force a local sort order to be used
  3281. OwnedHqlExpr localAggregate = aggregate->clone(localAggregateArgs);
  3282. OwnedHqlExpr localGroupedAggregate = convertAggregateGroupingToGroupedAggregate(localAggregate, groupBy);
  3283. //Ensure the group,all is transformed to a local sort, local group
  3284. OwnedHqlExpr transformedFirstAggregate = transform(localGroupedAggregate);
  3285. //Use distribute(,MERGE) to move rows globally, and remain sorted
  3286. //Note grouping fields need to be mapped using the fields projected by the aggregate
  3287. TableProjectMapper mapper(transformedFirstAggregate);
  3288. bool groupCanBeMapped = false;
  3289. OwnedHqlExpr mappedGrouping = mapper.collapseFields(groupBy, dataset, transformedFirstAggregate, &groupCanBeMapped);
  3290. assertex(groupCanBeMapped);
  3291. OwnedHqlExpr sortOrder = getExistingSortOrder(transformedFirstAggregate, true, true);
  3292. OwnedHqlExpr mergeAttr = createExprAttribute(mergeAtom, replaceSelector(sortOrder, queryActiveTableSelector(), transformedFirstAggregate));
  3293. OwnedHqlExpr hashed = createValue(no_hash32, LINK(unsignedType), LINK(mappedGrouping), createAttribute(internalAtom));
  3294. OwnedHqlExpr redistributed = createDatasetF(no_distribute, LINK(transformedFirstAggregate), LINK(hashed), mergeAttr.getClear(), NULL);
  3295. redistributed.setown(cloneInheritedAnnotations(expr, redistributed));
  3296. OwnedHqlExpr grouped = createDatasetF(no_group, LINK(redistributed), LINK(mappedGrouping), createLocalAttribute(), NULL);
  3297. grouped.setown(cloneInheritedAnnotations(expr, grouped));
  3298. HqlExprArray args;
  3299. args.append(*LINK(grouped));
  3300. args.append(*LINK(localAggregate->queryChild(1)));
  3301. args.append(*getMergeTransform(grouped->queryNormalizedSelector(), localAggregate->queryChild(2)));
  3302. unwindChildren(args, localAggregate, 4);
  3303. OwnedHqlExpr newAggregate = localAggregate->clone(args);
  3304. if (aggregate == normalized)
  3305. return newAggregate.getClear();
  3306. return replaceChildDataset(normalized, newAggregate, 0);
  3307. }
  3308. static bool reorderAggregateFields(HqlExprArray & aggregateFields, HqlExprArray & aggregateAssigns)
  3309. {
  3310. bool needToReorder = false;
  3311. bool dynamicOffset = false; // Will the offset of the next field vary with each row processed?
  3312. ForEachItemIn(i, aggregateFields)
  3313. {
  3314. IHqlExpression & cur = aggregateFields.item(i);
  3315. IHqlExpression * value = aggregateAssigns.item(i).queryChild(1);
  3316. //If any field follows a dynamic sized aggregate then the record needs to be reordered
  3317. if (dynamicOffset)
  3318. needToReorder = true;
  3319. if (isUnknownSize(&cur))
  3320. {
  3321. if (value->isGroupAggregateFunction())
  3322. dynamicOffset = true;
  3323. }
  3324. }
  3325. if (!needToReorder)
  3326. return false;
  3327. HqlExprArray newFields;
  3328. HqlExprArray newAssigns;
  3329. for (unsigned pass = 0; pass < 3; pass++)
  3330. {
  3331. //Fixed size fields first, then non aggregates, then aggregates
  3332. ForEachItemIn(i1, aggregateFields)
  3333. {
  3334. IHqlExpression & cur = aggregateFields.item(i1);
  3335. IHqlExpression & assign = aggregateAssigns.item(i1);
  3336. bool copy = false;
  3337. if (!isUnknownSize(&cur))
  3338. {
  3339. //Place all fixed size fields first
  3340. copy = (pass == 0);
  3341. }
  3342. else if (!assign.queryChild(1)->isGroupAggregateFunction())
  3343. {
  3344. //Then any variable size fields that are only assigned once
  3345. copy = (pass == 1);
  3346. }
  3347. else
  3348. {
  3349. //Finally any variable size aggregates
  3350. copy = (pass == 2);
  3351. }
  3352. if (copy)
  3353. {
  3354. newFields.append(OLINK(cur));
  3355. newAssigns.append(OLINK(assign));
  3356. }
  3357. }
  3358. }
  3359. aggregateFields.swapWith(newFields);
  3360. aggregateAssigns.swapWith(newAssigns);
  3361. return true;
  3362. }
  3363. IHqlExpression * ThorHqlTransformer::normalizeTableToAggregate(IHqlExpression * expr, bool canOptimizeCasts)
  3364. {
  3365. IHqlExpression * dataset = expr->queryChild(0);
  3366. IHqlExpression * record = expr->queryChild(1);
  3367. IHqlExpression * transform = expr->queryChild(2);
  3368. IHqlExpression * groupBy = expr->queryChild(3);
  3369. if (!isAggregateDataset(expr) || expr->hasAttribute(_normalized_Atom))
  3370. return NULL;
  3371. //MORE: Should fail if asked to group by variable length field, or do max/min on variable length field.
  3372. HqlExprArray aggregateFields;
  3373. HqlExprArray aggregateAssigns;
  3374. HqlExprArray extraAssigns;
  3375. bool extraSelectNeeded = false;
  3376. OwnedHqlExpr self = getSelf(expr);
  3377. ForEachChild(idx, transform)
  3378. {
  3379. IHqlExpression * assign=transform->queryChild(idx);
  3380. IHqlExpression * cur = assign->queryChild(0);
  3381. IHqlExpression * src = assign->queryChild(1);
  3382. IHqlExpression * mapped;
  3383. if (src->isGroupAggregateFunction())
  3384. {
  3385. mapped = normalizeAggregateExpr(dataset->queryNormalizedSelector(), cur, src, aggregateFields, aggregateAssigns, extraSelectNeeded, canOptimizeCasts);
  3386. }
  3387. else
  3388. {
  3389. mapped = replaceSelector(cur, self, queryActiveTableSelector());
  3390. // Not an aggregate - must be an expression that is used in the grouping
  3391. aggregateFields.append(*LINK(cur->queryChild(1)));
  3392. aggregateAssigns.append(*createAssign(LINK(mapped), LINK(src)));
  3393. }
  3394. // Add expression to calculate the fields to the second projection
  3395. extraAssigns.append(*createAssign(LINK(cur), mapped));
  3396. }
  3397. //Now add any grouping fields.......
  3398. IHqlExpression * newGroupBy = NULL;
  3399. if (groupBy && !groupBy->isAttribute())
  3400. {
  3401. unsigned numGroupBy = groupBy->numChildren();
  3402. HqlExprArray newGroupElement;
  3403. for (unsigned idx = 0; idx < numGroupBy; idx++)
  3404. {
  3405. IHqlExpression * curGroup = groupBy->queryChild(idx);
  3406. bool matched = false;
  3407. ForEachItemIn(idxa, aggregateAssigns)
  3408. {
  3409. IHqlExpression * rhs = aggregateAssigns.item(idxa).queryChild(1);
  3410. if (rhs->getOperator() == no_activerow)
  3411. rhs = rhs->queryChild(0);
  3412. if (rhs == curGroup)
  3413. {
  3414. matched = true;
  3415. break;
  3416. }
  3417. }
  3418. if (!matched)
  3419. {
  3420. StringBuffer temp;
  3421. temp.append("_agg_").append(aggregateAssigns.ordinality());
  3422. IHqlExpression * targetField = createFieldFromValue(createIdAtom(temp.str()), curGroup);
  3423. aggregateFields.append(*targetField);
  3424. aggregateAssigns.append(*createAssign(createSelectExpr(getActiveTableSelector(), LINK(targetField)), LINK(curGroup)));
  3425. extraSelectNeeded = true;
  3426. }
  3427. newGroupElement.append(*LINK(curGroup));
  3428. }
  3429. newGroupBy = createSortList(newGroupElement);
  3430. }
  3431. if (reorderAggregateFields(aggregateFields, aggregateAssigns))
  3432. extraSelectNeeded = true;
  3433. IHqlExpression * aggregateRecord = extraSelectNeeded ? createRecord(aggregateFields) : LINK(record);
  3434. OwnedHqlExpr aggregateSelf = getSelf(aggregateRecord);
  3435. replaceAssignSelector(aggregateAssigns, aggregateSelf);
  3436. IHqlExpression * aggregateTransform = createValue(no_newtransform, makeTransformType(aggregateRecord->getType()), aggregateAssigns);
  3437. HqlExprArray newAggregateArgs;
  3438. newAggregateArgs.append(*LINK(dataset));
  3439. newAggregateArgs.append(*aggregateRecord);
  3440. newAggregateArgs.append(*aggregateTransform);
  3441. if (newGroupBy)
  3442. newAggregateArgs.append(*newGroupBy);
  3443. unwindAttributes(newAggregateArgs, expr);
  3444. if (!expr->hasAttribute(localAtom) && newGroupBy && !isGrouped(dataset) && isPartitionedForGroup(dataset, newGroupBy, true))
  3445. newAggregateArgs.append(*createLocalAttribute());
  3446. newAggregateArgs.append(*createAttribute(_normalized_Atom));
  3447. OwnedHqlExpr ret = createDataset(no_newaggregate, newAggregateArgs);
  3448. if (extraSelectNeeded)
  3449. ret.setown(cloneInheritedAnnotations(expr, ret));
  3450. else
  3451. ret.setown(expr->cloneAllAnnotations(ret));
  3452. if (expr->hasAttribute(mergeAtom))
  3453. ret.setown(normalizeMergeAggregate(ret));
  3454. if (extraSelectNeeded)
  3455. {
  3456. replaceAssignSelector(extraAssigns, ret);
  3457. IHqlExpression * projectTransform = createValue(no_newtransform, makeTransformType(record->getType()), extraAssigns);
  3458. ret.setown(createDataset(no_newusertable, ret.getClear(), createComma(LINK(record), projectTransform)));
  3459. ret.setown(expr->cloneAllAnnotations(ret));
  3460. }
  3461. return ret.getClear();
  3462. }
  3463. IHqlExpression * ThorHqlTransformer::normalizeTableGrouping(IHqlExpression * expr)
  3464. {
  3465. //Transform table(x,y,z) to table(group(x,z),y)
  3466. IHqlExpression * dataset = expr->queryChild(0);
  3467. LinkedHqlExpr group = queryRealChild(expr, 3);
  3468. if (group)
  3469. {
  3470. if (expr->hasAttribute(mergeAtom))
  3471. return normalizeMergeAggregate(expr);
  3472. bool useHashAggregate = expr->hasAttribute(fewAtom);
  3473. if (expr->getOperator() == no_aggregate)
  3474. {
  3475. OwnedHqlExpr selector = createSelector(no_left, dataset->queryRecord(), querySelSeq(expr));
  3476. group.setown(replaceSelector(group, selector, dataset));
  3477. //Cannot use a hash aggregate if we don't know the mapping from input to output fields...
  3478. if (!isKnownTransform(expr->queryChild(2)))
  3479. useHashAggregate = false;
  3480. }
  3481. if (useHashAggregate && group->isConstant() && !translator.targetThor())
  3482. return removeAttribute(expr, fewAtom);
  3483. if (!expr->hasAttribute(manyAtom) && !expr->hasAttribute(sortedAtom))
  3484. {
  3485. if (isSmallGrouping(group))
  3486. {
  3487. OwnedHqlExpr newsort = simplifySortlistComplexity(group);
  3488. if (!newsort)
  3489. newsort.set(group);
  3490. LinkedHqlExpr ds = dataset;
  3491. if (isGrouped(ds))
  3492. {
  3493. ds.setown(createDataset(no_group, ds.getClear(), NULL));
  3494. ds.setown(cloneInheritedAnnotations(expr, ds));
  3495. }
  3496. OwnedHqlExpr sorted = ensureSortedForGroup(ds, newsort, expr->hasAttribute(localAtom), !translator.targetThor(), options.implicitGroupSubSort);
  3497. //For thor a global grouped aggregate would transfer elements between nodes so it is still likely to
  3498. //be more efficient to do a hash aggregate. Even better would be to check the distribution
  3499. if ((sorted != ds) ||
  3500. (translator.targetThor() && !expr->hasAttribute(localAtom) && !isPartitionedForGroup(ds, newsort, true)))
  3501. useHashAggregate = true;
  3502. }
  3503. //Default to a hash aggregate for child queries/normalized sources
  3504. IHqlExpression * rootDs = queryExpression(dataset->queryDataset()->queryRootTable());
  3505. if (rootDs && rootDs->getOperator() == no_select)
  3506. useHashAggregate = true;
  3507. }
  3508. if (!useHashAggregate && !expr->hasAttribute(groupedAtom))
  3509. return convertAggregateGroupingToGroupedAggregate(expr, group);
  3510. }
  3511. return NULL;
  3512. }
  3513. void HqlCppTranslator::convertLogicalToActivities(WorkflowItem & curWorkflow)
  3514. {
  3515. {
  3516. ThorHqlTransformer transformer(*this, targetClusterType, wu(), implicitFunctionId);
  3517. HqlExprArray & exprs = curWorkflow.queryExprs();
  3518. transformer.transformRoot(exprs);
  3519. }
  3520. if (queryOptions().normalizeLocations)
  3521. normalizeAnnotations(*this, curWorkflow.queryExprs());
  3522. }
  3523. //------------------------------------------------------------------------
  3524. static HqlTransformerInfo setResultToExtractTransformerInfo("SetResultToExtractTransformer");
  3525. SetResultToExtractTransformer::SetResultToExtractTransformer()
  3526. : NewHqlTransformer(setResultToExtractTransformerInfo)
  3527. {
  3528. }
  3529. IHqlExpression * SetResultToExtractTransformer::createTransformed(IHqlExpression * expr)
  3530. {
  3531. OwnedHqlExpr transformed = PARENT::createTransformed(expr);
  3532. if (transformed->getOperator() == no_setresult)
  3533. {
  3534. OwnedHqlExpr normalized = convertSetResultToExtract(transformed);
  3535. if (normalized && (normalized != transformed))
  3536. transformed.set(normalized);
  3537. }
  3538. return transformed.getClear();
  3539. }
  3540. //------------------------------------------------------------------------
  3541. CompoundSourceInfo::CompoundSourceInfo(IHqlExpression * _original) : NewTransformInfo(_original)
  3542. {
  3543. sourceOp = no_none;
  3544. mode = no_none;
  3545. splitCount = 0;
  3546. reset();
  3547. }
  3548. void CompoundSourceInfo::reset()
  3549. {
  3550. forceCompound = false;
  3551. isBoundary = false;
  3552. isPreloaded = false;
  3553. isLimited = false;
  3554. hasChoosen = false;
  3555. hasSkipLimit = false;
  3556. isCloned = false;
  3557. isFiltered = false;
  3558. isPostFiltered = false;
  3559. isCreateRowLimited = false;
  3560. hasOnFail = false;
  3561. }
  3562. bool CompoundSourceInfo::canMergeLimit(IHqlExpression * expr, ClusterType targetClusterType) const
  3563. {
  3564. if (isAggregate() || isChooseNAllLimit(expr->queryChild(1)) || !isBinary())
  3565. return false;
  3566. node_operator op = expr->getOperator();
  3567. switch (op)
  3568. {
  3569. case no_limit:
  3570. //Can't merge a limit into a choosen() because the limit will be applied first
  3571. if (isLimited || hasChoosen)
  3572. return false;
  3573. //Don't merge skip and onfail limits into activities that can't implement them completely
  3574. if (targetClusterType != RoxieCluster)
  3575. {
  3576. if (expr->hasAttribute(skipAtom) || expr->hasAttribute(onFailAtom))
  3577. return false;
  3578. }
  3579. else
  3580. {
  3581. //Can always limit a count/aggregate with a skip limit - just resets count to 0
  3582. if (expr->hasAttribute(skipAtom))
  3583. return true;
  3584. }
  3585. break;
  3586. case no_choosen:
  3587. if (hasChoosen)
  3588. return false;
  3589. break;
  3590. }
  3591. switch (sourceOp)
  3592. {
  3593. case no_compound_diskread:
  3594. case no_compound_disknormalize:
  3595. case no_compound_indexread:
  3596. case no_compound_indexnormalize:
  3597. return true;
  3598. }
  3599. return false;
  3600. }
  3601. void CompoundSourceInfo::ensureCompound()
  3602. {
  3603. if (sourceOp != no_none)
  3604. {
  3605. forceCompound = true;
  3606. #if 0
  3607. //MORE: We should really remove the sharing for entries that are going to become compound activities.
  3608. //However, that isn't just for this case - should be iterative
  3609. //e.g. while (spotMoreCompoundActivities())....
  3610. IHqlExpression * search = original;
  3611. for (;;)
  3612. {
  3613. CompoundSourceInfo * extra = queryExtra(search);
  3614. if (extra->sharedCount-- > 1)
  3615. break;
  3616. search = search->queryChild(0);
  3617. }
  3618. #endif
  3619. }
  3620. }
  3621. bool CompoundSourceInfo::inherit(const CompoundSourceInfo & other, node_operator newSourceOp)
  3622. {
  3623. isLimited = other.isLimited;
  3624. hasSkipLimit = other.hasSkipLimit;
  3625. hasChoosen = other.hasChoosen;
  3626. isFiltered = other.isFiltered;
  3627. isPostFiltered = other.isPostFiltered;
  3628. isPreloaded = other.isPreloaded;
  3629. isCreateRowLimited = other.isCreateRowLimited;
  3630. hasOnFail = other.hasOnFail;
  3631. mode = other.mode;
  3632. uid.set(other.uid);
  3633. if (other.sourceOp == no_none)
  3634. return false;
  3635. if (newSourceOp == no_none)
  3636. {
  3637. if (other.isCloned)
  3638. return false;
  3639. newSourceOp = other.sourceOp;
  3640. }
  3641. sourceOp = newSourceOp;
  3642. return true;
  3643. }
  3644. bool CompoundSourceInfo::isAggregate() const
  3645. {
  3646. switch (sourceOp)
  3647. {
  3648. case no_compound_diskaggregate:
  3649. case no_compound_diskcount:
  3650. case no_compound_diskgroupaggregate:
  3651. case no_compound_indexaggregate:
  3652. case no_compound_indexcount:
  3653. case no_compound_indexgroupaggregate:
  3654. case no_compound_childaggregate:
  3655. case no_compound_childcount:
  3656. case no_compound_childgroupaggregate:
  3657. return true;
  3658. }
  3659. return false;
  3660. }
  3661. //This doesn't try to restrict creating the compound nodes to the inner level, but will also create them for nested children.
  3662. //This shouldn't cause any problems, since compound operators within compounds are ignored, and it means that this transformer
  3663. //doesn't have to cope with being scope dependent.
  3664. //Calling the transformer again later on child queries should extend the compound activities if appropriate.
  3665. static HqlTransformerInfo compoundSourceTransformerInfo("CompoundSourceTransformer");
  3666. CompoundSourceTransformer::CompoundSourceTransformer(HqlCppTranslator & _translator, unsigned _flags)
  3667. : NewHqlTransformer(compoundSourceTransformerInfo), translator(_translator)
  3668. {
  3669. targetClusterType = translator.getTargetClusterType();
  3670. flags = _flags;
  3671. insideCompound = false;
  3672. candidate = false;
  3673. }
  3674. void CompoundSourceTransformer::analyseGatherInfo(IHqlExpression * expr)
  3675. {
  3676. CompoundSourceInfo * extra = queryBodyExtra(expr);
  3677. node_operator op = expr->getOperator();
  3678. bool wasInsideCompound = insideCompound;
  3679. if (!insideCompound)
  3680. extra->noteUsage();
  3681. if (!expr->isDataset())
  3682. insideCompound = false;
  3683. switch (op)
  3684. {
  3685. case no_fetch:
  3686. {
  3687. unsigned max = expr->numChildren();
  3688. for (unsigned i =1; i < max; i++)
  3689. analyseExpr(expr->queryChild(i));
  3690. break;
  3691. }
  3692. case no_keyed:
  3693. case no_record:
  3694. case no_attr:
  3695. case no_attr_expr:
  3696. break;
  3697. case no_keyedlimit:
  3698. case no_compound_diskread:
  3699. case no_compound_disknormalize:
  3700. case no_compound_diskaggregate:
  3701. case no_compound_diskcount:
  3702. case no_compound_diskgroupaggregate:
  3703. case no_compound_indexread:
  3704. case no_compound_indexnormalize:
  3705. case no_compound_indexaggregate:
  3706. case no_compound_indexcount:
  3707. case no_compound_indexgroupaggregate:
  3708. case no_compound_childread:
  3709. case no_compound_childnormalize:
  3710. case no_compound_childaggregate:
  3711. case no_compound_childcount:
  3712. case no_compound_childgroupaggregate:
  3713. case no_compound_selectnew:
  3714. case no_compound_inline:
  3715. case no_preload:
  3716. insideCompound = true;
  3717. NewHqlTransformer::analyseExpr(expr);
  3718. break;
  3719. case no_filter:
  3720. if (filterIsKeyed(expr))
  3721. insideCompound = true;
  3722. NewHqlTransformer::analyseExpr(expr);
  3723. break;
  3724. case no_hqlproject:
  3725. case no_newusertable:
  3726. case no_aggregate:
  3727. case no_newaggregate:
  3728. if (expr->hasAttribute(keyedAtom))
  3729. insideCompound = true;
  3730. NewHqlTransformer::analyseExpr(expr);
  3731. break;
  3732. case no_join:
  3733. if (isKeyedJoin(expr) && !expr->hasAttribute(_complexKeyed_Atom))
  3734. {
  3735. analyseExpr(expr->queryChild(0));
  3736. doAnalyseChildren(expr, 2);
  3737. }
  3738. else
  3739. NewHqlTransformer::analyseExpr(expr);
  3740. break;
  3741. default:
  3742. NewHqlTransformer::analyseExpr(expr);
  3743. break;
  3744. }
  3745. switch (op)
  3746. {
  3747. case no_newkeyindex:
  3748. extra->sourceOp = no_compound_indexread;
  3749. extra->uid.set(expr->queryAttribute(_uid_Atom));
  3750. extra->mode = no_thor;
  3751. break;
  3752. case no_table:
  3753. {
  3754. IHqlExpression * mode = expr->queryChild(2);
  3755. if (!mode)
  3756. break;
  3757. switch (mode->getOperator())
  3758. {
  3759. case no_thor:
  3760. case no_flat:
  3761. if ((flags & CSFcompoundSpill) || !expr->hasAttribute(_spill_Atom))
  3762. {
  3763. extra->sourceOp = no_compound_diskread;
  3764. extra->isPreloaded = expr->hasAttribute(preloadAtom);
  3765. extra->uid.set(expr->queryAttribute(_uid_Atom));
  3766. extra->mode = no_thor;
  3767. }
  3768. break;
  3769. case no_csv:
  3770. if (translator.queryOptions().enableCompoundCsvRead)
  3771. {
  3772. extra->sourceOp = no_compound_diskread;
  3773. extra->isPreloaded = expr->hasAttribute(preloadAtom);
  3774. extra->uid.set(expr->queryAttribute(_uid_Atom));
  3775. extra->mode = mode->getOperator();
  3776. }
  3777. break;
  3778. }
  3779. break;
  3780. }
  3781. case no_hqlproject:
  3782. {
  3783. if (!expr->hasAttribute(prefetchAtom))
  3784. {
  3785. IHqlExpression * transform = expr->queryChild(1);
  3786. IHqlExpression * counter = queryAttributeChild(expr, _countProject_Atom, 0);
  3787. if (!counter || !transformContainsCounter(transform, counter))
  3788. {
  3789. IHqlExpression * dataset = expr->queryChild(0);
  3790. CompoundSourceInfo * parentExtra = queryBodyExtra(dataset);
  3791. //Skips in datasets don't work very well at the moment - pure() is a bit strict really.
  3792. if ((dataset->isPure() || expr->hasAttribute(keyedAtom)) && !parentExtra->isAggregate())
  3793. {
  3794. extra->inherit(*parentExtra);
  3795. if (expr->hasAttribute(keyedAtom))
  3796. extra->ensureCompound();
  3797. if (!isPureActivity(expr))
  3798. {
  3799. extra->isFiltered = true;
  3800. extra->isPostFiltered = true;
  3801. }
  3802. }
  3803. }
  3804. }
  3805. break;
  3806. }
  3807. case no_keyedlimit:
  3808. {
  3809. IHqlExpression * dataset = expr->queryChild(0);
  3810. CompoundSourceInfo * parentExtra = queryBodyExtra(dataset);
  3811. if (!parentExtra->isAggregate() && parentExtra->isBinary())
  3812. {
  3813. extra->inherit(*parentExtra);
  3814. extra->ensureCompound();
  3815. if (expr->hasAttribute(onFailAtom))
  3816. extra->hasOnFail = true;
  3817. }
  3818. break;
  3819. }
  3820. case no_inlinetable:
  3821. case no_temptable:
  3822. case no_datasetfromrow:
  3823. extra->sourceOp = no_compound_inline;
  3824. extra->mode = no_inlinetable;
  3825. break;
  3826. case no_workunit_dataset:
  3827. // extra->sourceOp = no_compound_childread;
  3828. break;
  3829. case no_getgraphresult:
  3830. case no_externalcall:
  3831. // if (expr->isDataset())
  3832. // extra->sourceOp = no_compound_childread;
  3833. break;
  3834. case no_compound_diskread:
  3835. case no_compound_disknormalize:
  3836. case no_compound_diskaggregate:
  3837. case no_compound_diskcount:
  3838. case no_compound_diskgroupaggregate:
  3839. case no_compound_indexread:
  3840. case no_compound_indexnormalize:
  3841. case no_compound_indexaggregate:
  3842. case no_compound_indexcount:
  3843. case no_compound_indexgroupaggregate:
  3844. case no_compound_childread:
  3845. case no_compound_childnormalize:
  3846. case no_compound_childaggregate:
  3847. case no_compound_childcount:
  3848. case no_compound_childgroupaggregate:
  3849. case no_compound_selectnew:
  3850. case no_compound_inline:
  3851. {
  3852. IHqlExpression * dataset = expr->queryChild(0);
  3853. CompoundSourceInfo * parentExtra = queryBodyExtra(dataset);
  3854. extra->inherit(*parentExtra, op);
  3855. extra->uid.set(expr->queryAttribute(_uid_Atom));
  3856. break;
  3857. }
  3858. case no_select:
  3859. if (expr->isDataset())
  3860. {
  3861. if (expr->hasAttribute(newAtom))
  3862. {
  3863. IHqlExpression * dataset = expr->queryChild(0);
  3864. CompoundSourceInfo * parentExtra = queryBodyExtra(dataset);
  3865. if (!parentExtra->isAggregate() && !parentExtra->hasAnyLimit() && parentExtra->isBinary())
  3866. {
  3867. node_operator newOp = no_none;
  3868. switch (parentExtra->sourceOp)
  3869. {
  3870. case no_compound_diskread:
  3871. case no_compound_disknormalize:
  3872. if (flags & CSFnewdisk)
  3873. newOp = no_compound_disknormalize;
  3874. break;
  3875. case no_compound_indexread:
  3876. case no_compound_indexnormalize:
  3877. if (flags & CSFnewindex)
  3878. newOp = no_compound_indexnormalize;
  3879. break;
  3880. case no_compound_childread:
  3881. case no_compound_childnormalize:
  3882. if (flags & CSFnewchild)
  3883. newOp = no_compound_childnormalize;
  3884. break;
  3885. }
  3886. if (newOp)
  3887. {
  3888. if (extra->inherit(*parentExtra))
  3889. {
  3890. extra->sourceOp = newOp;
  3891. extra->ensureCompound();
  3892. }
  3893. }
  3894. }
  3895. if ((flags & CSFnewchild) && (extra->sourceOp == no_none))
  3896. {
  3897. extra->reset();
  3898. extra->sourceOp = no_compound_selectnew;
  3899. extra->ensureCompound();
  3900. }
  3901. }
  3902. else
  3903. {
  3904. if ((flags & CSFnewchild) && !isTargetSelector(expr)) // latter is optimization - still works without this
  3905. {
  3906. extra->sourceOp = no_compound_childread;
  3907. }
  3908. }
  3909. }
  3910. break;
  3911. case no_choosen:
  3912. {
  3913. IHqlExpression * arg2 = expr->queryChild(2);
  3914. if (arg2 && !arg2->isPure())
  3915. break;
  3916. //fall through
  3917. }
  3918. case no_limit:
  3919. {
  3920. IHqlExpression * dataset = expr->queryChild(0);
  3921. CompoundSourceInfo * parentExtra = queryBodyExtra(dataset);
  3922. bool cloneRequired = needToCloneLimit(expr, parentExtra->sourceOp);
  3923. if (cloneRequired && !expr->queryChild(1)->isPure())
  3924. break;
  3925. if (parentExtra->canMergeLimit(expr, targetClusterType) && !isGrouped(expr) && parentExtra->isBinary())
  3926. {
  3927. if (extra->inherit(*parentExtra))
  3928. {
  3929. if (op == no_choosen)
  3930. {
  3931. extra->hasChoosen = true;
  3932. }
  3933. else
  3934. {
  3935. extra->isLimited = true;
  3936. if (expr->hasAttribute(skipAtom))
  3937. extra->hasSkipLimit = true;
  3938. }
  3939. if (expr->hasAttribute(onFailAtom))
  3940. extra->isCreateRowLimited = true;
  3941. extra->isCloned = cloneRequired;
  3942. }
  3943. }
  3944. break;
  3945. }
  3946. case no_aggregate:
  3947. case no_newusertable:
  3948. case no_newaggregate:
  3949. {
  3950. IHqlExpression * dataset = expr->queryChild(0);
  3951. CompoundSourceInfo * parentExtra = queryBodyExtra(dataset);
  3952. if (isAggregateDataset(expr))
  3953. {
  3954. //Don't yet have csv/xml variants!
  3955. if (!parentExtra->isBinary())
  3956. break;
  3957. //ONFAIL isn't supported for compound aggregates at the moment - although it could be....
  3958. if (parentExtra->hasOnFail)
  3959. break;
  3960. IHqlExpression * root = queryRoot(dataset);
  3961. if (!root || isGrouped(root) || expr->hasAttribute(localAtom))
  3962. break;
  3963. bool isSimpleCountExists = isSimpleCountExistsAggregate(expr, true, false);
  3964. if (parentExtra->isCreateRowLimited)
  3965. break;
  3966. if (parentExtra->hasAnyLimit() && !isSimpleCountExists)
  3967. break;
  3968. //We either have a limit or choosen as the input
  3969. if (parentExtra->isCloned)
  3970. {
  3971. assertex(isSimpleCountExists);
  3972. //Too complicated if the limit is local
  3973. if ((targetClusterType == ThorLCRCluster) && dataset->hasAttribute(localAtom))
  3974. break;
  3975. //CHOOSEN(ds, x, <n>)
  3976. if ((dataset->getOperator() == no_choosen) && queryRealChild(dataset, 2))
  3977. break;
  3978. }
  3979. node_operator newOp = no_none;
  3980. node_operator parentOp = parentExtra->sourceOp;
  3981. if (queryRealChild(expr, 3))
  3982. {
  3983. //Grouped aggregate
  3984. switch (parentOp)
  3985. {
  3986. case no_compound_diskread:
  3987. case no_compound_disknormalize:
  3988. if (flags & CSFnewdisk)
  3989. newOp = no_compound_diskgroupaggregate;
  3990. break;
  3991. case no_compound_indexread:
  3992. case no_compound_indexnormalize:
  3993. if (flags & CSFnewindex)
  3994. newOp = no_compound_indexgroupaggregate;
  3995. break;
  3996. case no_compound_childread:
  3997. case no_compound_childnormalize:
  3998. if (flags & CSFnewchild)
  3999. newOp = no_compound_childgroupaggregate;
  4000. break;
  4001. }
  4002. }
  4003. else
  4004. {
  4005. switch (parentOp)
  4006. {
  4007. case no_compound_diskread:
  4008. case no_compound_disknormalize:
  4009. if (flags & CSFnewdisk)
  4010. {
  4011. newOp = no_compound_diskaggregate;
  4012. if (isSimpleCountExists && !parentExtra->isFiltered && (parentOp == no_compound_diskread))
  4013. {
  4014. IHqlExpression * root = queryRoot(expr);
  4015. if (root)
  4016. {
  4017. if (translator.isFixedRecordSize(root->queryRecord()))
  4018. extra->forceCompound = true;
  4019. }
  4020. }
  4021. }
  4022. break;
  4023. case no_compound_indexread:
  4024. case no_compound_indexnormalize:
  4025. //Don't create counts for (non-keyed) skip limits - little benefit, and could cause problems
  4026. //correctly returning the counts - e.g. especially for exists()
  4027. if ((flags & CSFnewindex) && !parentExtra->hasSkipLimit)
  4028. {
  4029. newOp = no_compound_indexaggregate;
  4030. //Force counts on indexes to become a new compound activity
  4031. //otherwise if(count(x) > n, f(x), g()) will always cause x to be read and spilt.
  4032. //The commented out test would do a better job, but not all keyed filters have an explicit keyed() so it is insufficient
  4033. //
  4034. //Really this should become a count if there are no index reads with the same level of conditionality, or if all accesses
  4035. //are counts.
  4036. //That can be logged as a future enhancement.....
  4037. // if (isSimpleCountExists && !parentExtra->isPostFiltered && (parentOp == no_compound_indexread))
  4038. if (isSimpleCountExists && (parentOp == no_compound_indexread))
  4039. {
  4040. //A skip limit will require everything to be read anyway - so no point splitting in two
  4041. if (!parentExtra->hasSkipLimit)
  4042. extra->forceCompound = true;
  4043. }
  4044. }
  4045. break;
  4046. case no_compound_childread:
  4047. case no_compound_childnormalize:
  4048. if (flags & CSFnewchild)
  4049. newOp = no_compound_childaggregate;
  4050. break;
  4051. case no_compound_inline:
  4052. if (flags & CSFnewinline)
  4053. newOp = no_compound_inline;
  4054. break;
  4055. }
  4056. }
  4057. if (newOp)
  4058. {
  4059. //NB: When creating a limited aggregate, it is ok if the input indicates it is cloned
  4060. //because the new compound count operation will take it into account.
  4061. extra->inherit(*parentExtra, newOp);
  4062. }
  4063. }
  4064. else
  4065. {
  4066. if (!parentExtra->isAggregate())
  4067. extra->inherit(*queryBodyExtra(dataset));
  4068. }
  4069. if (expr->hasAttribute(keyedAtom))
  4070. extra->ensureCompound();
  4071. }
  4072. break;
  4073. case no_filter:
  4074. {
  4075. IHqlExpression * dataset = expr->queryChild(0);
  4076. CompoundSourceInfo * parentExtra = queryBodyExtra(dataset);
  4077. if (!parentExtra->hasAnyLimit() && !parentExtra->isAggregate())
  4078. {
  4079. if (extra->inherit(*parentExtra))
  4080. {
  4081. extra->isFiltered = true;
  4082. if (filterIsKeyed(expr))
  4083. extra->ensureCompound();
  4084. if (filterIsUnkeyed(expr))
  4085. extra->isPostFiltered = true;
  4086. }
  4087. }
  4088. }
  4089. break;
  4090. case no_preload:
  4091. {
  4092. IHqlExpression * dataset = expr->queryChild(0);
  4093. extra->inherit(*queryBodyExtra(dataset));
  4094. extra->isPreloaded = true;
  4095. break;
  4096. }
  4097. case no_sorted:
  4098. case no_preservemeta:
  4099. case no_distributed:
  4100. case no_unordered:
  4101. case no_grouped:
  4102. case no_stepped:
  4103. case no_section:
  4104. case no_sectioninput:
  4105. case no_dataset_alias:
  4106. {
  4107. IHqlExpression * dataset = expr->queryChild(0);
  4108. extra->inherit(*queryBodyExtra(dataset));
  4109. break;
  4110. }
  4111. case no_usertable:
  4112. case no_selectfields:
  4113. UNIMPLEMENTED;
  4114. break;
  4115. case no_addfiles:
  4116. if (canProcessInline(NULL, expr) && (flags & CSFnewinline))
  4117. extra->sourceOp = no_compound_inline;
  4118. break;
  4119. }
  4120. insideCompound = wasInsideCompound;
  4121. }
  4122. void CompoundSourceTransformer::analyseMarkBoundaries(IHqlExpression * expr)
  4123. {
  4124. //This code means that child-query compounds inside a compound aren't yet spotted, they are spotted later.
  4125. if (createCompoundSource(expr))
  4126. {
  4127. queryBodyExtra(expr)->isBoundary = true;
  4128. candidate = true;
  4129. return;
  4130. }
  4131. else if (isCompoundSource(expr))
  4132. return;
  4133. //Might cause problems if Some items are references (e.g., keyed, fetch(0), keyedjoin(1) and don't want translating.
  4134. NewHqlTransformer::analyseExpr(expr);
  4135. }
  4136. void CompoundSourceTransformer::analyseExpr(IHqlExpression * expr)
  4137. {
  4138. if (alreadyVisited(expr->queryBody()))
  4139. {
  4140. if ((pass == 0) && !insideCompound)
  4141. {
  4142. if (!queryBodyExtra(expr)->isNoteUsageFirst())
  4143. return;
  4144. }
  4145. else
  4146. return;
  4147. }
  4148. if (expr->isConstant())
  4149. return;
  4150. switch (pass)
  4151. {
  4152. case 0:
  4153. analyseGatherInfo(expr);
  4154. break;
  4155. case 1:
  4156. analyseMarkBoundaries(expr);
  4157. break;
  4158. default:
  4159. throwUnexpected();
  4160. break;
  4161. }
  4162. }
  4163. bool CompoundSourceTransformer::childrenAreShared(IHqlExpression * expr)
  4164. {
  4165. if (isCompoundSource(expr))
  4166. return false;
  4167. unsigned numChildren = getNumChildTables(expr);
  4168. for (unsigned i=0; i < numChildren; i++)
  4169. {
  4170. IHqlExpression * cur = expr->queryChild(i);
  4171. if (queryBodyExtra(cur)->isShared() || childrenAreShared(cur))
  4172. return true;
  4173. }
  4174. return false;
  4175. }
  4176. bool CompoundSourceTransformer::createCompoundSource(IHqlExpression * expr)
  4177. {
  4178. CompoundSourceInfo * extra = queryBodyExtra(expr);
  4179. if (extra->sourceOp == no_none)
  4180. return false;
  4181. if (extra->forceCompound)
  4182. return true;
  4183. if (expr->getOperator() == no_preservemeta)
  4184. expr = expr->queryChild(0);
  4185. if (isSourceActivity(expr))
  4186. return false;
  4187. if (extra->isPreloaded)
  4188. return (flags & CSFpreload) != 0;
  4189. switch (extra->sourceOp)
  4190. {
  4191. case no_compound_diskread:
  4192. case no_compound_diskaggregate:
  4193. case no_compound_diskcount:
  4194. case no_compound_diskgroupaggregate:
  4195. return ((flags & CSFignoreShared) || !childrenAreShared(expr));
  4196. case no_compound_disknormalize:
  4197. return true;
  4198. case no_compound_indexaggregate:
  4199. case no_compound_indexcount:
  4200. case no_compound_indexgroupaggregate:
  4201. case no_compound_indexread:
  4202. //MORE: Should stop at sufficiently shared children - e.g.,
  4203. //* if the children are aggregates, when we get that far.
  4204. //* if child actions don't change the filter significantly (e.g, just projects, or no seg monitors)
  4205. {
  4206. if (!(flags & CSFindex))
  4207. return false;
  4208. CompoundSourceInfo * parentExtra = queryBodyExtra(expr->queryChild(0));
  4209. return ((flags & CSFignoreShared) || !childrenAreShared(expr) || !parentExtra->isFiltered);
  4210. }
  4211. case no_compound_indexnormalize:
  4212. return ((flags & CSFindex) != 0);
  4213. case no_compound_inline:
  4214. if (!(flags & CSFnewinline))
  4215. return false;
  4216. return !childrenAreShared(expr);
  4217. case no_compound_childread:
  4218. case no_compound_childnormalize:
  4219. case no_compound_childaggregate:
  4220. case no_compound_childcount:
  4221. case no_compound_childgroupaggregate:
  4222. case no_compound_selectnew:
  4223. return true;
  4224. }
  4225. UNIMPLEMENTED;
  4226. return false;
  4227. }
  4228. IHqlExpression * CompoundSourceTransformer::createTransformed(IHqlExpression * expr)
  4229. {
  4230. if (expr->isConstant())
  4231. return LINK(expr);
  4232. OwnedHqlExpr ret = queryTransformAnnotation(expr);
  4233. if (ret)
  4234. return ret.getClear();
  4235. OwnedHqlExpr transformed = NewHqlTransformer::createTransformed(expr);
  4236. CompoundSourceInfo * extra = queryBodyExtra(expr);
  4237. if (extra->isBoundary)
  4238. {
  4239. HqlExprAttr def = transformed;
  4240. if (extra->isCloned)
  4241. transformed.setown(appendLocalAttribute(transformed));
  4242. transformed.setown(createDataset(extra->sourceOp, LINK(transformed), LINK(extra->uid)));
  4243. if (extra->isCloned)
  4244. {
  4245. HqlExprArray args;
  4246. unwindChildren(args, def);
  4247. args.replace(*transformed.getClear(), 0);
  4248. transformed.setown(def->clone(args));
  4249. }
  4250. }
  4251. return transformed.getClear();
  4252. }
  4253. ANewTransformInfo * CompoundSourceTransformer::createTransformInfo(IHqlExpression * expr)
  4254. {
  4255. return CREATE_NEWTRANSFORMINFO(CompoundSourceInfo, expr);
  4256. }
  4257. bool CompoundSourceTransformer::needToCloneLimit(IHqlExpression * expr, node_operator sourceOp)
  4258. {
  4259. node_operator op = expr->getOperator();
  4260. switch (op)
  4261. {
  4262. case no_choosen:
  4263. if (queryRealChild(expr, 2))
  4264. return true;
  4265. break;
  4266. case no_limit:
  4267. if (expr->hasAttribute(skipAtom) && (targetClusterType != RoxieCluster))
  4268. return true;
  4269. break;
  4270. }
  4271. switch (targetClusterType)
  4272. {
  4273. case RoxieCluster:
  4274. return false;
  4275. case HThorCluster:
  4276. return (sourceOp != no_compound_indexread) || (op != no_limit);
  4277. case ThorLCRCluster:
  4278. return true;
  4279. default:
  4280. UNIMPLEMENTED;
  4281. }
  4282. }
  4283. IHqlExpression * CompoundSourceTransformer::process(IHqlExpression * expr)
  4284. {
  4285. analyse(expr, 0);
  4286. analyse(expr, 1);
  4287. if (candidate)
  4288. return transformRoot(expr);
  4289. return LINK(expr);
  4290. }
  4291. //---------------------------------------------------------------------------
  4292. IHqlExpression * getMergedFetch(IHqlExpression * expr)
  4293. {
  4294. IHqlExpression * child = expr->queryChild(0);
  4295. if (isLimitedDataset(child))
  4296. return LINK(expr);
  4297. HqlExprArray args;
  4298. if (child->getOperator() == no_compound_fetch)
  4299. return swapDatasets(expr);
  4300. if (child->getOperator() != no_fetch)
  4301. return LINK(expr);
  4302. args.append(*LINK(expr));
  4303. return createDataset(no_compound_fetch, args);
  4304. }
  4305. static HqlTransformerInfo compoundActivityTransformerInfo("CompoundActivityTransformer");
  4306. CompoundActivityTransformer::CompoundActivityTransformer(ClusterType _targetClusterType) : NewHqlTransformer(compoundActivityTransformerInfo)
  4307. {
  4308. targetClusterType = _targetClusterType;
  4309. }
  4310. IHqlExpression * CompoundActivityTransformer::createTransformed(IHqlExpression * expr)
  4311. {
  4312. OwnedHqlExpr transformed = NewHqlTransformer::createTransformed(expr);
  4313. updateOrphanedSelectors(transformed, expr);
  4314. switch (transformed->getOperator())
  4315. {
  4316. case no_filter:
  4317. return getMergedFetch(transformed);
  4318. case no_limit:
  4319. {
  4320. if (transformed->hasAttribute(onFailAtom))
  4321. break;
  4322. LinkedHqlExpr dataset = transformed->queryChild(0);
  4323. if (dataset->hasAttribute(limitAtom) || transformed->hasAttribute(skipAtom))
  4324. break;
  4325. // A limited KEYED-JOIN should never read more than limit rows from the index for each left row
  4326. // so add a LIMIT attribute onto the keyed join to ensure it doesn't return too many records.
  4327. switch (dataset->getOperator())
  4328. {
  4329. case no_join:
  4330. if (isKeyedJoin(dataset))
  4331. break;
  4332. return transformed.getClear();
  4333. default:
  4334. return transformed.getClear();
  4335. }
  4336. HqlExprArray args;
  4337. unwindChildren(args, transformed);
  4338. args.replace(*mergeLimitIntoDataset(dataset, transformed), 0);
  4339. return transformed->clone(args);
  4340. }
  4341. }
  4342. return transformed.getClear();
  4343. }
  4344. //------------------------------------------------------------------------
  4345. static HqlTransformerInfo optimizeActivityTransformerInfo("OptimizeActivityTransformer");
  4346. OptimizeActivityTransformer::OptimizeActivityTransformer(unsigned _wfid, bool _optimizeCountCompare, bool _optimizeNonEmpty)
  4347. : NewHqlTransformer(optimizeActivityTransformerInfo), wfid(_wfid)
  4348. {
  4349. optimizeCountCompare = _optimizeCountCompare; optimizeNonEmpty = _optimizeNonEmpty;
  4350. }
  4351. void OptimizeActivityTransformer::analyseExpr(IHqlExpression * expr)
  4352. {
  4353. expr = expr->queryBody();
  4354. queryBodyExtra(expr)->noteUsed();
  4355. if (alreadyVisited(expr))
  4356. return;
  4357. NewHqlTransformer::analyseExpr(expr);
  4358. }
  4359. static bool isWorthLimitingDataset(IHqlExpression * ds)
  4360. {
  4361. node_operator dsOp = ds->getOperator();
  4362. switch (dsOp)
  4363. {
  4364. //Any dataset expression which is evaluated in a single go, rather than iterated is likely to be better
  4365. //especially if it is evaluated as an inline operation.
  4366. case no_select:
  4367. case no_choosen:
  4368. case no_rows:
  4369. case no_temptable:
  4370. case no_getresult:
  4371. return false;
  4372. }
  4373. return true;
  4374. }
  4375. //either a simple count, or isCountAggregate is guaranteed to be true - so structure is well defined
  4376. IHqlExpression * OptimizeActivityTransformer::insertChoosen(IHqlExpression * lhs, IHqlExpression * limit, __int64 limitDelta)
  4377. {
  4378. if (isShared(lhs))
  4379. return NULL;
  4380. IHqlExpression * ds = lhs->queryChild(0);
  4381. HqlExprArray args;
  4382. switch (lhs->getOperator())
  4383. {
  4384. case no_choosen:
  4385. return NULL;
  4386. case no_count:
  4387. case no_newaggregate:
  4388. {
  4389. if (!isWorthLimitingDataset(ds))
  4390. return NULL;
  4391. args.append(*createDataset(no_choosen, LINK(ds), adjustValue(limit, limitDelta)));
  4392. break;
  4393. }
  4394. case no_implicitcast:
  4395. case no_cast:
  4396. case no_compound_childaggregate:
  4397. case no_compound_diskaggregate:
  4398. case no_compound_indexaggregate:
  4399. case no_select:
  4400. {
  4401. IHqlExpression * newDs = insertChoosen(ds, limit, limitDelta);
  4402. if (!newDs)
  4403. return NULL;
  4404. args.append(*newDs);
  4405. break;
  4406. }
  4407. default:
  4408. throwUnexpectedOp(lhs->getOperator());
  4409. }
  4410. unwindChildren(args, lhs, 1);
  4411. return lhs->clone(args);
  4412. }
  4413. static bool looksLikeSimpleCount(IHqlExpression * expr)
  4414. {
  4415. if ((expr->getOperator() == no_select) && expr->hasAttribute(newAtom))
  4416. {
  4417. IHqlExpression * ds = expr->queryChild(0);
  4418. return isSimpleCountAggregate(ds, false);
  4419. }
  4420. return (expr->getOperator() == no_count);
  4421. }
  4422. IHqlExpression * OptimizeActivityTransformer::optimizeCompare(IHqlExpression * lhs, IHqlExpression * rhs, node_operator op)
  4423. {
  4424. if (isShared(lhs))
  4425. return NULL;
  4426. if (!isIndependentOfScope(rhs))
  4427. return NULL;
  4428. if (!looksLikeSimpleCount(lhs))
  4429. return NULL;
  4430. // count(x) op count(y) - not clear if a choosen should be added to either, so assume neither for the moment,
  4431. // (we definitely don't want it added to both, which happens without the second test.)
  4432. if (looksLikeSimpleCount(rhs))
  4433. return NULL;
  4434. //Convert count(x) >= 1 to exists(x) (and other varients)
  4435. node_operator existOp = no_none;
  4436. switch (op)
  4437. {
  4438. case no_ne:
  4439. case no_gt:
  4440. if (matchesConstantValue(rhs, 0))
  4441. existOp = no_exists;
  4442. break;
  4443. case no_eq:
  4444. case no_le:
  4445. if (matchesConstantValue(rhs, 0))
  4446. existOp = no_not;
  4447. break;
  4448. case no_lt:
  4449. if (matchesConstantValue(rhs, 1))
  4450. existOp = no_not;
  4451. break;
  4452. case no_ge:
  4453. if (matchesConstantValue(rhs, 1))
  4454. existOp = no_exists;
  4455. break;
  4456. }
  4457. if (existOp != no_none)
  4458. {
  4459. if (lhs->getOperator() == no_count)
  4460. {
  4461. IHqlExpression * ds = lhs->queryChild(0);
  4462. HqlExprArray args;
  4463. unwindChildren(args, lhs);
  4464. OwnedHqlExpr ret = createValue(no_exists, makeBoolType(), args);
  4465. if (existOp == no_not)
  4466. return createValue(no_not, makeBoolType(), ret.getClear());
  4467. return ret.getClear();
  4468. }
  4469. }
  4470. unsigned choosenDelta =0;
  4471. switch (op)
  4472. {
  4473. case no_eq:
  4474. //count(x) == n -> count(choosen(x,n+1)) == n
  4475. choosenDelta = 1;
  4476. break;
  4477. case no_ne:
  4478. //count(x) != 0 -> count(choosen(x,n+1)) != n
  4479. choosenDelta = 1;
  4480. break;
  4481. case no_lt:
  4482. //count(x) < n -> count(choosen(x,n)) < n
  4483. break;
  4484. case no_le:
  4485. //count(x) <= n -> count(choosen(x,n+1)) <= n
  4486. choosenDelta = 1;
  4487. break;
  4488. case no_gt:
  4489. //count(x) > n -> count(choosen(x,n+1)) > n
  4490. choosenDelta = 1;
  4491. break;
  4492. case no_ge:
  4493. //count(x) >= n -> count(choosen(x,n)) >= n
  4494. break;
  4495. }
  4496. IHqlExpression * newLhs = insertChoosen(lhs, rhs, choosenDelta);
  4497. if (!newLhs)
  4498. return NULL;
  4499. return createValue(op, makeBoolType(), newLhs, LINK(rhs));
  4500. }
  4501. static IHqlExpression * queryNormalizedAggregateParameter(IHqlExpression * expr)
  4502. {
  4503. for (;;)
  4504. {
  4505. switch (expr->getOperator())
  4506. {
  4507. case no_choosen:
  4508. if (queryRealChild(expr, 2))
  4509. return expr;
  4510. break;
  4511. case no_sort:
  4512. case no_subsort:
  4513. case no_distribute:
  4514. break;
  4515. default:
  4516. return expr;
  4517. }
  4518. expr = expr->queryChild(0);
  4519. }
  4520. }
  4521. static bool aggregateMatchesDataset(IHqlExpression * agg, IHqlExpression * ds)
  4522. {
  4523. return queryNormalizedAggregateParameter(agg)->queryBody() == queryNormalizedAggregateParameter(ds)->queryBody();
  4524. }
  4525. static bool isCheckExistsAtleast(IHqlExpression * cond, IHqlExpression * ds, __int64 minMinElements, __int64 maxMinElements)
  4526. {
  4527. if (maxMinElements <= 0)
  4528. return false;
  4529. switch (cond->getOperator())
  4530. {
  4531. case no_exists:
  4532. if (aggregateMatchesDataset(cond->queryChild(0), ds))
  4533. return true;
  4534. break;
  4535. case no_ne:
  4536. {
  4537. IHqlExpression * condLhs = cond->queryChild(0);
  4538. if ((condLhs->getOperator() == no_count) && isZero(cond->queryChild(1)) && (minMinElements == 1))
  4539. {
  4540. if (aggregateMatchesDataset(condLhs->queryChild(0), ds))
  4541. return true;
  4542. }
  4543. break;
  4544. }
  4545. case no_gt:
  4546. minMinElements--;
  4547. maxMinElements--;
  4548. //fallthrough
  4549. case no_ge:
  4550. {
  4551. IHqlExpression * condLhs = cond->queryChild(0);
  4552. if (condLhs->getOperator() == no_count)
  4553. {
  4554. IHqlExpression * limit = cond->queryChild(1);
  4555. if (limit->queryValue())
  4556. {
  4557. __int64 limitVal = limit->queryValue()->getIntValue();
  4558. if ((limitVal <= maxMinElements) && (limitVal >= minMinElements))
  4559. {
  4560. if (aggregateMatchesDataset(condLhs->queryChild(0), ds))
  4561. return true;
  4562. }
  4563. }
  4564. }
  4565. }
  4566. break;
  4567. }
  4568. return false;
  4569. }
  4570. //is "value" of the form ds[n].x and other the same as the null expression for that field?
  4571. //if so we may be able to remove a condition
  4572. IHqlExpression * queryNullDsSelect(__int64 & selectIndex, IHqlExpression * value, IHqlExpression * other)
  4573. {
  4574. if (isCast(value))
  4575. value = value->queryChild(0);
  4576. if (value->getOperator() != no_select)
  4577. return NULL;
  4578. bool isNew;
  4579. IHqlExpression * ds = querySelectorDataset(value, isNew);
  4580. if (!isNew || ds->getOperator() != no_selectnth)
  4581. return NULL;
  4582. IValue * index = ds->queryChild(1)->queryValue();
  4583. if (!index)
  4584. return NULL;
  4585. if (!isNullExpr(other, value))
  4586. return NULL;
  4587. selectIndex = index->getIntValue();
  4588. return ds->queryChild(0);
  4589. }
  4590. IHqlExpression * OptimizeActivityTransformer::createTransformed(IHqlExpression * expr)
  4591. {
  4592. OwnedHqlExpr transformed = doCreateTransformed(expr);
  4593. if (transformed)
  4594. {
  4595. assertex(transformed != expr);
  4596. queryBodyExtra(transformed)->inherit(queryBodyExtra(expr));
  4597. return transform(transformed);
  4598. }
  4599. transformed.setown(NewHqlTransformer::createTransformed(expr));
  4600. if (transformed != expr)
  4601. queryBodyExtra(transformed)->inherit(queryBodyExtra(expr));
  4602. return transformed.getClear();
  4603. }
  4604. IHqlExpression * OptimizeActivityTransformer::doCreateTransformed(IHqlExpression * expr)
  4605. {
  4606. node_operator op = expr->getOperator();
  4607. switch (op)
  4608. {
  4609. case no_if:
  4610. {
  4611. IHqlExpression * cond = expr->queryChild(0);
  4612. IHqlExpression * lhs = expr->queryChild(1);
  4613. //convert if(exists(x)|count(x)>0, x, y) to nonempty(x, y);
  4614. //must happen before the count(x)>n optimization below....
  4615. if (expr->isDataset())
  4616. {
  4617. if (isCheckExistsAtleast(cond, lhs, 1, 1))
  4618. {
  4619. IHqlExpression * rhs = expr->queryChild(2);
  4620. //always convert if(exists(x),x) to x regardless of x, or the optimizeNonEmpty option
  4621. if (rhs->getOperator() == no_null)
  4622. return transform(lhs);
  4623. if (optimizeNonEmpty && !canProcessInline(NULL, expr))
  4624. {
  4625. HqlExprArray args;
  4626. args.append(*transform(lhs));
  4627. args.append(*transform(rhs));
  4628. OwnedHqlExpr ret = createDataset(no_nonempty, args);
  4629. return expr->cloneAllAnnotations(ret);
  4630. }
  4631. }
  4632. }
  4633. __int64 selectIndex = 0;
  4634. //check for if (count(x) >= 10, x[10].value, <null>) and convert to x[10].value
  4635. //also valid for count(x) >= 1, but not count(x) >= 11
  4636. IHqlExpression * ds = queryNullDsSelect(selectIndex, expr->queryChild(1), expr->queryChild(2));
  4637. if (ds)
  4638. {
  4639. if (isCheckExistsAtleast(cond, ds, 1, selectIndex))
  4640. return LINK(lhs);
  4641. }
  4642. break;
  4643. }
  4644. case no_selectnth:
  4645. {
  4646. IHqlExpression * ds = expr->queryChild(0);
  4647. if ((ds->getOperator() != no_sort) || isShared(ds))
  4648. break;
  4649. IHqlExpression * index = expr->queryChild(1);
  4650. if (getIntValue(index, 99999) > 100)
  4651. break;
  4652. OwnedHqlExpr transformedDs = transform(ds);
  4653. OwnedHqlExpr transformedIndex = transform(index);
  4654. HqlExprArray args;
  4655. unwindChildren(args, transformedDs);
  4656. args.add(*LINK(transformedIndex), 2);
  4657. OwnedHqlExpr topn = createDataset(no_topn, args);
  4658. args.kill();
  4659. args.append(*ds->cloneAllAnnotations(topn));
  4660. args.append(*LINK(transformedIndex));
  4661. unwindChildren(args, expr, 2);
  4662. return expr->clone(args);
  4663. }
  4664. case no_clustersize:
  4665. if (wfid && !expr->hasAttribute(pureAtom))
  4666. {
  4667. OwnedHqlExpr attribute = createExprAttribute(pureAtom, getSizetConstant(wfid));
  4668. OwnedHqlExpr modified = appendOwnedOperand(expr, attribute.getClear());
  4669. return expr->cloneAllAnnotations(modified);
  4670. }
  4671. break;
  4672. case no_eq:
  4673. case no_ne:
  4674. case no_le:
  4675. case no_lt:
  4676. case no_ge:
  4677. case no_gt:
  4678. //MORE Would still be worth doing for thor i) if a no_select non-new, ii) if the lhs was an aggregate on
  4679. //a compound_disk_aggregate iii) possibly others.
  4680. if (optimizeCountCompare)
  4681. {
  4682. IHqlExpression * lhs = expr->queryChild(0);
  4683. IHqlExpression * rhs = expr->queryChild(1);
  4684. OwnedHqlExpr ret = optimizeCompare(lhs, rhs, op);
  4685. if (!ret)
  4686. ret.setown(optimizeCompare(rhs, lhs, getReverseOp(op)));
  4687. if (ret)
  4688. return ret.getClear();
  4689. }
  4690. break;
  4691. }
  4692. return NULL;
  4693. }
  4694. void optimizeActivities(unsigned wfid, HqlExprArray & exprs, bool optimizeCountCompare, bool optimizeNonEmpty)
  4695. {
  4696. OptimizeActivityTransformer transformer(wfid, optimizeCountCompare, optimizeNonEmpty);
  4697. transformer.analyseArray(exprs, 0);
  4698. transformer.transformRoot(exprs);
  4699. }
  4700. IHqlExpression * GlobalAttributeInfo::queryAlias(IHqlExpression * value)
  4701. {
  4702. if (!aliasName)
  4703. {
  4704. if (storedName)
  4705. aliasName.set(storedName);
  4706. else
  4707. aliasName.setown(createNextStringValue(value, storedPrefix));
  4708. }
  4709. return aliasName;
  4710. }
  4711. IHqlExpression * GlobalAttributeInfo::queryFilename(IHqlExpression * value, IConstWorkUnit * wu, bool isRoxie)
  4712. {
  4713. if (!cachedFilename)
  4714. {
  4715. if (storedName)
  4716. cachedFilename.set(storedName);
  4717. else
  4718. cachedFilename.setown(createNextStringValue(value, filePrefix));
  4719. if (persistOp != no_persist)
  4720. {
  4721. StringBuffer prefix("~");
  4722. if (storedName)
  4723. {
  4724. if (persistOp == no_stored)
  4725. prefix.append("jobtemp::stored");
  4726. else if (persistOp == no_checkpoint)
  4727. prefix.append("jobtemp::checkpoint");
  4728. }
  4729. if (persistOp == no_once)
  4730. prefix.append("once::");
  4731. bool wuidIsConstant = isRoxie || !wu->getCloneable();
  4732. if (wuidIsConstant)
  4733. {
  4734. StringBuffer s;
  4735. cachedFilename->queryValue()->getStringValue(s.append(prefix));
  4736. cachedFilename.setown(createConstant(s.str()));
  4737. }
  4738. else
  4739. {
  4740. ITypeInfo * type = makeStringType(UNKNOWN_LENGTH, NULL, NULL);
  4741. OwnedHqlExpr filename = createValue(no_concat, type, createConstant(prefix), cachedFilename.getClear());
  4742. cachedFilename.setown(foldHqlExpression(filename));
  4743. }
  4744. }
  4745. }
  4746. return cachedFilename;
  4747. }
  4748. IHqlExpression * GlobalAttributeInfo::createSetValue(IHqlExpression * value, IHqlExpression * name)
  4749. {
  4750. HqlExprArray args;
  4751. args.append(*LINK(value));
  4752. args.append(*createAttribute(sequenceAtom, LINK(sequence)));
  4753. args.append(*createAttribute(namedAtom, LINK(name)));
  4754. if (extraSetAttr)
  4755. extraSetAttr->unwindList(args, no_comma);
  4756. if (cluster)
  4757. args.append(*createAttribute(clusterAtom, LINK(cluster)));
  4758. if (fieldFormat)
  4759. args.append(*LINK(fieldFormat));
  4760. if (setOp == no_setresult)
  4761. return createSetResult(args);
  4762. return createValue(setOp, makeVoidType(), args);
  4763. }
  4764. IHqlExpression * GlobalAttributeInfo::getStoredKey()
  4765. {
  4766. return createAttribute(nameAtom, LINK(sequence), getLowerCaseConstantExpr(originalLabel));
  4767. }
  4768. void GlobalAttributeInfo::setCluster(IHqlExpression * expr)
  4769. {
  4770. if (expr && !isBlankString(expr))
  4771. cluster.set(expr);
  4772. }
  4773. void GlobalAttributeInfo::extractGlobal(IHqlExpression * global, ClusterType platform)
  4774. {
  4775. few = spillToWorkunitNotFile(value, platform) || value->isDictionary();
  4776. if (global)
  4777. {
  4778. if (global->hasAttribute(fewAtom))
  4779. few = true;
  4780. else if (global->hasAttribute(manyAtom) && (platform != RoxieCluster))
  4781. few = false;
  4782. }
  4783. setOp = no_setresult;
  4784. sequence.setown(getLocalSequenceNumber());
  4785. persistOp = no_global;
  4786. }
  4787. void GlobalAttributeInfo::extractStoredInfo(IHqlExpression * expr, const char * id, IHqlExpression * _codehash, bool isRoxie, int multiplePersistInstances)
  4788. {
  4789. node_operator op = expr->getOperator();
  4790. few = expr->hasAttribute(fewAtom) || (isRoxie) || (value->isDictionary() && !expr->hasAttribute(manyAtom));
  4791. getUTF8Value(label, queryAttributeChild(expr, labelAtom, 0), nullptr);
  4792. switch (op)
  4793. {
  4794. case no_stored:
  4795. setOp = no_ensureresult;
  4796. storedName.set(expr->queryChild(0));
  4797. originalLabel.set(storedName);
  4798. sequence.setown(getStoredSequenceNumber());
  4799. fieldFormat.set(expr->queryAttribute(storedFieldFormatAtom));
  4800. few = true;
  4801. break;
  4802. case no_checkpoint:
  4803. setOp = no_ensureresult;
  4804. storedName.set(expr->queryChild(0));
  4805. originalLabel.set(storedName);
  4806. sequence.setown(getLocalSequenceNumber());
  4807. extraSetAttr.setown(createAttribute(checkpointAtom));
  4808. break;
  4809. case no_persist:
  4810. assertex(_codehash);
  4811. codehash.set(_codehash);
  4812. setOp = no_ensureresult;
  4813. storedName.set(expr->queryChild(0));
  4814. originalLabel.set(storedName);
  4815. sequence.setown(getGlobalSequenceNumber());
  4816. extraSetAttr.setown(createAttribute(_workflowPersist_Atom, LINK(codehash)));
  4817. setCluster(queryRealChild(expr, 1));
  4818. few = expr->hasAttribute(fewAtom); // PERSISTs need a consistent format.
  4819. extraOutputAttr.setown(createComma(LINK(expr->queryAttribute(expireAtom)), LINK(expr->queryAttribute(clusterAtom))));
  4820. numPersistInstances = multiplePersistInstances;
  4821. if (expr->hasAttribute(multipleAtom))
  4822. numPersistInstances = (int)getIntValue(queryAttributeChild(expr, multipleAtom, 0), -1);
  4823. else if (expr->hasAttribute(singleAtom))
  4824. numPersistInstances = 0;
  4825. if (numPersistInstances != 0)
  4826. {
  4827. StringBuffer s;
  4828. getStringValue(s, storedName);
  4829. s.append("__p");
  4830. getStringValue(s, codehash);
  4831. storedName.setown(createConstant(s.str()));
  4832. }
  4833. persistRefresh = getBoolValue(queryAttributeChild(expr, refreshAtom, 0), true);
  4834. break;
  4835. case no_critical:
  4836. setOp = no_setresult;
  4837. storedName.set(expr->queryChild(0));
  4838. originalLabel.set(storedName);
  4839. sequence.setown(getLocalSequenceNumber());
  4840. extraSetAttr.setown(createAttribute(_workflow_Atom));
  4841. break;
  4842. case no_global:
  4843. throwUnexpected();
  4844. case no_independent:
  4845. setOp = no_setresult;
  4846. storedName.clear();
  4847. sequence.setown(getLocalSequenceNumber());
  4848. extraSetAttr.setown(createAttribute(_workflow_Atom));
  4849. setCluster(queryRealChild(expr, 0));
  4850. op = no_global;
  4851. break;
  4852. case no_once:
  4853. setOp = no_setresult;
  4854. storedName.clear();
  4855. sequence.setown(getOnceSequenceNumber());
  4856. extraSetAttr.setown(createAttribute(_workflow_Atom));
  4857. break;
  4858. case no_success:
  4859. case no_failure:
  4860. case no_recovery:
  4861. if(setOp == no_none)
  4862. {
  4863. storedName.clear();
  4864. setOp = no_setresult;
  4865. sequence.setown(getLocalSequenceNumber());
  4866. }
  4867. break;
  4868. default:
  4869. return;
  4870. }
  4871. if (!label)
  4872. {
  4873. switch (op)
  4874. {
  4875. case no_stored:
  4876. case no_persist:
  4877. label.append("Create ");
  4878. getStoredDescription(label, sequence, storedName, true);
  4879. break;
  4880. case no_critical:
  4881. label.append("CRITICAL(");
  4882. getStringValue(label, storedName);
  4883. label.append(")");
  4884. break;
  4885. default:
  4886. label.append(id);
  4887. break;
  4888. }
  4889. }
  4890. persistOp = op;
  4891. }
  4892. void GlobalAttributeInfo::splitGlobalDefinition(ITypeInfo * type, IHqlExpression * value, IConstWorkUnit * wu, SharedHqlExpr & setOutput, OwnedHqlExpr * getOutput, bool isRoxie)
  4893. {
  4894. doSplitGlobalDefinition(type, value, wu, setOutput, getOutput, isRoxie);
  4895. }
  4896. void GlobalAttributeInfo::doSplitGlobalDefinition(ITypeInfo * type, IHqlExpression * value, IConstWorkUnit * wu, SharedHqlExpr & setOutput, OwnedHqlExpr * getOutput, bool isRoxie)
  4897. {
  4898. OwnedHqlExpr targetName;
  4899. if (storedName)
  4900. targetName.set(storedName);
  4901. else
  4902. targetName.setown(createNextStringValue(value));
  4903. ITypeInfo * valueType = value->queryType();
  4904. if (value->isDataset() || value->isDictionary())
  4905. {
  4906. if (few)
  4907. {
  4908. splitSmallDataset(value, setOutput, getOutput);
  4909. return;
  4910. }
  4911. LinkedHqlExpr filename = queryFilename(value, wu, isRoxie);
  4912. HqlExprArray args;
  4913. if (value->isDictionary())
  4914. args.append(*createDataset(no_datasetfromdictionary, LINK(value)));
  4915. else
  4916. args.append(*LINK(value));
  4917. args.append(*LINK(filename));
  4918. //NB: Also update the dataset node at the end...
  4919. if (valueType->getTypeCode() == type_groupedtable)
  4920. args.append(*createAttribute(groupedAtom));
  4921. else
  4922. assertex(!isGrouped(valueType));
  4923. bool compressFile = true;
  4924. switch (persistOp)
  4925. {
  4926. case no_persist:
  4927. {
  4928. args.append(*createAttribute(_workflowPersist_Atom));
  4929. args.append(*createAttribute(sequenceAtom, getGlobalSequenceNumber()));
  4930. //add a flag to help get the resourcing right - may need to hash distribute on different size thor
  4931. IHqlExpression * distribution = queryDistribution(value);
  4932. if (distribution && !distribution->isAttribute())
  4933. args.append(*createAttribute(distributedAtom));
  4934. break;
  4935. }
  4936. case no_stored:
  4937. args.append(*createAttribute(ownedAtom));
  4938. args.append(*createAttribute(sequenceAtom, getStoredSequenceNumber()));
  4939. break;
  4940. case no_checkpoint:
  4941. args.append(*createAttribute(ownedAtom));
  4942. args.append(*createAttribute(sequenceAtom, getLocalSequenceNumber()));
  4943. break;
  4944. case no_once:
  4945. args.append(*createAttribute(ownedAtom));
  4946. args.append(*createAttribute(sequenceAtom, getOnceSequenceNumber()));
  4947. break;
  4948. case no_global:
  4949. //May extend over several different graphs
  4950. args.append(*createAttribute(sequenceAtom, getLocalSequenceNumber()));
  4951. args.append(*createAttribute(ownedAtom));
  4952. args.append(*createAttribute(jobTempAtom));
  4953. break;
  4954. default:
  4955. //global, independent, success, failure, etc. etc.
  4956. args.append(*createAttribute(ownedAtom));
  4957. args.append(*createAttribute(jobTempAtom));
  4958. args.append(*createAttribute(sequenceAtom, getLocalSequenceNumber()));
  4959. break;
  4960. }
  4961. if (compressFile)
  4962. args.append(*createAttribute(__compressed__Atom));
  4963. args.append(*createAttribute(overwriteAtom));
  4964. if (extraOutputAttr)
  4965. extraOutputAttr->unwindList(args, no_comma);
  4966. OwnedHqlExpr output = createValue(no_output, makeVoidType(), args);
  4967. // if (persistOp == no_independent)
  4968. if (setOp == no_setresult)
  4969. setOutput.set(output);
  4970. else
  4971. setOutput.setown(createSetValue(output, queryAlias(value)));
  4972. if(getOutput)
  4973. {
  4974. IHqlExpression * record = value->queryRecord();
  4975. args.kill();
  4976. args.append(*LINK(filename));
  4977. args.append(*LINK(record));
  4978. args.append(*createValue(no_thor));
  4979. args.append(*createAttribute(_noVirtual_Atom)); // don't interpret virtual fields in spilled output
  4980. args.append(*createExprAttribute(_signed_Atom, createConstant("hpcc")));
  4981. if (persistOp == no_persist)
  4982. args.append(*createAttribute(_workflowPersist_Atom));
  4983. if (isGrouped(value))
  4984. args.append(*createAttribute(groupedAtom));
  4985. if (compressFile)
  4986. args.append(*createAttribute(__compressed__Atom));
  4987. if (hasSingleRow(value))
  4988. args.append(*createAttribute(rowAtom));
  4989. if (output->hasAttribute(jobTempAtom))
  4990. args.append(*createAttribute(jobTempAtom));
  4991. if (persistOp != no_stored)
  4992. {
  4993. IHqlExpression * recordCountAttr = queryRecordCountInfo(value);
  4994. if (recordCountAttr)
  4995. args.append(*LINK(recordCountAttr));
  4996. }
  4997. OwnedHqlExpr getValue = createDataset(no_table, args);
  4998. //getValue.setown(cloneInheritedAnnotations(value, getValue));
  4999. if (persistOp != no_stored)
  5000. {
  5001. bool clusterSizeMayChange = (persistOp == no_persist) || (cluster != nullptr);
  5002. getValue.setown(preserveTableInfo(getValue, value, false, clusterSizeMayChange));
  5003. }
  5004. //Note: getValue->queryType() != valueType because the dataset used for field resolution has changed...
  5005. if (value->isDictionary())
  5006. getValue.setown(createDictionary(no_createdictionary, getValue.getClear()));
  5007. getOutput->setown(getValue.getClear());
  5008. }
  5009. }
  5010. else if (type->getTypeCode() == type_void)
  5011. {
  5012. switch (persistOp)
  5013. {
  5014. case no_stored:
  5015. case no_checkpoint:
  5016. case no_once:
  5017. case no_persist:
  5018. setOutput.setown(createSetValue(value, queryAlias(value)));
  5019. break;
  5020. default:
  5021. setOutput.set(value);
  5022. break;
  5023. }
  5024. if(getOutput) getOutput->setown(createValue(no_null, makeVoidType(), createAttribute(_internal_Atom, LINK(sequence), LINK(queryAlias(value)))));
  5025. }
  5026. else
  5027. {
  5028. if (type->getTypeCode() == type_set)
  5029. extraSetAttr.setown(createComma(extraSetAttr.getClear(), createAttribute(_original_Atom, createValue(no_implicitcast, LINK(type), LINK(value)))));
  5030. setOutput.setown(createSetValue(value, queryAlias(value)));
  5031. if(getOutput) getOutput->setown(createGetResultFromSetResult(setOutput, type));
  5032. }
  5033. }
  5034. void GlobalAttributeInfo::createSmallOutput(IHqlExpression * value, SharedHqlExpr & setOutput)
  5035. {
  5036. if (value->getOperator() == no_temptable)
  5037. {
  5038. IHqlExpression * values = value->queryChild(0);
  5039. if ((values->getOperator() == no_null) ||
  5040. ((values->getOperator() == no_list) && (values->numChildren() == 0)))
  5041. {
  5042. OwnedHqlExpr newNull = createDataset(no_null, LINK(value->queryRecord()));
  5043. setOutput.setown(createSetValue(newNull, queryAlias(value)));
  5044. return;
  5045. }
  5046. else if (values->getOperator() == no_all)
  5047. {
  5048. OwnedHqlExpr newAll = createDataset(no_all, LINK(value->queryRecord()));
  5049. setOutput.setown(createSetValue(newAll, queryAlias(value)));
  5050. return;
  5051. }
  5052. }
  5053. // else if (value->getOperator() == no_null)
  5054. // {
  5055. // setOutput.setown(createSetValue(value, queryAlias()));
  5056. // return;
  5057. // }
  5058. HqlExprArray args;
  5059. args.append(*LINK(value));
  5060. args.append(*createAttribute(sequenceAtom, LINK(sequence)));
  5061. args.append(*createAttribute(namedAtom, LINK(queryAlias(value))));
  5062. if (isGrouped(value))
  5063. args.append(*createAttribute(groupedAtom));
  5064. setOutput.setown(createValue(no_output, makeVoidType(), args));
  5065. if (setOp != no_setresult)
  5066. {
  5067. extraSetAttr.setown(createComma(LINK(extraSetAttr), createAttribute(noSetAtom)));
  5068. setOutput.setown(createSetValue(setOutput, queryAlias(value)));
  5069. }
  5070. }
  5071. void GlobalAttributeInfo::checkFew(HqlCppTranslator & translator)
  5072. {
  5073. // if (few && isGrouped(value))
  5074. // translator.WARNINGAT(queryLocation(value), HQLWRN_GroupedGlobalFew);
  5075. }
  5076. void GlobalAttributeInfo::splitSmallDataset(IHqlExpression * value, SharedHqlExpr & setOutput, OwnedHqlExpr * getOutput)
  5077. {
  5078. createSmallOutput(value, setOutput);
  5079. if(getOutput)
  5080. {
  5081. IHqlExpression * record = value->queryRecord();
  5082. HqlExprArray args;
  5083. args.append(*LINK(record));
  5084. args.append(*createAttribute(nameAtom, LINK(queryAlias(value))));
  5085. args.append(*createAttribute(sequenceAtom, LINK(sequence)));
  5086. if (isGrouped(value))
  5087. args.append(*createAttribute(groupedAtom));
  5088. if (persistOp != no_stored)
  5089. {
  5090. IHqlExpression * recordCountAttr = queryRecordCountInfo(value);
  5091. if (recordCountAttr)
  5092. args.append(*LINK(recordCountAttr));
  5093. }
  5094. OwnedHqlExpr wuRead = value->isDictionary() ? createDictionary(no_workunit_dataset, args) : createDataset(no_workunit_dataset, args);
  5095. //wuRead.setown(cloneInheritedAnnotations(value, wuRead));
  5096. if (persistOp != no_stored)
  5097. getOutput->setown(preserveTableInfo(wuRead, value, true, false));
  5098. else
  5099. getOutput->set(wuRead);
  5100. }
  5101. }
  5102. //------------------------------------------------------------------------
  5103. static bool isStored(IHqlExpression * set)
  5104. {
  5105. switch (set->getOperator())
  5106. {
  5107. case no_setresult:
  5108. case no_ensureresult:
  5109. case no_output:
  5110. return matchesConstantValue(queryAttributeChild(set, sequenceAtom, 0), ResultSequenceStored);
  5111. }
  5112. return false;
  5113. }
  5114. static bool isTrivialStored(IHqlExpression * set)
  5115. {
  5116. switch (set->getOperator())
  5117. {
  5118. case no_setresult:
  5119. case no_ensureresult:
  5120. if (matchesConstantValue(queryAttributeChild(set, sequenceAtom, 0), ResultSequenceStored))
  5121. {
  5122. IHqlExpression * value = set->queryChild(0);
  5123. for (;;)
  5124. {
  5125. switch (value->getOperator())
  5126. {
  5127. case no_constant:
  5128. case no_all:
  5129. case no_null:
  5130. return true;
  5131. case no_list:
  5132. return (value->numChildren() == 0);
  5133. case no_cast:
  5134. case no_implicitcast:
  5135. value = value->queryChild(0);
  5136. break;
  5137. case no_output:
  5138. return isTrivialInlineOutput(value);
  5139. default:
  5140. return false;
  5141. }
  5142. }
  5143. }
  5144. break;
  5145. case no_output:
  5146. return isTrivialInlineOutput(set);
  5147. }
  5148. return false;
  5149. }
  5150. inline bool isWorkflowAction(IHqlExpression * expr)
  5151. {
  5152. return expr && (expr->getOperator() == no_workflow_action);
  5153. }
  5154. void cloneDependencies(UnsignedArray & tgt, const UnsignedArray & src)
  5155. {
  5156. ForEachItemIn(i, src)
  5157. tgt.append(src.item(i));
  5158. }
  5159. inline bool addDependency(UnsignedArray & tgt, unsigned wfid)
  5160. {
  5161. if (!tgt.contains(wfid))
  5162. {
  5163. tgt.append(wfid);
  5164. return true;
  5165. }
  5166. return false;
  5167. }
  5168. void inheritDependencies(UnsignedArray & tgt, const UnsignedArray & src)
  5169. {
  5170. ForEachItemIn(i, src)
  5171. addDependency(tgt, src.item(i));
  5172. }
  5173. bool hasSameDependencies(UnsignedArray const & d1, UnsignedArray const & d2)
  5174. {
  5175. if (d1.ordinality() != d2.ordinality())
  5176. return false;
  5177. ForEachItemIn(i, d2)
  5178. {
  5179. if (d1.find(d2.item(i)) == NotFound)
  5180. return false;
  5181. }
  5182. return true;
  5183. }
  5184. bool hasExtraDependencies(UnsignedArray const & p, UnsignedArray const & n, UnsignedArray const & ignore)
  5185. {
  5186. if (n.ordinality() > p.ordinality() + ignore.ordinality())
  5187. return true;
  5188. ForEachItemIn(i, n)
  5189. {
  5190. unsigned cur = n.item(i);
  5191. if (!p.contains(cur) && !ignore.contains(cur))
  5192. return true;
  5193. }
  5194. return false;
  5195. }
  5196. void diffDependencies(UnsignedArray & target, UnsignedArray const & d1, UnsignedArray const & d2)
  5197. {
  5198. ForEachItemIn(i, d1)
  5199. {
  5200. unsigned cur = d1.item(i);
  5201. if (d2.find(cur) == NotFound)
  5202. addDependency(target, cur);
  5203. }
  5204. ForEachItemIn(j, d2)
  5205. {
  5206. unsigned cur = d2.item(j);
  5207. if (d1.find(cur) == NotFound)
  5208. addDependency(target, cur);
  5209. }
  5210. }
  5211. void intersectDependencies(UnsignedArray & target, UnsignedArray const & d1, UnsignedArray const & d2)
  5212. {
  5213. ForEachItemIn(i, d1)
  5214. {
  5215. unsigned cur = d1.item(i);
  5216. if (d2.find(cur) != NotFound)
  5217. addDependency(target, cur);
  5218. }
  5219. }
  5220. static IHqlExpression * createResultsAttribute(const HqlExprArray & results)
  5221. {
  5222. HqlExprArray globalResults;
  5223. ForEachItemIn(i, results)
  5224. {
  5225. IHqlExpression & cur = results.item(i);
  5226. //Only include global results - it may be possible to have a result read in a child query, from a workunit with an unknown wuid.
  5227. if (cur.isIndependentOfScope())
  5228. globalResults.append(OLINK(cur));
  5229. }
  5230. return createExprAttribute(_results_Atom, globalResults);
  5231. }
  5232. //------------------------------------------------------------------------
  5233. static HqlTransformerInfo workflowTransformerInfo("WorkflowTransformer");
  5234. WorkflowTransformer::WorkflowTransformer(IWorkUnit * _wu, HqlCppTranslator & _translator)
  5235. : NewHqlTransformer(workflowTransformerInfo), wu(_wu), translator(_translator), wfidCount(0)
  5236. {
  5237. const HqlCppOptions & options = translator.queryOptions();
  5238. trivialStoredWfid = 0;
  5239. nextInternalFunctionId = 0;
  5240. onceWfid = 0;
  5241. combineAllStored = options.combineAllStored;
  5242. combineTrivialStored = options.combineTrivialStored;
  5243. expandPersistInputDependencies = options.expandPersistInputDependencies;
  5244. multiplePersistInstances = options.multiplePersistInstances ? options.defaultNumPersistInstances : 0;
  5245. isRootAction = true;
  5246. isRoxie = (translator.getTargetClusterType() == RoxieCluster);
  5247. isConditional = false;
  5248. insideStored = false;
  5249. activeWfid = 0;
  5250. }
  5251. //-- Helper routines --
  5252. IWorkflowItem * WorkflowTransformer::addWorkflowToWorkunit(unsigned wfid, WFType type, WFMode mode, UnsignedArray const & dependencies, ContingencyData const & conts, IHqlExpression * cluster, const char * label)
  5253. {
  5254. Owned<IWorkflowItem> wf(wu->addWorkflowItem(wfid, type, mode, conts.success, conts.failure, conts.recovery, conts.retries, conts.contingencyFor));
  5255. if (cluster)
  5256. {
  5257. StringBuffer clusterText;
  5258. getStringValue(clusterText, cluster);
  5259. wf->setCluster(clusterText);
  5260. }
  5261. if (label && *label)
  5262. wf->setLabel(label);
  5263. ForEachItemIn(idx, dependencies)
  5264. wf->addDependency(dependencies.item(idx));
  5265. return wf.getClear();
  5266. }
  5267. void WorkflowTransformer::setWorkflowSchedule(IWorkflowItem * wf, const ScheduleData & sched)
  5268. {
  5269. if(sched.now)
  5270. {
  5271. wf->setScheduledNow();
  5272. }
  5273. else
  5274. {
  5275. wu->incEventScheduledCount();
  5276. wf->setScheduledOn(sched.eventName.str(), sched.eventText.str());
  5277. if(sched.counting)
  5278. {
  5279. wf->setScheduleCount(sched.count);
  5280. if (sched.count == 0)
  5281. wf->setState(WFStateDone);
  5282. }
  5283. }
  5284. int priority = sched.priority;
  5285. if(priority > 100) priority = 100;
  5286. if(priority < 0) priority = 0;
  5287. wf->setSchedulePriority(priority);
  5288. }
  5289. void WorkflowTransformer::setWorkflowPersist(IWorkflowItem * wf, char const * persistName, unsigned persistWfid, int numPersistInstances, bool refresh)
  5290. {
  5291. wf->setPersistInfo(persistName, persistWfid, numPersistInstances, refresh);
  5292. }
  5293. void WorkflowTransformer::setWorkflowCritical(IWorkflowItem * wf, char const * criticalName)
  5294. {
  5295. wf->setCriticalInfo(criticalName);
  5296. }
  5297. WorkflowItem * WorkflowTransformer::createWorkflowItem(IHqlExpression * expr, unsigned wfid, node_operator workflowOp, const char * label)
  5298. {
  5299. WorkflowItem * item = new WorkflowItem(wfid, workflowOp, label);
  5300. expr->unwindList(item->queryExprs(), no_comma);
  5301. gatherIndirectDependencies(item->dependencies, expr);
  5302. return item;
  5303. }
  5304. IWorkflowItem * WorkflowTransformer::lookupWorkflowItem(unsigned wfid)
  5305. {
  5306. Owned<IWorkflowItemIterator> iter = wu->updateWorkflowItems();
  5307. ForEach(*iter)
  5308. {
  5309. Owned<IWorkflowItem> cur = iter->get();
  5310. if (cur->queryWfid() == wfid)
  5311. return cur.getClear();
  5312. }
  5313. return NULL;
  5314. }
  5315. bool WorkflowTransformer::hasStoredDependencies(IHqlExpression * expr)
  5316. {
  5317. return false;
  5318. }
  5319. void WorkflowTransformer::inheritDependencies(IHqlExpression * expr)
  5320. {
  5321. ForEachChild(i, expr)
  5322. copyDependencies(queryBodyExtra(expr->queryChild(i)), queryBodyExtra(expr));
  5323. }
  5324. void WorkflowTransformer::copyDependencies(WorkflowTransformInfo * source, WorkflowTransformInfo * dest)
  5325. {
  5326. if(!source) return;
  5327. UnsignedArray const & dependencies = source->queryDependencies();
  5328. ForEachItemIn(idx, dependencies)
  5329. dest->addDependency(dependencies.item(idx));
  5330. }
  5331. void WorkflowTransformer::copySetValueDependencies(WorkflowTransformInfo * source, IHqlExpression * expr)
  5332. {
  5333. node_operator op = expr->getOperator();
  5334. if (op == no_compound || op==no_actionlist)
  5335. {
  5336. copySetValueDependencies(source, expr->queryChild(expr->numChildren()-1));
  5337. inheritDependencies(expr);
  5338. }
  5339. else
  5340. copyDependencies(source, queryBodyExtra(expr));
  5341. }
  5342. unsigned WorkflowTransformer::ensureWorkflowAction(IHqlExpression * expr)
  5343. {
  5344. if (isWorkflowAction(expr))
  5345. return (unsigned)getIntValue(expr->queryChild(0));
  5346. unsigned wfid = ++wfidCount;
  5347. const char * label = str(expr->queryId());
  5348. Owned<IWorkflowItem> wf = addWorkflowToWorkunit(wfid, WFTypeNormal, WFModeNormal, queryDirectDependencies(expr), rootCluster, label);
  5349. addWorkflowItem(*createWorkflowItem(expr, wfid, no_actionlist, label));
  5350. return wfid;
  5351. }
  5352. //-- first pass - extracting workflow
  5353. unsigned WorkflowTransformer::splitValue(IHqlExpression * value)
  5354. {
  5355. GlobalAttributeInfo info("jobtemp::wf", "wf", value);
  5356. info.sequence.setown(getLocalSequenceNumber());
  5357. info.setOp = no_setresult;
  5358. info.persistOp = no_global;
  5359. OwnedHqlExpr setValue;
  5360. info.checkFew(translator);
  5361. info.splitGlobalDefinition(value->queryType(), value, wu, setValue, 0, (translator.getTargetClusterType() == RoxieCluster));
  5362. inheritDependencies(setValue);
  5363. unsigned wfid = ++wfidCount;
  5364. addWorkflowItem(*createWorkflowItem(setValue, wfid, no_global, nullptr));
  5365. return wfid;
  5366. }
  5367. WorkflowItem * WorkflowTransformer::findWorkflowItem(unsigned wfid)
  5368. {
  5369. ForEachItemIn(i, workflow)
  5370. {
  5371. WorkflowItem & cur = workflow.item(i);
  5372. if (cur.wfid == wfid)
  5373. return &cur;
  5374. }
  5375. return NULL;
  5376. }
  5377. void WorkflowTransformer::extractDependentInputs(UnsignedArray & visited, DependenciesUsed & dependencies, const UnsignedArray & wfids)
  5378. {
  5379. ForEachItemIn(i, wfids)
  5380. {
  5381. unsigned wfid = wfids.item(i);
  5382. if (wfid == trivialStoredWfid)
  5383. continue;
  5384. if (visited.contains(wfid))
  5385. continue;
  5386. visited.append(wfid);
  5387. const WorkflowItem * match = findWorkflowItem(wfid);
  5388. assertex(match);
  5389. switch (match->workflowOp)
  5390. {
  5391. case no_persist:
  5392. if (expandPersistInputDependencies)
  5393. break;
  5394. continue;
  5395. case no_stored:
  5396. continue;
  5397. }
  5398. extractDependentInputs(visited, dependencies, match->dependencies);
  5399. ForEachItemIn(iExpr, match->exprs)
  5400. gatherDependencies(&match->exprs.item(iExpr), dependencies, GatherAll);
  5401. }
  5402. }
  5403. IHqlExpression * WorkflowTransformer::extractWorkflow(IHqlExpression * untransformed, IHqlExpression * expr)
  5404. {
  5405. const char * id = nullptr;
  5406. ForEachItemInRev(i, activeLocations)
  5407. {
  5408. IHqlExpression & next = activeLocations.item(i);
  5409. if (next.queryBody() != untransformed)
  5410. break;
  5411. if (next.queryId())
  5412. {
  5413. id = str(next.queryId());
  5414. break;
  5415. }
  5416. }
  5417. IHqlExpression * value = expr->queryChild(0);
  5418. GlobalAttributeInfo info("jobtemp::wf", "wf", value);
  5419. info.sequence.setown(getLocalSequenceNumber());
  5420. OwnedHqlExpr scheduleActions;
  5421. HqlExprArray actions;
  5422. unwindChildren(actions, expr, 1);
  5423. IHqlExpression * originalAttr = queryAttribute(_original_Atom, actions);
  5424. OwnedHqlExpr codehash;
  5425. if (originalAttr)
  5426. {
  5427. unsigned crc = getExpressionCRC(originalAttr->queryChild(0)) + PERSIST_VERSION;
  5428. codehash.setown(getSizetConstant(crc));
  5429. }
  5430. //First check for duplicate expressions, and cope with the weird case where they are identical except for the annotations.
  5431. //Do it before wfid is allocated to make life simpler
  5432. ForEachItemIn(iCheck, actions)
  5433. {
  5434. IHqlExpression & cur = actions.item(iCheck);
  5435. node_operator curOp = cur.getOperator();
  5436. switch (curOp)
  5437. {
  5438. case no_persist:
  5439. case no_checkpoint:
  5440. case no_stored:
  5441. info.extractStoredInfo(&cur, id, codehash, isRoxie, multiplePersistInstances);
  5442. OwnedHqlExpr id = info.getStoredKey();
  5443. unsigned match = alreadyProcessed.find(*id);
  5444. if (match == NotFound)
  5445. break;
  5446. //Compare the definitions - not the expressions, otherwise the original attribute can create false negatives
  5447. IHqlExpression * prevValue = alreadyProcessedExpr.item(match).queryChild(0);
  5448. if(prevValue->queryBody() != value->queryBody())
  5449. {
  5450. StringBuffer s;
  5451. getStoredDescription(s, info.sequence, info.originalLabel, true);
  5452. if(prevValue->queryType() != value->queryBody()->queryType())
  5453. {
  5454. #ifdef _DEBUG
  5455. debugFindFirstDifference(alreadyProcessedExpr.item(match).queryBody(), expr->queryBody());
  5456. #endif
  5457. if (curOp == no_stored)
  5458. throwError1(HQLERR_DuplicateStoredDiffType, s.str());
  5459. else
  5460. throwError1(HQLERR_DuplicateDefinitionDiffType, s.str());
  5461. }
  5462. else if (translator.queryOptions().allowStoredDuplicate) // only here as a temporary workaround
  5463. translator.reportWarning(CategoryMistake, SeverityUnknown, queryActiveLocation(expr), HQLERR_DuplicateDefinition, HQLERR_DuplicateDefinition_Text, s.str());
  5464. else
  5465. {
  5466. if (queryLocationIndependent(prevValue) != queryLocationIndependent(value))
  5467. {
  5468. EclIR::dbglogIR(2, queryLocationIndependent(prevValue), queryLocationIndependent(value));
  5469. if (curOp == no_stored)
  5470. throwError1(HQLERR_DuplicateStoredDefinition, s.str());
  5471. else
  5472. throwError1(HQLERR_DuplicateDefinition, s.str());
  5473. }
  5474. }
  5475. }
  5476. //If the body was essentially the same, call transform on the previous value - so
  5477. return transform(&alreadyProcessedUntransformed.item(match));
  5478. }
  5479. }
  5480. ContingencyData conts;
  5481. ScheduleData sched;
  5482. unsigned wfid = ++wfidCount;
  5483. unsigned schedWfid = 0;
  5484. ForEachItemIn(idx, actions)
  5485. {
  5486. IHqlExpression & cur = actions.item(idx);
  5487. node_operator curOp = cur.getOperator();
  5488. switch (curOp)
  5489. {
  5490. case no_persist:
  5491. if (isRoxie)
  5492. {
  5493. // MORE - Add dynamic attribute to ensure the file is not pre-resolved
  5494. }
  5495. //fall through
  5496. case no_critical:
  5497. case no_checkpoint:
  5498. case no_stored:
  5499. {
  5500. info.extractStoredInfo(&cur, id, codehash, isRoxie, multiplePersistInstances);
  5501. OwnedHqlExpr id = info.getStoredKey();
  5502. alreadyProcessed.append(*id.getClear());
  5503. alreadyProcessedExpr.append(*LINK(expr));
  5504. alreadyProcessedUntransformed.append(*LINK(untransformed));
  5505. }
  5506. break;
  5507. case no_independent:
  5508. case no_once:
  5509. info.extractStoredInfo(&cur, id, codehash, isRoxie, multiplePersistInstances);
  5510. break;
  5511. case no_success:
  5512. {
  5513. OwnedHqlExpr successExpr = transformSequentialEtc(cur.queryChild(0));
  5514. conts.success = splitValue(successExpr);
  5515. Owned<IWorkflowItem> wf = addWorkflowContingencyToWorkunit(conts.success, WFTypeSuccess, WFModeNormal, queryDirectDependencies(successExpr), NULL, wfid, info.queryLabel());
  5516. info.extractStoredInfo(&cur, id, codehash, isRoxie, multiplePersistInstances);
  5517. break;
  5518. }
  5519. case no_failure:
  5520. {
  5521. OwnedHqlExpr failureExpr = transformSequentialEtc(cur.queryChild(0));
  5522. conts.failure = splitValue(failureExpr);
  5523. Owned<IWorkflowItem> wf = addWorkflowContingencyToWorkunit(conts.failure, WFTypeFailure, WFModeNormal, queryDirectDependencies(failureExpr), NULL, wfid, info.queryLabel());
  5524. info.extractStoredInfo(&cur, id, codehash, isRoxie, multiplePersistInstances);
  5525. break;
  5526. }
  5527. case no_recovery:
  5528. {
  5529. conts.recovery = splitValue(cur.queryChild(0));
  5530. conts.retries = (unsigned)getIntValue(cur.queryChild(1), 0);
  5531. Owned<IWorkflowItem> wf = addWorkflowContingencyToWorkunit(conts.recovery, WFTypeRecovery, WFModeNormal, queryDirectDependencies(cur.queryChild(0)), NULL, wfid, info.queryLabel());
  5532. info.extractStoredInfo(&cur, id, codehash, isRoxie, multiplePersistInstances);
  5533. break;
  5534. }
  5535. case no_attr:
  5536. assertex(cur.queryName() == _original_Atom);
  5537. break;
  5538. case no_when:
  5539. {
  5540. OwnedHqlExpr folded = foldHqlExpression(&cur);
  5541. IHqlExpression * event = folded->queryChild(0);
  5542. IHqlExpression * eventFilter = event->queryChild(1);
  5543. sched.now = false;
  5544. event->queryChild(0)->queryValue()->getStringValue(sched.eventName);
  5545. if (eventFilter)
  5546. eventFilter->queryValue()->getStringValue(sched.eventText);
  5547. else
  5548. sched.eventText.append("*");
  5549. if(cur.numChildren()>1)
  5550. {
  5551. sched.counting = true;
  5552. sched.count = (unsigned)getIntValue(folded->queryChild(1));
  5553. }
  5554. sched.independent = true;
  5555. }
  5556. break;
  5557. case no_priority:
  5558. {
  5559. sched.priority = (int)getIntValue(cur.queryChild(0));
  5560. sched.independent = true;
  5561. break;
  5562. }
  5563. default:
  5564. throwUnexpectedOp(curOp);
  5565. }
  5566. }
  5567. if (isDependentOnParameter(expr))
  5568. {
  5569. StringBuffer s;
  5570. getStoredDescription(s, info.sequence, info.originalLabel, true);
  5571. translator.reportWarning(CategoryMistake, SeverityUnknown, queryActiveLocation(expr), HQLWRN_WorkflowDependParameter, HQLWRN_WorkflowDependParameter_Text, s.str());
  5572. }
  5573. OwnedHqlExpr setValue;
  5574. OwnedHqlExpr getValue;
  5575. bool done = false;
  5576. if (info.setOp != no_none)
  5577. {
  5578. assertex(!sched.independent); // should have been enforced by the tree normalization
  5579. ITypeInfo * type = expr->queryType();
  5580. info.checkFew(translator);
  5581. info.splitGlobalDefinition(type, value, wu, setValue, &getValue, isRoxie);
  5582. copySetValueDependencies(queryBodyExtra(value), setValue);
  5583. }
  5584. else
  5585. {
  5586. assertex(sched.independent);
  5587. getValue.set(value);
  5588. done = true;
  5589. schedWfid = wfid;
  5590. }
  5591. if(!sched.independent && !conts.success && !conts.failure && !conts.recovery)
  5592. {
  5593. bool combine = false;
  5594. if (combineAllStored && !hasNonTrivialDependencies(setValue))
  5595. {
  5596. switch (getResultSequenceValue(setValue))
  5597. {
  5598. case ResultSequenceStored:
  5599. combine = true;
  5600. break;
  5601. case ResultSequenceInternal:
  5602. combine = insideStored;
  5603. break;
  5604. }
  5605. }
  5606. if (info.persistOp == no_once)
  5607. {
  5608. //MORE: Error if refers to stored or persist
  5609. if (queryDirectDependencies(setValue).ordinality())
  5610. translator.ERRORAT(queryLocation(untransformed), HQLERR_OnceCannotAccessStored);
  5611. if (onceWfid == 0)
  5612. {
  5613. onceWfid = wfid;
  5614. }
  5615. else
  5616. {
  5617. wfid = onceWfid;
  5618. wfidCount--;
  5619. }
  5620. if (!onceExprs.contains(*setValue))
  5621. onceExprs.append(*LINK(setValue));
  5622. done = true;
  5623. }
  5624. if (combineTrivialStored && isTrivialStored(setValue))
  5625. combine = true;
  5626. if (combine)
  5627. {
  5628. if (trivialStoredWfid == 0)
  5629. {
  5630. trivialStoredWfid = wfid;
  5631. storedWfids.append(wfid);
  5632. }
  5633. else
  5634. {
  5635. wfid = trivialStoredWfid;
  5636. wfidCount--;
  5637. }
  5638. if (trivialStoredExprs.find(*setValue) == NotFound)
  5639. trivialStoredExprs.append(*LINK(setValue));
  5640. done = true;
  5641. }
  5642. }
  5643. if (!done)
  5644. {
  5645. if (info.persistOp == no_stored)
  5646. storedWfids.append(wfid);
  5647. //If you really want side effects within a no_persist to be processed in the correct sequence
  5648. //you need to use persist(failure(independent, f(independent))
  5649. //It generally makes worse code (and incorrect in jbellow.xhql) if they are expanded.
  5650. //because there is no ensure result in the expected wfid.
  5651. if ((info.persistOp != no_persist) && expr->isAction())
  5652. setValue.setown(transformSequentialEtc(setValue));
  5653. if(info.persistOp == no_critical)
  5654. {
  5655. StringBuffer criticalName;
  5656. info.storedName->queryValue()->getStringValue(criticalName);
  5657. Owned<IWorkflowItem> wf = addWorkflowToWorkunit(wfid, WFTypeNormal, WFModeCritical, queryDirectDependencies(setValue), conts, info.queryCluster(), info.queryLabel());
  5658. setWorkflowCritical(wf, criticalName.str());
  5659. addWorkflowItem(*createWorkflowItem(getValue, wfid, no_none, info.queryLabel()));
  5660. getValue.setown(createNullExpr(expr->queryType()));
  5661. }
  5662. else if(info.persistOp == no_persist)
  5663. {
  5664. StringBuffer persistName;
  5665. info.storedName->queryValue()->getStringValue(persistName);
  5666. unsigned persistWfid = ++wfidCount;
  5667. Owned<IWorkflowItem> wf = addWorkflowToWorkunit(wfid, WFTypeNormal, WFModePersist, queryDirectDependencies(setValue), conts, info.queryCluster(), info.queryLabel());
  5668. setWorkflowPersist(wf, persistName.str(), persistWfid, info.queryMaxPersistCopies(), info.queryPersistRefresh());
  5669. DependenciesUsed dependencies(false);
  5670. UnsignedArray visited;
  5671. extractDependentInputs(visited, dependencies, queryDirectDependencies(setValue));
  5672. gatherDependencies(setValue, dependencies, GatherAll);
  5673. dependencies.removeInternalReads();
  5674. HqlExprArray checkArgs;
  5675. checkArgs.append(*createExprAttribute(_files_Atom, dependencies.tablesRead));
  5676. inheritDependencies(&checkArgs.item(0));
  5677. if (dependencies.resultsRead.ordinality())
  5678. {
  5679. checkArgs.append(*createResultsAttribute(dependencies.resultsRead));
  5680. inheritDependencies(&checkArgs.item(1));
  5681. }
  5682. checkArgs.append(*createAttribute(_codehash_Atom, LINK(codehash)));
  5683. checkArgs.append(*createAttribute(namedAtom, LINK(info.storedName)));
  5684. if (expr->isDataset())
  5685. checkArgs.append(*createAttribute(fileAtom));
  5686. OwnedHqlExpr check = createValue(no_persist_check, makeVoidType(), checkArgs);
  5687. inheritDependencies(check);
  5688. addWorkflowItem(*createWorkflowItem(check, persistWfid, no_actionlist, nullptr));
  5689. addWorkflowItem(*createWorkflowItem(setValue, wfid, no_persist, info.queryLabel()));
  5690. Owned<IWorkflowItem> wfPersist = addWorkflowToWorkunit(persistWfid, WFTypeNormal, WFModeNormal, queryDirectDependencies(check), nullptr, nullptr);
  5691. }
  5692. else
  5693. {
  5694. if (info.queryCluster())
  5695. {
  5696. OwnedHqlExpr cluster = createValue(no_cluster, makeVoidType(), LINK(setValue), LINK(info.queryCluster()));
  5697. inheritDependencies(cluster);
  5698. setValue.set(cluster);
  5699. }
  5700. Owned<IWorkflowItem> wf = addWorkflowToWorkunit(wfid, WFTypeNormal, WFModeNormal, queryDirectDependencies(setValue), conts, info.queryCluster(), info.queryLabel());
  5701. addWorkflowItem(*createWorkflowItem(setValue, wfid, info.persistOp, info.queryLabel()));
  5702. }
  5703. }
  5704. if(sched.independent)
  5705. {
  5706. if (schedWfid == 0)
  5707. schedWfid = ++wfidCount;
  5708. Owned<IWorkflowItem> wf = addWorkflowToWorkunit(schedWfid, WFTypeNormal, WFModeNormal, queryDirectDependencies(getValue), info.queryCluster(), info.queryLabel());
  5709. setWorkflowSchedule(wf, sched);
  5710. addWorkflowItem(*createWorkflowItem(getValue, schedWfid, no_none, info.queryLabel()));
  5711. getValue.setown(createNullExpr(expr->queryType()));
  5712. }
  5713. else
  5714. queryBodyExtra(getValue.get())->addDependency(wfid);
  5715. return getValue.getClear();
  5716. }
  5717. IHqlExpression * WorkflowTransformer::extractCommonWorkflow(IHqlExpression * expr, IHqlExpression * transformed)
  5718. {
  5719. if (!transformed->queryDataset())
  5720. return LINK(transformed);
  5721. WorkflowTransformInfo * extra = queryBodyExtra(expr);
  5722. if (!extra->isCommonUpCandidate() || !isWorthHoisting(transformed, false))
  5723. return LINK(transformed);
  5724. if (isContextDependent(transformed) || !isIndependentOfScope(transformed))
  5725. return LINK(transformed);
  5726. StringBuffer s;
  5727. IHqlExpression * location = activeLocations.ordinality() ? &activeLocations.tos() : NULL;
  5728. if (!translator.queryOptions().performWorkflowCse)
  5729. {
  5730. s.appendf("AutoWorkflow: Try adding ': INDEPENDENT' to %s ", getOpString(expr->getOperator()));
  5731. if (expr->queryName())
  5732. s.append("[").append(expr->queryName()).append("] ");
  5733. s.append(" to common up code between workflow items");
  5734. DBGLOG("%s", s.str());
  5735. translator.addWorkunitException(SeverityInformation, HQLWRN_TryAddingIndependent, s.str(), location);
  5736. if (!translator.queryOptions().performWorkflowCse)
  5737. return LINK(transformed);
  5738. }
  5739. //This code would need a lot more work for it to be enabled by default.
  5740. // e.g., ensure it really is worth commoning up, the expressions aren't to be evaluated on different clusters etc. etc.
  5741. unsigned wfid = ++wfidCount;
  5742. s.appendf("AutoWorkflow: Spotted %s ", getOpString(expr->getOperator()));
  5743. if (expr->queryId())
  5744. s.append("[").append(expr->queryId()).append("] ");
  5745. s.append(" to common up between workflow items [").append(wfid).append("]");
  5746. DBGLOG("%s", s.str());
  5747. translator.addWorkunitException(SeverityInformation, 0, s.str(), location);
  5748. GlobalAttributeInfo info("jobtemp::wfa", "wfa", transformed);
  5749. info.extractGlobal(NULL, translator.getTargetClusterType()); // should really be a slightly different function
  5750. OwnedHqlExpr setValue;
  5751. OwnedHqlExpr getValue;
  5752. ContingencyData conts;
  5753. WorkflowTransformInfo * transformedExtra = queryBodyExtra(transformed);
  5754. info.splitGlobalDefinition(transformed->queryType(), transformed, wu, setValue, &getValue, isRoxie);
  5755. copySetValueDependencies(transformedExtra, setValue);
  5756. Owned<IWorkflowItem> wf = addWorkflowToWorkunit(wfid, WFTypeNormal, WFModeNormal, queryDirectDependencies(setValue), conts, NULL, info.queryLabel());
  5757. addWorkflowItem(*createWorkflowItem(setValue, wfid, no_actionlist, str(expr->queryId())));
  5758. queryBodyExtra(getValue.get())->addDependency(wfid);
  5759. return getValue.getClear();
  5760. }
  5761. IHqlExpression * WorkflowTransformer::transformInternalFunction(IHqlExpression * newFuncDef)
  5762. {
  5763. IHqlExpression * body = newFuncDef->queryChild(0);
  5764. if (body->getOperator() != no_outofline)
  5765. return LINK(newFuncDef);
  5766. IHqlExpression * ecl = body->queryChild(0);
  5767. StringBuffer funcname;
  5768. funcname.append("user").append(++nextInternalFunctionId);
  5769. if (translator.queryOptions().debugGeneratedCpp)
  5770. funcname.append("_").append(newFuncDef->queryName()).toLowerCase();
  5771. OwnedHqlExpr funcNameExpr = createConstant(funcname);
  5772. HqlExprArray bodyArgs;
  5773. unwindChildren(bodyArgs, body, 0);
  5774. bodyArgs.append(*createLocalAttribute());
  5775. bodyArgs.append(*createExprAttribute(entrypointAtom, LINK(funcNameExpr)));
  5776. IHqlExpression *timeAttr = ecl->queryAttribute(timeAtom);
  5777. if (timeAttr)
  5778. bodyArgs.append(*LINK(timeAttr));
  5779. OwnedHqlExpr newBody = body->clone(bodyArgs);
  5780. inheritDependencies(newBody);
  5781. copyDependencies(queryBodyExtra(ecl), queryBodyExtra(newBody));
  5782. HqlExprArray funcdefArgs;
  5783. funcdefArgs.append(*LINK(newBody));
  5784. if (ecl->getOperator() == no_embedbody && ecl->hasAttribute(languageAtom))
  5785. {
  5786. IHqlExpression * outofline = newFuncDef->queryChild(0);
  5787. IHqlExpression * formals = newFuncDef->queryChild(1);
  5788. IHqlExpression * defaults = newFuncDef->queryChild(2);
  5789. assertex(outofline->getOperator() == no_outofline);
  5790. IHqlExpression * bodyCode = outofline->queryChild(0);
  5791. HqlExprArray newFormals;
  5792. unwindChildren(newFormals, formals);
  5793. HqlExprArray attrs;
  5794. attrs.append(*createAttribute(_hidden_Atom));
  5795. newFormals.append(*createParameter(__optionsId, newFormals.length(), LINK(unknownVarStringType), attrs));
  5796. HqlExprArray newDefaults;
  5797. if (defaults)
  5798. unwindChildren(newDefaults, defaults, 0);
  5799. while (newDefaults.length() < formals->numChildren())
  5800. newDefaults.append(*createOmittedValue());
  5801. newDefaults.append(*getEmbedOptionString(newFuncDef));
  5802. IHqlExpression *query = bodyCode->queryChild(0);
  5803. if (!query->queryValue())
  5804. {
  5805. newFormals.append(*createParameter(__queryId, newFormals.length(), LINK(unknownUtf8Type), attrs));
  5806. newDefaults.append(*LINK(query));
  5807. }
  5808. funcdefArgs.append(*formals->clone(newFormals));
  5809. funcdefArgs.append(*createValueSafe(no_sortlist, makeSortListType(NULL), newDefaults));
  5810. unwindChildren(funcdefArgs, newFuncDef, 3);
  5811. OwnedHqlExpr namedFuncDef = newFuncDef->clone(funcdefArgs);
  5812. inheritDependencies(namedFuncDef);
  5813. return namedFuncDef.getClear();
  5814. }
  5815. else
  5816. {
  5817. unwindChildren(funcdefArgs, newFuncDef, 1);
  5818. OwnedHqlExpr namedFuncDef = newFuncDef->clone(funcdefArgs);
  5819. inheritDependencies(namedFuncDef);
  5820. if (ecl->getOperator() == no_embedbody)
  5821. return namedFuncDef.getClear();
  5822. WorkflowItem * item = new WorkflowItem(namedFuncDef);
  5823. functions.append(*item);
  5824. OwnedHqlExpr external = createExternalFuncdefFromInternal(namedFuncDef);
  5825. copyDependencies(queryBodyExtra(namedFuncDef), queryBodyExtra(external));
  5826. return external.getClear();
  5827. }
  5828. }
  5829. IHqlExpression * WorkflowTransformer::transformInternalCall(IHqlExpression * transformed)
  5830. {
  5831. IHqlExpression * funcDef = transformed->queryDefinition();
  5832. Owned<IHqlExpression> newFuncDef = transform(funcDef);
  5833. HqlExprArray parameters;
  5834. unwindChildren(parameters, transformed);
  5835. IHqlExpression * body = newFuncDef->queryChild(0);
  5836. if (body && body->getOperator() == no_outofline)
  5837. {
  5838. IHqlExpression * ecl = body->queryChild(0);
  5839. if (ecl->getOperator() == no_embedbody && ecl->hasAttribute(languageAtom))
  5840. {
  5841. // Copy the new default value(s) into the end of the parameters array
  5842. IHqlExpression *formals = newFuncDef->queryChild(1);
  5843. IHqlExpression *defaults = newFuncDef->queryChild(2);
  5844. while (parameters.length() < formals->numChildren())
  5845. parameters.append(*LINK(defaults->queryChild(parameters.length())));
  5846. }
  5847. }
  5848. OwnedHqlExpr rebound = createReboundFunction(newFuncDef, parameters);
  5849. inheritDependencies(rebound);
  5850. copyDependencies(queryBodyExtra(newFuncDef), queryBodyExtra(rebound));
  5851. return rebound.getClear();
  5852. }
  5853. IHqlExpression * WorkflowTransformer::createTransformed(IHqlExpression * expr)
  5854. {
  5855. //Could short-circuit if doesn't contain workflow, but it also modifies outputs/buildindex...
  5856. //Force record to be transformed - so any stored values in record (ifblock!!) are hoisted.
  5857. node_operator op = expr->getOperator();
  5858. if (op == no_param)
  5859. return LINK(expr);
  5860. if (op == no_transform || op == no_newtransform)
  5861. ::Release(transform(expr->queryRecord()));
  5862. IHqlExpression * body = expr->queryBody(true);
  5863. if (expr != body)
  5864. {
  5865. switch (expr->getAnnotationKind())
  5866. {
  5867. case annotate_location:
  5868. case annotate_symbol:
  5869. activeLocations.append(*expr);
  5870. break;
  5871. }
  5872. OwnedHqlExpr transformedBody = transform(body);
  5873. switch (expr->getAnnotationKind())
  5874. {
  5875. case annotate_location:
  5876. case annotate_symbol:
  5877. activeLocations.pop();
  5878. break;
  5879. }
  5880. OwnedHqlExpr transformed = (transformedBody == body) ? LINK(expr) : expr->cloneAnnotation(transformedBody);
  5881. //more: this really shouldn't be needed
  5882. inheritDependencies(transformed);
  5883. return transformed.getClear();
  5884. }
  5885. bool wasInsideStored = insideStored;
  5886. if ((op == no_colon) && queryOperatorInList(no_stored, expr->queryChild(1)))
  5887. insideStored = true;
  5888. OwnedHqlExpr transformed = NewHqlTransformer::createTransformed(expr);
  5889. insideStored = wasInsideStored;
  5890. inheritDependencies(transformed);
  5891. switch (op)
  5892. {
  5893. #if 0
  5894. //MORE: Workflow in user functions doesn't work for roxie at the moment
  5895. case no_call:
  5896. transformed.setown(transformCall(transformed));
  5897. inheritDependencies(transformed);
  5898. copyDependencies(queryBodyExtra(transformed->queryExternalDefinition()), queryBodyExtra(transformed));
  5899. break;
  5900. case no_externalcall:
  5901. transformed.setown(transformExternalCall(transformed));
  5902. inheritDependencies(transformed);
  5903. copyDependencies(queryExtra(transformed->queryExternalDefinition()), queryExtra(transformed));
  5904. break;
  5905. #endif
  5906. case no_colon:
  5907. if (translator.insideLibrary())
  5908. {
  5909. SCMStringBuffer libraryName;
  5910. StringBuffer colonText(" (");
  5911. getOutputLibraryName(libraryName, wu);
  5912. getExprECL(expr, colonText);
  5913. colonText.append(")");
  5914. throwError2(HQLERR_LibraryCannotContainWorkflow, libraryName.str(), colonText.str());
  5915. }
  5916. transformed.setown(extractWorkflow(expr, transformed));
  5917. break;
  5918. case no_output:
  5919. case no_buildindex:
  5920. {
  5921. IHqlExpression * updateAttr = transformed->queryAttribute(updateAtom);
  5922. if (updateAttr)
  5923. {
  5924. DependenciesUsed dependencies(false);
  5925. gatherDependencies(transformed->queryChild(0), dependencies, GatherAll);
  5926. dependencies.removeInternalReads();
  5927. bool canEvaluateFilenames = true;
  5928. HqlExprArray updateArgs;
  5929. unwindChildren(updateArgs, updateAttr);
  5930. if (dependencies.tablesRead.ordinality())
  5931. {
  5932. OwnedHqlExpr attr = createExprAttribute(_files_Atom, dependencies.tablesRead);
  5933. if (!isIndependentOfScope(attr) || isContextDependent(attr))
  5934. {
  5935. if (!updateAttr->hasAttribute(alwaysAtom))
  5936. throwError(HQLERR_InputsAreTooComplexToUpdate);
  5937. canEvaluateFilenames = false;
  5938. }
  5939. else
  5940. updateArgs.append(*attr.getClear());
  5941. }
  5942. if (dependencies.resultsRead.ordinality())
  5943. updateArgs.append(*createResultsAttribute(dependencies.resultsRead));
  5944. HqlExprArray args;
  5945. unwindChildren(args, transformed);
  5946. args.zap(*updateAttr);
  5947. if (canEvaluateFilenames)
  5948. args.append(*createExprAttribute(updateAtom, updateArgs));
  5949. transformed.setown(transformed->clone(args));
  5950. inheritDependencies(transformed);
  5951. }
  5952. break;
  5953. }
  5954. case no_funcdef:
  5955. transformed.setown(transformInternalFunction(transformed));
  5956. break;
  5957. case no_call:
  5958. transformed.setown(transformInternalCall(transformed));
  5959. break;
  5960. }
  5961. return extractCommonWorkflow(expr, transformed);
  5962. }
  5963. //-- second pass - sort out sequential etc.
  5964. /*
  5965. This is very tricky... The problem is we only want to create workflow actions for sequential/parallel and conditions if they
  5966. are necessary. In particular.
  5967. o workflow items are only executed once per invocation
  5968. o create them for sequential if the dependencies haven't already been evaluated
  5969. o create them for conditions if the non-intersection of the dependencies for the branches haven't already been evaluated
  5970. o create if a workflow action has been created for a child action.
  5971. o can't rely on createTransform() updating the dependencies so-far because the transform() may be cached.
  5972. o Need to be careful that dependencies done so far are set up correctly before each call to transform()
  5973. */
  5974. UnsignedArray const & WorkflowTransformer::queryDependencies(unsigned wfid)
  5975. {
  5976. if (wfid == trivialStoredWfid)
  5977. return emptyDependencies;
  5978. ForEachItemIn(i, workflow)
  5979. {
  5980. WorkflowItem & cur = workflow.item(i);
  5981. if (cur.wfid == wfid)
  5982. return cur.dependencies;
  5983. }
  5984. throwUnexpected();
  5985. }
  5986. void WorkflowTransformer::gatherIndirectDependencies(UnsignedArray & result, IHqlExpression * expr)
  5987. {
  5988. if (isWorkflowAction(expr))
  5989. {
  5990. unsigned wfid = (unsigned)getIntValue(expr->queryChild(0));
  5991. ::inheritDependencies(result, queryDependencies(wfid));
  5992. }
  5993. else
  5994. {
  5995. const UnsignedArray & direct = queryBodyExtra(expr)->queryDependencies();
  5996. ForEachItemIn(i, direct)
  5997. {
  5998. unsigned wfid = direct.item(i);
  5999. if (addDependency(result, wfid))
  6000. ::inheritDependencies(result, queryDependencies(wfid));
  6001. }
  6002. }
  6003. }
  6004. bool WorkflowTransformer::hasNonTrivialDependencies(IHqlExpression * expr)
  6005. {
  6006. UnsignedArray const & dependencies = queryDirectDependencies(expr);
  6007. ForEachItemIn(i, dependencies)
  6008. {
  6009. unsigned cur = dependencies.item(i);
  6010. if ((cur != trivialStoredWfid) && (cur != onceWfid))
  6011. return true;
  6012. }
  6013. return false;
  6014. }
  6015. UnsignedArray const & WorkflowTransformer::queryDirectDependencies(IHqlExpression * expr)
  6016. {
  6017. return queryBodyExtra(expr)->queryDependencies();
  6018. }
  6019. void WorkflowTransformer::cacheWorkflowDependencies(unsigned wfid, UnsignedArray & extra)
  6020. {
  6021. WorkflowItem * item = new WorkflowItem(wfid, no_actionlist, nullptr);
  6022. ForEachItemIn(i, extra)
  6023. {
  6024. unsigned wfid = extra.item(i);
  6025. item->dependencies.append(wfid);
  6026. ::inheritDependencies(item->dependencies, queryDependencies(wfid));
  6027. }
  6028. addWorkflowItem(*item);
  6029. }
  6030. IHqlExpression * WorkflowTransformer::createWorkflowAction(unsigned wfid)
  6031. {
  6032. //NB: Needs to include wfid as an argument otherwise inherited dependencies get messed up
  6033. OwnedHqlExpr transformed = createValue(no_workflow_action, makeVoidType(), getSizetConstant(wfid));
  6034. queryBodyExtra(transformed)->addDependency(wfid);
  6035. return transformed.getClear();
  6036. }
  6037. void WorkflowTransformer::ensureWorkflowAction(UnsignedArray & dependencies, IHqlExpression * expr)
  6038. {
  6039. unsigned wfid = ensureWorkflowAction(expr);
  6040. addDependency(dependencies, wfid);
  6041. }
  6042. //Create a sequential workflow action if any of the branches contains a workflow action
  6043. IHqlExpression * WorkflowTransformer::createCompoundWorkflow(IHqlExpression * expr)
  6044. {
  6045. HqlExprArray pendingBranches;
  6046. UnsignedArray childWfid;
  6047. ForEachChild(i, expr)
  6048. {
  6049. IHqlExpression * cur = expr->queryChild(i);
  6050. unsigned mark = markDependencies();
  6051. OwnedHqlExpr transformed = transformRootAction(cur);
  6052. restoreDependencies(mark);
  6053. if (isWorkflowAction(transformed))
  6054. {
  6055. if (pendingBranches.ordinality())
  6056. {
  6057. OwnedHqlExpr branch = createActionList(pendingBranches);
  6058. inheritDependencies(branch);
  6059. ensureWorkflowAction(childWfid, branch);
  6060. pendingBranches.kill();
  6061. }
  6062. ensureWorkflowAction(childWfid, transformed);
  6063. }
  6064. else
  6065. {
  6066. pendingBranches.append(*LINK(transformed));
  6067. }
  6068. gatherIndirectDependencies(cumulativeDependencies, transformed);
  6069. }
  6070. if (childWfid.ordinality())
  6071. {
  6072. if (pendingBranches.ordinality())
  6073. {
  6074. OwnedHqlExpr branch = createActionList(pendingBranches);
  6075. inheritDependencies(branch);
  6076. ensureWorkflowAction(childWfid, branch);
  6077. }
  6078. unsigned wfid = ++wfidCount;
  6079. Owned<IWorkflowItem> wf = addWorkflowToWorkunit(wfid, WFTypeNormal, WFModeSequential, childWfid, rootCluster, str(expr->queryId()));
  6080. cacheWorkflowDependencies(wfid, childWfid);
  6081. return createWorkflowAction(wfid);
  6082. }
  6083. return LINK(expr);
  6084. }
  6085. //Create a sequential workflow action if any of the branches introduce new dependencies/or creates a workflow item (e.g., wait!)
  6086. IHqlExpression * WorkflowTransformer::createSequentialWorkflow(IHqlExpression * expr)
  6087. {
  6088. OwnedHqlExpr nextBranch;
  6089. UnsignedArray childWfid;
  6090. ForEachChild(i, expr)
  6091. {
  6092. IHqlExpression * cur = expr->queryChild(i);
  6093. unsigned mark = markDependencies();
  6094. OwnedHqlExpr transformed = transformRootAction(cur);
  6095. restoreDependencies(mark);
  6096. UnsignedArray dependencies;
  6097. gatherIndirectDependencies(dependencies, transformed);
  6098. if (hasExtraDependencies(cumulativeDependencies, dependencies, storedWfids) || isWorkflowAction(transformed))
  6099. {
  6100. if (nextBranch)
  6101. {
  6102. ensureWorkflowAction(childWfid, nextBranch);
  6103. nextBranch.clear();
  6104. }
  6105. ::inheritDependencies(cumulativeDependencies, dependencies);
  6106. if (isWorkflowAction(transformed))
  6107. ensureWorkflowAction(childWfid, transformed);
  6108. else
  6109. nextBranch.set(transformed);
  6110. }
  6111. else
  6112. {
  6113. if (nextBranch)
  6114. nextBranch.setown(createValue(expr->getOperator(), nextBranch.getClear(), LINK(transformed)));
  6115. else
  6116. nextBranch.set(transformed);
  6117. inheritDependencies(nextBranch);
  6118. }
  6119. }
  6120. if (childWfid.ordinality())
  6121. {
  6122. if (nextBranch)
  6123. ensureWorkflowAction(childWfid, nextBranch);
  6124. unsigned wfid = ++wfidCount;
  6125. Owned<IWorkflowItem> wf = addWorkflowToWorkunit(wfid, WFTypeNormal, WFModeSequential, childWfid, rootCluster, str(expr->queryId()));
  6126. cacheWorkflowDependencies(wfid, childWfid);
  6127. return createWorkflowAction(wfid);
  6128. }
  6129. return LINK(expr);
  6130. }
  6131. // Create a parallel workflow action if any of the child actions are workflow actions
  6132. IHqlExpression * WorkflowTransformer::createParallelWorkflow(IHqlExpression * expr)
  6133. {
  6134. HqlExprArray branches;
  6135. UnsignedArray childWfid;
  6136. unsigned mark = markDependencies();
  6137. ForEachChild(i, expr)
  6138. {
  6139. IHqlExpression * cur = expr->queryChild(i);
  6140. OwnedHqlExpr transformed = transformRootAction(cur);
  6141. if (isWorkflowAction(transformed))
  6142. ensureWorkflowAction(childWfid, transformed);
  6143. else
  6144. branches.append(*LINK(transformed));
  6145. restoreDependencies(mark);
  6146. }
  6147. if (childWfid.ordinality())
  6148. {
  6149. if (branches.ordinality())
  6150. {
  6151. OwnedHqlExpr branch = createActionList(branches);
  6152. inheritDependencies(branch);
  6153. ensureWorkflowAction(childWfid, branch);
  6154. }
  6155. unsigned wfid = ++wfidCount;
  6156. Owned<IWorkflowItem> wf = addWorkflowToWorkunit(wfid, WFTypeNormal, WFModeParallel, childWfid, rootCluster, str(expr->queryId()));
  6157. cacheWorkflowDependencies(wfid, childWfid);
  6158. return createWorkflowAction(wfid);
  6159. }
  6160. return LINK(expr);
  6161. }
  6162. IHqlExpression * WorkflowTransformer::createIfWorkflow(IHqlExpression * expr)
  6163. {
  6164. IHqlExpression * cond = expr->queryChild(0);
  6165. IHqlExpression * trueExpr = expr->queryChild(1);
  6166. IHqlExpression * falseExpr = expr->queryChild(2);
  6167. OwnedHqlExpr newCond = LINK(cond);
  6168. gatherIndirectDependencies(cumulativeDependencies, cond);
  6169. //more: inherit dependencies?
  6170. UnsignedArray trueDepends, falseDepends;
  6171. unsigned mark = markDependencies();
  6172. OwnedHqlExpr newTrueExpr = transformRootAction(trueExpr);
  6173. restoreDependencies(mark);
  6174. OwnedHqlExpr newFalseExpr = falseExpr ? transformRootAction(falseExpr) : NULL;
  6175. restoreDependencies(mark);
  6176. //Need to turn a conditional action into a conditional workflow item if
  6177. //i) it has a workflow action as a child.
  6178. //ii) the true/false branches are dependent on something that hasn't already been evaluated
  6179. // (and isn't shared between both branches)
  6180. bool needToCreateWorkflow = false;
  6181. if (hasDependencies(newTrueExpr) || (newFalseExpr && hasDependencies(newFalseExpr)))
  6182. {
  6183. needToCreateWorkflow = isWorkflowAction(newTrueExpr) || isWorkflowAction(newFalseExpr);
  6184. if (!needToCreateWorkflow)
  6185. {
  6186. //Failures are assumed to be exceptional, so don't worry about extra dependencies
  6187. if (!isFailAction(newTrueExpr) && !isFailAction(newFalseExpr))
  6188. {
  6189. UnsignedArray newTrueDepends;
  6190. gatherIndirectDependencies(newTrueDepends, newTrueExpr);
  6191. if (!falseExpr)
  6192. needToCreateWorkflow = hasExtraDependencies(cumulativeDependencies, newTrueDepends, storedWfids);
  6193. else
  6194. {
  6195. UnsignedArray newFalseDepends;
  6196. gatherIndirectDependencies(newFalseDepends, newFalseExpr);
  6197. UnsignedArray diff;
  6198. diffDependencies(diff, newTrueDepends, newFalseDepends);
  6199. needToCreateWorkflow = hasExtraDependencies(cumulativeDependencies, diff, storedWfids);
  6200. }
  6201. }
  6202. }
  6203. if (needToCreateWorkflow)
  6204. {
  6205. //Represent as wfid(cond-wfid, true-wfid, false-wfid)
  6206. UnsignedArray dependencies;
  6207. OwnedHqlExpr setCondExpr = createValue(no_setworkflow_cond, makeVoidType(), LINK(cond));
  6208. inheritDependencies(setCondExpr);
  6209. ensureWorkflowAction(dependencies, setCondExpr);
  6210. ensureWorkflowAction(dependencies, newTrueExpr);
  6211. if (newFalseExpr)
  6212. ensureWorkflowAction(dependencies, newFalseExpr);
  6213. unsigned wfid = ++wfidCount;
  6214. Owned<IWorkflowItem> wf = addWorkflowToWorkunit(wfid, WFTypeNormal, WFModeCondition, dependencies, rootCluster, str(expr->queryId()));
  6215. WorkflowItem * item = new WorkflowItem(wfid, no_if, "IF");
  6216. cloneDependencies(item->dependencies, dependencies);
  6217. if (falseExpr)
  6218. {
  6219. UnsignedArray newTrueDepends;
  6220. UnsignedArray newFalseDepends;
  6221. gatherIndirectDependencies(newTrueDepends, newTrueExpr);
  6222. gatherIndirectDependencies(newFalseDepends, newFalseExpr);
  6223. intersectDependencies(item->dependencies, newTrueDepends, newFalseDepends);
  6224. }
  6225. addWorkflowItem(*item);
  6226. return createWorkflowAction(wfid);
  6227. }
  6228. }
  6229. return LINK(expr);
  6230. }
  6231. IHqlExpression * WorkflowTransformer::createWaitWorkflow(IHqlExpression * expr)
  6232. {
  6233. //First create a EndWait workflow item which has a when clause of the wait criteria
  6234. OwnedHqlExpr folded = foldHqlExpression(expr);
  6235. IHqlExpression * event = folded->queryChild(0);
  6236. IHqlExpression * eventFilter = event->queryChild(1);
  6237. ScheduleData sched;
  6238. sched.now = false;
  6239. getStringValue(sched.eventName, event->queryChild(0));
  6240. if (eventFilter)
  6241. getStringValue(sched.eventText, eventFilter);
  6242. else
  6243. sched.eventText.append("*");
  6244. sched.counting = true;
  6245. sched.count = 0;
  6246. sched.independent = true;
  6247. unsigned endWaitWfid = ++wfidCount;
  6248. UnsignedArray noDependencies;
  6249. Owned<IWorkflowItem> wf = addWorkflowToWorkunit(endWaitWfid, WFTypeNormal, WFModeWait, noDependencies, rootCluster, str(expr->queryId()));
  6250. setWorkflowSchedule(wf, sched);
  6251. OwnedHqlExpr doNothing = createValue(no_null, makeVoidType());
  6252. addWorkflowItem(*createWorkflowItem(doNothing, endWaitWfid, no_wait, "WAIT"));
  6253. //Now create a wait entry, with the EndWait as the dependency
  6254. UnsignedArray dependencies;
  6255. dependencies.append(endWaitWfid);
  6256. unsigned beginWaitWfid = ++wfidCount;
  6257. Owned<IWorkflowItem> wfWait = addWorkflowToWorkunit(beginWaitWfid, WFTypeNormal, WFModeBeginWait, dependencies, rootCluster, str(expr->queryId()));
  6258. cacheWorkflowDependencies(beginWaitWfid, dependencies);
  6259. return createWorkflowAction(beginWaitWfid);
  6260. }
  6261. IHqlExpression * WorkflowTransformer::transformRootAction(IHqlExpression * expr)
  6262. {
  6263. node_operator op = expr->getOperator();
  6264. switch (op)
  6265. {
  6266. case no_compound:
  6267. if (expr->isAction())
  6268. return createCompoundWorkflow(expr);
  6269. break;
  6270. case no_parallel:
  6271. return createParallelWorkflow(expr);
  6272. case no_sequential:
  6273. case no_orderedactionlist:
  6274. return createSequentialWorkflow(expr);
  6275. case no_actionlist:
  6276. return createCompoundWorkflow(expr);
  6277. case no_if:
  6278. if (expr->isAction())
  6279. return createIfWorkflow(expr);
  6280. break;
  6281. case no_wait:
  6282. return createWaitWorkflow(expr);
  6283. case no_ensureresult:
  6284. {
  6285. IHqlExpression * value = expr->queryChild(0);
  6286. if (!value->isAction())
  6287. break;
  6288. OwnedHqlExpr transformed = transformRootAction(value);
  6289. if (value == transformed)
  6290. break;
  6291. HqlExprArray args;
  6292. args.append(*transformed.getClear());
  6293. unwindChildren(args, expr, 1);
  6294. OwnedHqlExpr ret = expr->clone(args);
  6295. inheritDependencies(ret);
  6296. return ret.getClear();
  6297. }
  6298. }
  6299. return LINK(expr);
  6300. }
  6301. IHqlExpression * WorkflowTransformer::transformSequentialEtc(IHqlExpression * expr)
  6302. {
  6303. unsigned mark = markDependencies();
  6304. //Ignore differences in access to trivial stored variables.
  6305. if (trivialStoredWfid)
  6306. cumulativeDependencies.append(trivialStoredWfid);
  6307. if (onceWfid)
  6308. cumulativeDependencies.append(onceWfid);
  6309. OwnedHqlExpr ret = transformRootAction(expr);
  6310. restoreDependencies(mark);
  6311. return ret.getClear();
  6312. }
  6313. void WorkflowTransformer::addWorkflowItem(WorkflowItem & item)
  6314. {
  6315. workflow.append(item);
  6316. }
  6317. void WorkflowTransformer::percolateScheduledIds(UnsignedArray & visited, const UnsignedArray & dependencies, unsigned rootWfid)
  6318. {
  6319. ForEachItemIn(i2, dependencies)
  6320. {
  6321. unsigned wfid = dependencies.item(i2);
  6322. if (visited.contains(wfid))
  6323. continue;
  6324. visited.append(wfid);
  6325. Owned<IWorkflowItem> child = lookupWorkflowItem(wfid);
  6326. if (child->queryMode() == WFModeWait)
  6327. child->setScheduledWfid(rootWfid);
  6328. else
  6329. {
  6330. WorkflowItem * cur = findWorkflowItem(wfid);
  6331. assertex(cur);
  6332. percolateScheduledIds(visited, cur->dependencies, rootWfid);
  6333. }
  6334. }
  6335. }
  6336. void WorkflowTransformer::percolateScheduledIds()
  6337. {
  6338. ForEachItemIn(i, workflow)
  6339. {
  6340. WorkflowItem & cur = workflow.item(i);
  6341. Owned<IWorkflowItem> wf = lookupWorkflowItem(cur.queryWfid());
  6342. if (wf && wf->isScheduledNow())
  6343. {
  6344. UnsignedArray visited;
  6345. percolateScheduledIds(visited, cur.dependencies, cur.queryWfid());
  6346. }
  6347. }
  6348. }
  6349. ///- workflow processing
  6350. void WorkflowTransformer::analyseExpr(IHqlExpression * expr)
  6351. {
  6352. WorkflowTransformInfo * extra = queryBodyExtra(expr);
  6353. if (extra->noteWorkflow(activeWfid, isConditional))
  6354. return;
  6355. switch (expr->getOperator())
  6356. {
  6357. case no_allnodes:
  6358. //MORE: Do I need to recurse and explicitly disable hoisting?
  6359. return;
  6360. case no_if:
  6361. {
  6362. bool wasConditional = isConditional;
  6363. analyseExpr(expr->queryChild(0));
  6364. isConditional = true;
  6365. analyseExpr(expr->queryChild(1));
  6366. if (expr->queryChild(2))
  6367. analyseExpr(expr->queryChild(2));
  6368. isConditional = wasConditional;
  6369. return;
  6370. }
  6371. case no_colon:
  6372. {
  6373. if (!isIndependentOfScope(expr->queryChild(0)))
  6374. {
  6375. StringBuffer s;
  6376. if (expr->queryName())
  6377. s.appendf(" '%s'", str(expr->queryName()));
  6378. //MORE: Better if we also kept nested track of locations
  6379. translator.WARNINGAT1(CategoryMistake, queryActiveLocation(expr), HQLWRN_WorkflowSeemsToBeDependent, s.str());
  6380. }
  6381. bool wasConditional = isConditional;
  6382. isConditional = false;
  6383. unsigned prevWfid = activeWfid;
  6384. activeWfid = ++wfidCount;
  6385. analyseExpr(expr->queryChild(0));
  6386. activeWfid = prevWfid;
  6387. isConditional = wasConditional;
  6388. return;
  6389. }
  6390. }
  6391. NewHqlTransformer::analyseExpr(expr);
  6392. }
  6393. void WorkflowTransformer::analyseAll(const HqlExprArray & in)
  6394. {
  6395. activeWfid = ++wfidCount;
  6396. analyseArray(in, 0);
  6397. wfidCount = 0;
  6398. }
  6399. void WorkflowTransformer::transformRoot(const HqlExprArray & in, WorkflowArray & out)
  6400. {
  6401. wfidCount = 0;
  6402. HqlExprArray transformed;
  6403. WorkflowTransformInfo globalInfo(NULL);
  6404. ForEachItemIn(idx, in)
  6405. {
  6406. OwnedHqlExpr ret = transform(&in.item(idx));
  6407. copyDependencies(queryBodyExtra(ret), &globalInfo);
  6408. //ignore results that do nothing, but still collect the dependencies...
  6409. if (ret->getOperator() != no_null)
  6410. transformed.append(*ret.getClear());
  6411. }
  6412. if (onceExprs.length())
  6413. {
  6414. //By definition they don't have any dependencies, so no need to call inheritDependencies.
  6415. OwnedHqlExpr onceExpr = createActionList(onceExprs);
  6416. Owned<IWorkflowItem> wf = addWorkflowToWorkunit(onceWfid, WFTypeNormal, WFModeOnce, queryDirectDependencies(onceExpr), NULL, "ONCE");
  6417. wf->setScheduledNow();
  6418. addWorkflowItem(*createWorkflowItem(onceExpr, onceWfid, no_once, "ONCE"));
  6419. }
  6420. if (trivialStoredExprs.length())
  6421. {
  6422. //By definition they don't have any dependencies, so no need to call inheritDependencies.
  6423. OwnedHqlExpr trivialStoredExpr = createActionList(trivialStoredExprs);
  6424. Owned<IWorkflowItem> wf = addWorkflowToWorkunit(trivialStoredWfid, WFTypeNormal, WFModeNormal, queryDirectDependencies(trivialStoredExpr), NULL, nullptr);
  6425. addWorkflowItem(*createWorkflowItem(trivialStoredExpr, trivialStoredWfid, no_stored, "Trivial STORED"));
  6426. }
  6427. if (transformed.ordinality())
  6428. {
  6429. //Handle sequential etc.
  6430. OwnedHqlExpr combined = createActionList(transformed);
  6431. OwnedHqlExpr result = transformSequentialEtc(combined);
  6432. transformed.kill();
  6433. transformed.append(*result.getClear());
  6434. }
  6435. UnsignedArray const & dependencies = globalInfo.queryDependencies();
  6436. if(transformed.ordinality() || dependencies.ordinality())
  6437. {
  6438. if ((transformed.ordinality() == 0) && (dependencies.ordinality() == 1))
  6439. {
  6440. Owned<IWorkflowItem> wf = lookupWorkflowItem(dependencies.item(0));
  6441. wf->setScheduledNow();
  6442. }
  6443. else
  6444. {
  6445. Owned<IHqlExpression> combinedItems = createComma(transformed);
  6446. if (!combinedItems)
  6447. combinedItems.setown(createValue(no_null, makeVoidType()));
  6448. unsigned wfid;
  6449. if (!isWorkflowAction(combinedItems))
  6450. {
  6451. wfid = ++wfidCount;
  6452. ScheduleData sched;
  6453. Owned<IWorkflowItem> wf = addWorkflowToWorkunit(wfid, WFTypeNormal, WFModeNormal, dependencies, NULL, nullptr);
  6454. setWorkflowSchedule(wf, sched);
  6455. addWorkflowItem(*createWorkflowItem(combinedItems, wfid, no_actionlist, nullptr));
  6456. }
  6457. else
  6458. wfid = ensureWorkflowAction(combinedItems);
  6459. Owned<IWorkflowItem> wf = lookupWorkflowItem(wfid);
  6460. wf->setScheduledNow();
  6461. }
  6462. }
  6463. percolateScheduledIds();
  6464. appendArray(out, workflow);
  6465. appendArray(out, functions);
  6466. }
  6467. void extractWorkflow(HqlCppTranslator & translator, HqlExprArray & exprs, WorkflowArray & out)
  6468. {
  6469. WorkflowTransformer transformer(translator.wu(), translator);
  6470. if (translator.queryOptions().performWorkflowCse || translator.queryOptions().notifyWorkflowCse)
  6471. transformer.analyseAll(exprs);
  6472. transformer.transformRoot(exprs, out);
  6473. }
  6474. //------------------------------------------------------------------------
  6475. enum { SIKnone, SIKhole, SIKagent, SIKthor };
  6476. class StatementInfo : public CInterface
  6477. {
  6478. public:
  6479. StatementInfo(IHqlExpression * _expr);
  6480. void calcDependencies();
  6481. bool canSwapOrder(StatementInfo & other)
  6482. {
  6483. return queryDependencies().canSwapOrder(other.queryDependencies());
  6484. }
  6485. inline bool isConditional() { return expr->getOperator() == no_if; }
  6486. inline bool isThorQuery() { return category == SIKthor; }
  6487. DependenciesUsed & queryDependencies()
  6488. {
  6489. if (!hasDependencies)
  6490. {
  6491. calcDependencies();
  6492. hasDependencies = true;
  6493. }
  6494. return dependencies;
  6495. }
  6496. public:
  6497. HqlExprAttr expr;
  6498. protected:
  6499. DependenciesUsed dependencies;
  6500. bool hasDependencies;
  6501. unsigned category;
  6502. };
  6503. StatementInfo::StatementInfo(IHqlExpression * _expr) : dependencies(true)
  6504. {
  6505. expr.set(_expr);
  6506. if (expr->getOperator() == no_thor)
  6507. category = SIKthor;
  6508. else
  6509. category = SIKagent;
  6510. hasDependencies = false;
  6511. }
  6512. void StatementInfo::calcDependencies()
  6513. {
  6514. gatherDependencies(expr, dependencies, GatherAll);
  6515. }
  6516. void groupThorGraphs(HqlExprArray & in)
  6517. {
  6518. //Gather information about the statements...
  6519. bool hadThor = false;
  6520. bool lastWasThor = false;
  6521. bool couldImprove = false;
  6522. CIArrayOf<StatementInfo> stmts;
  6523. ForEachItemIn(idx, in)
  6524. {
  6525. StatementInfo & cur = *new StatementInfo(&in.item(idx));
  6526. stmts.append(cur);
  6527. if (cur.isThorQuery())
  6528. {
  6529. if (hadThor && !lastWasThor)
  6530. couldImprove = true;
  6531. hadThor = true;
  6532. lastWasThor = true;
  6533. }
  6534. else
  6535. lastWasThor = false;
  6536. }
  6537. //If no thor queries are split by other queries, then may as well keep in the same order...
  6538. if (!couldImprove)
  6539. return;
  6540. //Need to work out the best order to generate the statements in. We want
  6541. //to move non thor queries to the front, so we do a insertion sort on them
  6542. CICopyArrayOf<StatementInfo> sorted;
  6543. ForEachItemIn(idx1, stmts)
  6544. {
  6545. StatementInfo & cur = stmts.item(idx1);
  6546. bool curIsThor = cur.isThorQuery();
  6547. unsigned insertPos = sorted.ordinality();
  6548. ForEachItemInRev(idx2, sorted)
  6549. {
  6550. StatementInfo & compare = sorted.item(idx2);
  6551. if (compare.isThorQuery() == curIsThor)
  6552. {
  6553. insertPos = idx2+1;
  6554. break;
  6555. }
  6556. if (!compare.canSwapOrder(cur))
  6557. break;
  6558. }
  6559. sorted.add(cur, insertPos);
  6560. }
  6561. //Finally see if there is any merit in moving an initial block of thor queries down to
  6562. //merge with a subsequent one.
  6563. StatementInfo & first = sorted.item(0);
  6564. if (first.isThorQuery())
  6565. {
  6566. unsigned max = sorted.ordinality();
  6567. unsigned numToMove;
  6568. for (numToMove = 1; numToMove < max; numToMove++)
  6569. {
  6570. if (!(sorted.item(numToMove)).isThorQuery())
  6571. break;
  6572. }
  6573. for (unsigned i=numToMove; i < max; i++)
  6574. {
  6575. StatementInfo & compare = sorted.item(i);
  6576. if (compare.isThorQuery())
  6577. {
  6578. for (unsigned j=0; j < numToMove; j++)
  6579. sorted.rotateL(0, i-1);
  6580. break;
  6581. }
  6582. for (unsigned j=0; j < numToMove; j++)
  6583. {
  6584. if (!compare.canSwapOrder(sorted.item(j)))
  6585. {
  6586. i = max - 1;
  6587. break;
  6588. }
  6589. }
  6590. }
  6591. }
  6592. in.kill();
  6593. ForEachItemIn(idxSorted, sorted)
  6594. {
  6595. StatementInfo & cur = sorted.item(idxSorted);
  6596. in.append(*cur.expr.getLink());
  6597. }
  6598. }
  6599. //------------------------------------------------------------------------
  6600. //We will generate better code if conditional statements precede unconditional statements because globals can
  6601. //be commoned up better.
  6602. bool moveUnconditionalEarlier(HqlExprArray & in)
  6603. {
  6604. //Gather information about the statements...
  6605. unsigned numConditionals = 0;
  6606. unsigned firstConditional = NotFound;
  6607. bool couldImprove = false;
  6608. CIArrayOf<StatementInfo> stmts;
  6609. ForEachItemIn(idx, in)
  6610. {
  6611. StatementInfo & cur = *new StatementInfo(&in.item(idx));
  6612. stmts.append(cur);
  6613. if (cur.isConditional())
  6614. {
  6615. if (numConditionals == 0)
  6616. firstConditional = idx;
  6617. numConditionals++;
  6618. }
  6619. else if (numConditionals)
  6620. couldImprove = true;
  6621. }
  6622. //If no unconditionals follow a conditional, and no conditionals to be combined, then keep in the same order...
  6623. if (!couldImprove && numConditionals <= 1)
  6624. return false;
  6625. //For each block of unconditional statements which follow a conditional statement, see if they can be moved over the conditional statements.
  6626. //(copies with no overhead if couldImprove is false)
  6627. CICopyArrayOf<StatementInfo> sorted;
  6628. unsigned max = stmts.ordinality();
  6629. for (unsigned idx1 = 0; idx1 < max;)
  6630. {
  6631. StatementInfo & cur = stmts.item(idx1);
  6632. bool isConditional = cur.isConditional();
  6633. unsigned cnt = 1;
  6634. if (isConditional || idx1 < firstConditional)
  6635. {
  6636. sorted.append(cur);
  6637. }
  6638. else
  6639. {
  6640. //calculate the number of contiguous unconditional statements
  6641. for (cnt=1; idx1+cnt < max; cnt++)
  6642. {
  6643. if (stmts.item(idx1+cnt).isConditional())
  6644. break;
  6645. }
  6646. unsigned movePosition = 0;
  6647. for (unsigned iBlock = 0; iBlock < cnt; iBlock++)
  6648. {
  6649. StatementInfo & curBlock = stmts.item(idx1+iBlock);
  6650. unsigned bestPosition = NotFound; // best position to add block.
  6651. unsigned prev = idx1;
  6652. while (prev-- > firstConditional)
  6653. {
  6654. StatementInfo & compare = sorted.item(prev);
  6655. if (!compare.canSwapOrder(curBlock))
  6656. break;
  6657. if (prev == firstConditional)
  6658. bestPosition = prev;
  6659. else if (compare.isConditional() && !sorted.item(prev-1).isConditional())
  6660. bestPosition = prev;
  6661. }
  6662. if (bestPosition == NotFound)
  6663. {
  6664. //can't move this element in the block => append the items to the list.
  6665. movePosition = sorted.ordinality();
  6666. break;
  6667. }
  6668. //Intersection of the best positions to provide earliest we can move the block
  6669. if (movePosition < bestPosition)
  6670. movePosition = bestPosition;
  6671. }
  6672. for (unsigned iBlock2 = 0; iBlock2 < cnt; iBlock2++)
  6673. sorted.add(stmts.item(idx1+iBlock2), movePosition+iBlock2);
  6674. }
  6675. idx1 += cnt;
  6676. }
  6677. //See if moving conditional statements could make some conditions next to each other
  6678. //Now see if any of the conditional statements can be combined.
  6679. //Finally replace the array
  6680. in.kill();
  6681. ForEachItemIn(idxSorted, sorted)
  6682. {
  6683. StatementInfo & cur = (StatementInfo &)sorted.item(idxSorted);
  6684. in.append(*cur.expr.getLink());
  6685. }
  6686. return true;
  6687. }
  6688. //------------------------------------------------------------------------
  6689. static void mergeThorGraphs(HqlExprArray & exprs, bool resourceConditionalActions, bool resourceSequential);
  6690. static IHqlExpression * mergeThorGraphs(IHqlExpression * expr, bool resourceConditionalActions, bool resourceSequential)
  6691. {
  6692. HqlExprArray args;
  6693. expr->unwindList(args, no_actionlist);
  6694. mergeThorGraphs(args, resourceConditionalActions, resourceSequential);
  6695. return createActionList(args);
  6696. }
  6697. static void mergeThorGraphs(HqlExprArray & exprs, bool resourceConditionalActions, bool resourceSequential)
  6698. {
  6699. HqlExprArray thorActions;
  6700. HqlExprArray combined;
  6701. ForEachItemIn(idx, exprs)
  6702. {
  6703. IHqlExpression * original = &exprs.item(idx);
  6704. LinkedHqlExpr cur = original;
  6705. const node_operator op = cur->getOperator();
  6706. switch (op)
  6707. {
  6708. case no_compound:
  6709. {
  6710. OwnedHqlExpr replace = mergeThorGraphs(cur->queryChild(0), resourceConditionalActions, resourceSequential);
  6711. cur.setown(replaceChild(cur, 0, replace));
  6712. break;
  6713. }
  6714. case no_if:
  6715. if (cur->isAction())
  6716. {
  6717. IHqlExpression * left = cur->queryChild(1);
  6718. IHqlExpression * right = cur->queryChild(2);
  6719. OwnedHqlExpr newLeft = mergeThorGraphs(left, resourceConditionalActions, resourceSequential);
  6720. OwnedHqlExpr newRight = right ? mergeThorGraphs(right, resourceConditionalActions, resourceSequential) : NULL;
  6721. if (left != newLeft || right != newRight)
  6722. {
  6723. HqlExprArray args;
  6724. unwindChildren(args, cur);
  6725. //Not sure about this - the test condition may not be evaluatable inside thor
  6726. if (resourceConditionalActions && ((newLeft->getOperator() == no_thor) && (!newRight || newRight->getOperator() == no_thor)))
  6727. {
  6728. args.replace(*LINK(newLeft->queryChild(0)), 1);
  6729. if (newRight)
  6730. args.replace(*LINK(newRight->queryChild(0)), 2);
  6731. cur.setown(createValue(no_thor, makeVoidType(), cur->clone(args)));
  6732. }
  6733. else
  6734. {
  6735. args.replace(*LINK(newLeft), 1);
  6736. if (newRight)
  6737. args.replace(*LINK(newRight), 2);
  6738. cur.setown(cur->clone(args));
  6739. }
  6740. }
  6741. }
  6742. break;
  6743. case no_parallel:
  6744. if (false)
  6745. {
  6746. HqlExprArray args;
  6747. bool allThor = true;
  6748. ForEachChild(i, cur)
  6749. {
  6750. IHqlExpression * merged = mergeThorGraphs(cur->queryChild(i), resourceConditionalActions, resourceSequential);
  6751. args.append(*merged);
  6752. if (merged->getOperator() != no_thor)
  6753. allThor = false;
  6754. }
  6755. if (allThor)
  6756. {
  6757. ForEachItemIn(i, args)
  6758. args.replace(*LINK(args.item(i).queryChild(0)), i);
  6759. cur.setown(cur->clone(args));
  6760. cur.setown(createValue(no_thor, makeVoidType(), cur.getClear()));
  6761. }
  6762. else
  6763. cur.setown(cur->clone(args));
  6764. break;
  6765. }
  6766. //fall through
  6767. case no_actionlist:
  6768. case no_orderedactionlist:
  6769. {
  6770. HqlExprArray args;
  6771. cur->unwindList(args, op);
  6772. mergeThorGraphs(args, resourceConditionalActions, resourceSequential);
  6773. cur.setown(cur->clone(args));
  6774. break;
  6775. }
  6776. case no_sequential:
  6777. {
  6778. HqlExprArray args;
  6779. bool allThor = true;
  6780. ForEachChild(i, cur)
  6781. {
  6782. IHqlExpression * merged = mergeThorGraphs(cur->queryChild(i), resourceConditionalActions, resourceSequential);
  6783. args.append(*merged);
  6784. if (merged->getOperator() != no_thor)
  6785. allThor = false;
  6786. }
  6787. if (resourceSequential && allThor)
  6788. {
  6789. ForEachItemIn(i, args)
  6790. args.replace(*LINK(args.item(i).queryChild(0)), i);
  6791. cur.setown(cur->clone(args));
  6792. cur.setown(createValue(no_thor, makeVoidType(), cur.getClear()));
  6793. }
  6794. else
  6795. cur.setown(cur->clone(args));
  6796. break;
  6797. }
  6798. case no_ensureresult:
  6799. {
  6800. HqlExprArray args;
  6801. unwindChildren(args, cur);
  6802. args.replace(*mergeThorGraphs(cur->queryChild(0), resourceConditionalActions, resourceSequential), 0);
  6803. cur.setown(cloneOrLink(cur, args));
  6804. break;
  6805. }
  6806. }
  6807. if (cur->getOperator() == no_thor)
  6808. {
  6809. thorActions.append(*LINK(cur->queryChild(0)));
  6810. }
  6811. else
  6812. {
  6813. if (thorActions.ordinality())
  6814. {
  6815. combined.append(*createValue(no_thor, makeVoidType(), createActionList(thorActions)));
  6816. thorActions.kill();
  6817. }
  6818. combined.append(*cur.getClear());
  6819. }
  6820. }
  6821. if (thorActions.ordinality())
  6822. combined.append(*createValue(no_thor, makeVoidType(), createActionList(thorActions)));
  6823. replaceArray(exprs, combined);
  6824. }
  6825. void mergeThorGraphs(WorkflowItem & workflow, bool resourceConditionalActions, bool resourceSequential)
  6826. {
  6827. groupThorGraphs(workflow.queryExprs());
  6828. mergeThorGraphs(workflow.queryExprs(), resourceConditionalActions, resourceSequential);
  6829. }
  6830. //------------------------------------------------------------------------
  6831. inline bool isTypeToHoist(ITypeInfo * type)
  6832. {
  6833. return isSingleValuedType(type);// || (type && type->getTypeCode() == type_set);
  6834. }
  6835. //Currently we do not create globals for common sub expressions between global expressions
  6836. //I suspect a better solution is to create them much later in the resourcing, but keep the
  6837. //code for the moment for reference.
  6838. static const bool createGlobalForGlobalCse = false;
  6839. static HqlTransformerInfo scalarGlobalTransformerInfo("ScalarGlobalTransformer");
  6840. ScalarGlobalTransformer::ScalarGlobalTransformer(HqlCppTranslator & _translator)
  6841. : HoistingHqlTransformer(scalarGlobalTransformerInfo, CTFtraverseallnodes), translator(_translator)
  6842. {
  6843. isGlobal = true;
  6844. isOkToHoist = true;
  6845. }
  6846. bool ScalarGlobalTransformer::isCandidate(IHqlExpression * expr)
  6847. {
  6848. // Commented line has problems with SELF used in HOLE definition, and explosion in thumphrey7 etc.
  6849. //return !expr->isConstant() && !isContextDependent(expr) && expr->isPure() && expr->isIndependentOfScope();
  6850. if (!containsAnyDataset(expr) && !expr->isConstant() && !isContextDependent(expr) && expr->isPure() && expr->isIndependentOfScope())
  6851. {
  6852. node_operator op = expr->getOperator();
  6853. switch (op)
  6854. {
  6855. case no_nofold:
  6856. case no_nocombine:
  6857. //try to hoist the thing that is nofolded instead
  6858. return false;
  6859. default:
  6860. return true;
  6861. }
  6862. }
  6863. return false;
  6864. }
  6865. bool ScalarGlobalTransformer::canCreateCandidate(IHqlExpression * expr)
  6866. {
  6867. ITypeInfo * type = expr->queryType();
  6868. if (isTypeToHoist(type))
  6869. {
  6870. if (canCreateTemporary(expr))
  6871. return true;
  6872. }
  6873. return false;
  6874. }
  6875. void ScalarGlobalTransformer::analyseExpr(IHqlExpression * expr)
  6876. {
  6877. ScalarGlobalExtra * extra = queryBodyExtra(expr);
  6878. bool wasGlobal = isGlobal;
  6879. bool wasOkToHoist = isOkToHoist;
  6880. analyseThis(expr);
  6881. if (++extra->numUses > 1)
  6882. {
  6883. //Already creating an alias => no need to look any further
  6884. if (extra->createGlobal)
  6885. return;
  6886. //If we have already visited this node globally then any elements that are candidates will already be tagged
  6887. if (extra->visitedAllowHoist)
  6888. return;
  6889. if (createGlobalForGlobalCse)
  6890. {
  6891. //Option1: no_globalscope is added for cses between no_globalscopes.
  6892. extra->visitedAllowHoist = true;
  6893. //Otherwise it may be worth creating a new candidate for an expression used from two places
  6894. isOkToHoist = true;
  6895. }
  6896. else
  6897. {
  6898. //Option2: Don't create a common node for cses between no_globalscopes
  6899. if (!isGlobal)
  6900. return;
  6901. extra->visitedAllowHoist = true;
  6902. }
  6903. }
  6904. else
  6905. extra->visitedAllowHoist = isOkToHoist;
  6906. if (isGlobal)
  6907. extra->isLocal = false;
  6908. doAnalyseExpr(expr);
  6909. isGlobal = wasGlobal;
  6910. isOkToHoist = wasOkToHoist;
  6911. }
  6912. void ScalarGlobalTransformer::doAnalyseExpr(IHqlExpression * expr)
  6913. {
  6914. switch (expr->getOperator())
  6915. {
  6916. case no_attr:
  6917. case no_constant:
  6918. case no_attr_link:
  6919. case no_null:
  6920. case no_all:
  6921. case no_funcdef:
  6922. case no_assert:
  6923. return;
  6924. case no_persist_check:
  6925. //No point spotting global within this since it will not create a subquery..
  6926. return;
  6927. case no_attr_expr:
  6928. {
  6929. IAtom * name = expr->queryName();
  6930. if ((name == _selectors_Atom) || (name == keyedAtom))
  6931. return;
  6932. analyseChildren(expr);
  6933. return;
  6934. }
  6935. case no_getresult:
  6936. case no_libraryinput:
  6937. case no_setresult:
  6938. queryBodyExtra(expr)->alreadyGlobal = true;
  6939. break;
  6940. case no_globalscope:
  6941. queryBodyExtra(expr)->alreadyGlobal = true; // don't tag again - even if opt flag is present
  6942. queryBodyExtra(expr->queryChild(0))->alreadyGlobal = true;
  6943. break;
  6944. case no_ensureresult:
  6945. //When transformed, transform tends to be called on the children only
  6946. //=> Stop the argument being turned unnecessarily to a global temporary
  6947. queryBodyExtra(expr)->alreadyGlobal = true;
  6948. queryBodyExtra(expr->queryChild(0))->alreadyGlobal = true;
  6949. break;
  6950. }
  6951. if (isOkToHoist && isCandidate(expr))
  6952. {
  6953. if (canCreateCandidate(expr))
  6954. {
  6955. queryBodyExtra(expr)->createGlobal = true;
  6956. isGlobal = false;
  6957. isOkToHoist = false;
  6958. }
  6959. }
  6960. HoistingHqlTransformer::doAnalyseExpr(expr);
  6961. }
  6962. /*
  6963. Try and decide what is trivial enough to serialise, and what should remain. It is more trial an error than particularly logical
  6964. o Better to store smaller objects because they will serialize smaller.
  6965. o If something is used more than once then probably worth serializing regardless - since calculation will be commoned up.
  6966. o Don't really want to serialize 'x' and 'x <> ''' to the same function - but much better to serialize 'x <> ''' rather than 'x' if only one used.
  6967. */
  6968. bool ScalarGlobalTransformer::isComplex(IHqlExpression * expr, bool checkGlobal)
  6969. {
  6970. ScalarGlobalExtra * extra = queryBodyExtra(expr);
  6971. if (checkGlobal)
  6972. {
  6973. //If something else has turned this into a global then no point.
  6974. if (extra->alreadyGlobal || extra->createGlobal)
  6975. return false;
  6976. }
  6977. switch (expr->getOperator())
  6978. {
  6979. case no_workunit_dataset:
  6980. case no_getresult:
  6981. return expr->hasAttribute(wuidAtom);
  6982. case no_constant:
  6983. case no_globalscope:
  6984. case no_libraryinput:
  6985. case no_clustersize:
  6986. case no_nothor:
  6987. return false;
  6988. case no_cast:
  6989. case no_implicitcast:
  6990. //serialize if the cast reduces the size of the item, otherwise check argument.
  6991. if (expr->queryType()->getSize() <= expr->queryChild(0)->queryType()->getSize())
  6992. return true;
  6993. //If used a lot then save lots of duplicated work.
  6994. if (extra->numUses > 2)
  6995. return true;
  6996. break;
  6997. case no_eq:
  6998. case no_ne:
  6999. case no_lt:
  7000. case no_gt:
  7001. case no_le:
  7002. case no_ge:
  7003. //Accessed more than once-> probably worth commoning up
  7004. if (extra->numUses > 1)
  7005. return true;
  7006. //Some possible improvements. Probably better to implement these and all hoisting in the resourcing phase.
  7007. //MORE: Probably more efficient to calculate a boolean and serialize just that - if it is used in a dataset context
  7008. //MORE: If an argument is a string probably worth commoning up - it depends on whether the string is also used in
  7009. // the same context.
  7010. break;
  7011. //f[1..length(trim(x))] = x is very common, and if the length(trim)) was serialized separately then
  7012. //the generated code would be worse.
  7013. case no_trim:
  7014. case no_charlen:
  7015. case no_sorted:
  7016. case no_not:
  7017. break;
  7018. case no_substring:
  7019. //single character substring - don't create separate items just for this, since likely to have many of them.
  7020. if (!expr->queryChild(1)->queryValue())
  7021. return true;
  7022. break;
  7023. case no_likely:
  7024. case no_unlikely:
  7025. break;
  7026. default:
  7027. if (expr->isConstant())
  7028. return false;
  7029. return true;
  7030. }
  7031. ForEachChild(i, expr)
  7032. {
  7033. if (isComplex(expr->queryChild(i), true))
  7034. return true;
  7035. }
  7036. return false;
  7037. }
  7038. IHqlExpression * ScalarGlobalTransformer::createTransformed(IHqlExpression * expr)
  7039. {
  7040. IHqlExpression * ret = queryTransformAnnotation(expr);
  7041. if (ret)
  7042. return ret;
  7043. OwnedHqlExpr transformed = HoistingHqlTransformer::createTransformed(expr);
  7044. switch (transformed->getOperator())
  7045. {
  7046. case no_setresult:
  7047. case no_globalscope:
  7048. case no_ensureresult:
  7049. {
  7050. //Don't prevent no_globalscope being added in analyse because other expression may require
  7051. //it to be made global. However if a no_globalscope has been added to the child, remove it.
  7052. IHqlExpression * child = transformed->queryChild(0);
  7053. if (child->getOperator() == no_globalscope)
  7054. {
  7055. if (expr->queryChild(0)->getOperator() != no_globalscope)
  7056. {
  7057. HqlExprArray args;
  7058. args.append(*LINK(child->queryChild(0)));
  7059. unwindChildren(args, transformed, 1);
  7060. return transformed->clone(args);
  7061. }
  7062. }
  7063. break;
  7064. }
  7065. }
  7066. ScalarGlobalExtra * extra = queryBodyExtra(expr);
  7067. if (extra->createGlobal)
  7068. {
  7069. if (!extra->alreadyGlobal && isComplex(expr, false))
  7070. {
  7071. #ifdef _DEBUG
  7072. translator.traceExpression("Mark as global", expr);
  7073. #endif
  7074. //mark as global, so isComplex() can take it into account.
  7075. extra->alreadyGlobal = true;
  7076. if (expr->getOperator() == no_createset)
  7077. transformed.setown(projectCreateSetDataset(transformed));
  7078. return createValue(no_globalscope, transformed->getType(), LINK(transformed));
  7079. }
  7080. else
  7081. extra->createGlobal = false;
  7082. }
  7083. return transformed.getClear();
  7084. }
  7085. //------------------------------------------------------------------------
  7086. static HqlTransformerInfo explicitGlobalTransformerInfo("ExplicitGlobalTransformer");
  7087. ExplicitGlobalTransformer::ExplicitGlobalTransformer(IWorkUnit * _wu, HqlCppTranslator & _translator)
  7088. : HoistingHqlTransformer(explicitGlobalTransformerInfo, CTFnoteifactions|CTFtraverseallnodes), translator(_translator)
  7089. {
  7090. wu = _wu;
  7091. isRoxie = (translator.getTargetClusterType() == RoxieCluster);
  7092. seenGlobalScope = false;
  7093. seenLocalGlobalScope = false;
  7094. }
  7095. void ExplicitGlobalTransformer::doAnalyseExpr(IHqlExpression * expr)
  7096. {
  7097. node_operator op = expr->getOperator();
  7098. switch (op)
  7099. {
  7100. case no_nothor:
  7101. if (expr->isAction())
  7102. break;
  7103. //fall through
  7104. case no_globalscope:
  7105. //Try and avoid transforms (especially nested ones) as much as possible.
  7106. seenGlobalScope = true;
  7107. //If local attribute is present on a global, then an independent transform may cause an extra
  7108. //transformation because it may become unconditional, when previously conditional
  7109. if (expr->hasAttribute(localAtom))
  7110. seenLocalGlobalScope = true;
  7111. break;
  7112. }
  7113. HoistingHqlTransformer::doAnalyseExpr(expr);
  7114. }
  7115. IHqlExpression * ExplicitGlobalTransformer::createTransformed(IHqlExpression * expr)
  7116. {
  7117. if (expr->isConstant())
  7118. return LINK(expr);
  7119. IHqlExpression * ret = queryTransformAnnotation(expr);
  7120. if (ret)
  7121. return ret;
  7122. OwnedHqlExpr transformed = HoistingHqlTransformer::createTransformed(expr);
  7123. node_operator op = expr->getOperator();
  7124. switch (op)
  7125. {
  7126. case no_nothor:
  7127. if (transformed->isAction())
  7128. break;
  7129. //fall through
  7130. case no_globalscope:
  7131. {
  7132. IHqlExpression * value = transformed->queryChild(0);
  7133. if (expr->hasAttribute(optAtom))
  7134. {
  7135. if (!isIndependentOfScope(value))
  7136. return LINK(value);
  7137. }
  7138. if (!expr->hasAttribute(localAtom) || isUsedUnconditionally(expr))
  7139. {
  7140. if (!isIndependentOfScope(value))
  7141. {
  7142. IHqlExpression * symbol = queryActiveSymbol();
  7143. StringBuffer s;
  7144. if (symbol && symbol->queryBody() == expr)
  7145. s.appendf(" '%s'", str(symbol->queryName()));
  7146. else
  7147. {
  7148. s.append(" ").append(getOpString(value->getOperator()));
  7149. if (symbol)
  7150. s.append(" in ").append(symbol->queryName());
  7151. }
  7152. if (op == no_nothor)
  7153. translator.reportWarning(CategoryMistake, SeverityUnknown, queryActiveLocation(expr), ECODETEXT(HQLWRN_NoThorContextDependent), s.str());
  7154. else
  7155. translator.reportWarning(CategoryMistake, SeverityUnknown, queryActiveLocation(expr), ECODETEXT(HQLWRN_GlobalDoesntSeemToBe), s.str());
  7156. }
  7157. if (value->getOperator() == no_createset)
  7158. {
  7159. OwnedHqlExpr createset = projectCreateSetDataset(value);
  7160. IHqlExpression * ds = createset->queryChild(0);
  7161. HqlExprArray outArgs, setArgs;
  7162. outArgs.append(*LINK(ds));
  7163. outArgs.append(*createAttribute(sequenceAtom, getLocalSequenceNumber()));
  7164. outArgs.append(*createAttribute(namedAtom, createNextStringValue(value)));
  7165. IHqlExpression * setResult = createValue(no_output, makeVoidType(), outArgs);
  7166. appendToTarget(*setResult);
  7167. transformed.setown(createGetResultFromSetResult(setResult, expr->queryType()));
  7168. }
  7169. else
  7170. {
  7171. GlobalAttributeInfo info("jobtemp::global","gl", transformed);
  7172. if (op == no_nothor)
  7173. info.extractGlobal(NULL, RoxieCluster);
  7174. else
  7175. info.extractGlobal(transformed, translator.getTargetClusterType());
  7176. OwnedHqlExpr getResult, setResult;
  7177. info.checkFew(translator);
  7178. info.splitGlobalDefinition(transformed->queryType(), value, wu, setResult, &getResult, isRoxie);
  7179. if (op == no_nothor)
  7180. setResult.setown(createValue(no_nothor, makeVoidType(), LINK(setResult)));
  7181. IHqlExpression * cluster = queryRealChild(transformed, 1);
  7182. if (cluster && !isBlankString(cluster))
  7183. setResult.setown(createValue(no_cluster, makeVoidType(), LINK(setResult), LINK(cluster)));
  7184. appendToTarget(*setResult.getClear());
  7185. transformed.setown(getResult.getClear());
  7186. }
  7187. break;
  7188. }
  7189. }
  7190. }
  7191. return transformed.getClear();
  7192. }
  7193. static HqlTransformerInfo optGlobalTransformerInfo("OptGlobalTransformer");
  7194. OptGlobalTransformer::OptGlobalTransformer() : NewHqlTransformer(optGlobalTransformerInfo)
  7195. {
  7196. seenOptGlobal = false;
  7197. }
  7198. void OptGlobalTransformer::analyseExpr(IHqlExpression * expr)
  7199. {
  7200. if (alreadyVisited(expr))
  7201. return;
  7202. node_operator op = expr->getOperator();
  7203. switch (op)
  7204. {
  7205. case no_globalscope:
  7206. if (expr->hasAttribute(optAtom))
  7207. seenOptGlobal = true;
  7208. break;
  7209. }
  7210. NewHqlTransformer::analyseExpr(expr);
  7211. }
  7212. IHqlExpression * OptGlobalTransformer::createTransformed(IHqlExpression * expr)
  7213. {
  7214. OwnedHqlExpr transformed = NewHqlTransformer::createTransformed(expr);
  7215. node_operator op = transformed->getOperator();
  7216. switch (op)
  7217. {
  7218. case no_globalscope:
  7219. {
  7220. if (transformed->hasAttribute(optAtom))
  7221. {
  7222. IHqlExpression * value = transformed->queryChild(0);
  7223. if (!isIndependentOfScope(value))
  7224. return LINK(value);
  7225. return removeAttribute(transformed, optAtom);
  7226. }
  7227. break;
  7228. }
  7229. }
  7230. return transformed.getClear();
  7231. }
  7232. //------------------------------------------------------------------------
  7233. static HqlTransformerInfo scopeIndependentActionCheckerInfo("ScopeIndependentActionChecker");
  7234. class ScopeIndependentActionChecker : public NewHqlTransformer
  7235. {
  7236. public:
  7237. ScopeIndependentActionChecker(HqlCppTranslator & _translator) : NewHqlTransformer(scopeIndependentActionCheckerInfo), translator(_translator)
  7238. {
  7239. }
  7240. protected:
  7241. void analyseExpr(IHqlExpression * expr)
  7242. {
  7243. switch (expr->getOperator())
  7244. {
  7245. case no_parallel:
  7246. case no_if:
  7247. case no_sequential:
  7248. case no_compound:
  7249. case no_actionlist:
  7250. case no_orderedactionlist:
  7251. NewHqlTransformer::analyseExpr(expr);
  7252. break;
  7253. case no_output:
  7254. if (!isIndependentOfScope(expr))
  7255. {
  7256. IHqlExpression * filename = queryRealChild(expr, 2);
  7257. if (!filename)
  7258. filename = expr->queryAttribute(namedAtom);
  7259. StringBuffer s;
  7260. if (filename)
  7261. getExprECL(filename, s);
  7262. translator.WARNINGAT1(CategoryMistake, queryActiveLocation(expr), HQLWRN_OutputDependendOnScope, s.str());
  7263. #if 0
  7264. checkIndependentOfScope(expr);
  7265. #endif
  7266. }
  7267. break;
  7268. default:
  7269. if (!isIndependentOfScope(expr))
  7270. {
  7271. //There appears to be partial support for functions generated as workflow items - which are not
  7272. //scope independent since they depend on their paramaters => don't complain about them
  7273. if (expr->getOperator() != no_return_stmt)
  7274. translator.WARNINGAT(CategoryMistake, queryActiveLocation(expr), HQLWRN_GlobalActionDependendOnScope);
  7275. #if 0
  7276. checkIndependentOfScope(expr);
  7277. #endif
  7278. }
  7279. break;
  7280. }
  7281. }
  7282. private:
  7283. HqlCppTranslator & translator;
  7284. };
  7285. void checkGlobalActionsIndependentOfScope(HqlCppTranslator & translator, const HqlExprArray & exprs)
  7286. {
  7287. ScopeIndependentActionChecker checker(translator);
  7288. checker.analyseArray(exprs, 0);
  7289. }
  7290. //------------------------------------------------------------------------
  7291. IHqlDataset * queryRootDataset(IHqlExpression * dataset)
  7292. {
  7293. return dataset->queryDataset()->queryRootTable();
  7294. }
  7295. //roxie only executes outputs to temporaries if they are required, or if not all references are from within the graph
  7296. //therefore, there is no need to special case if actions. Thor on the other hand will cause it to be executed unnecessarily.
  7297. static HqlTransformerInfo newScopeMigrateTransformerInfo("NewScopeMigrateTransformer");
  7298. NewScopeMigrateTransformer::NewScopeMigrateTransformer(IWorkUnit * _wu, HqlCppTranslator & _translator)
  7299. : HoistingHqlTransformer(newScopeMigrateTransformerInfo, 0), translator(_translator)
  7300. {
  7301. wu = _wu;
  7302. isRoxie = translator.targetRoxie();
  7303. if (!isRoxie && !_translator.queryOptions().resourceConditionalActions)
  7304. setFlags(CTFnoteifactions);
  7305. minimizeWorkunitTemporaries = translator.queryOptions().minimizeWorkunitTemporaries;
  7306. #ifdef REMOVE_GLOBAL_ANNOTATION
  7307. activityDepth = 0; // should be 0 to actually have any effect - but causes problems...
  7308. #else
  7309. activityDepth = 999; // should be 0 to actually have any effect - but causes problems...
  7310. #endif
  7311. }
  7312. void NewScopeMigrateTransformer::analyseExpr(IHqlExpression * expr)
  7313. {
  7314. ScopeMigrateInfo * extra = queryBodyExtra(expr);
  7315. if (activityDepth > extra->maxActivityDepth)
  7316. {
  7317. if (extra->maxActivityDepth == 0)
  7318. extra->setUnvisited(); // so we walk children again
  7319. extra->maxActivityDepth = activityDepth;
  7320. }
  7321. unsigned savedActivityDepth = activityDepth;
  7322. node_operator op = expr->getOperator();
  7323. switch (op)
  7324. {
  7325. case NO_AGGREGATE:
  7326. case no_createset:
  7327. case NO_ACTION_REQUIRES_GRAPH:
  7328. case no_extractresult:
  7329. case no_distributer:
  7330. case no_within:
  7331. case no_notwithin:
  7332. case no_soapaction_ds:
  7333. case no_returnresult:
  7334. activityDepth++;
  7335. break;
  7336. case no_setresult:
  7337. if (expr->queryChild(0)->isDataset())
  7338. activityDepth++;
  7339. break;
  7340. case no_select:
  7341. if (expr->hasAttribute(newAtom))
  7342. activityDepth++;
  7343. break;
  7344. }
  7345. HoistingHqlTransformer::analyseExpr(expr);
  7346. activityDepth = savedActivityDepth;
  7347. }
  7348. IHqlExpression * NewScopeMigrateTransformer::hoist(IHqlExpression * expr, IHqlExpression * hoisted)
  7349. {
  7350. if (minimizeWorkunitTemporaries)
  7351. return createWrapper(no_globalscope, LINK(hoisted));
  7352. IHqlExpression * setResult = createSetResult(hoisted);
  7353. appendToTarget(*setResult);
  7354. return createGetResultFromSetResult(setResult);
  7355. }
  7356. IHqlExpression * NewScopeMigrateTransformer::createTransformed(IHqlExpression * expr)
  7357. {
  7358. if (expr->isConstant())
  7359. return LINK(expr);
  7360. IHqlExpression * ret = queryTransformAnnotation(expr);
  7361. if (ret)
  7362. return ret;
  7363. OwnedHqlExpr transformed = HoistingHqlTransformer::createTransformed(expr);
  7364. ScopeMigrateInfo * extra = queryBodyExtra(expr);
  7365. node_operator op = expr->getOperator();
  7366. switch (op)
  7367. {
  7368. case no_createset:
  7369. {
  7370. if (isUsedUnconditionally(expr))
  7371. {
  7372. if (isIndependentOfScope(transformed) && !isContextDependent(expr))
  7373. {
  7374. OwnedHqlExpr createset = projectCreateSetDataset(transformed);
  7375. if (minimizeWorkunitTemporaries)
  7376. return createWrapper(no_globalscope, LINK(createset));
  7377. //MORE: This is only temporary until child datasets come into existence, then it will need improving
  7378. //Save it as a temporary dataset in the wu, and retrieve it as a getresult(set)
  7379. IHqlExpression * ds = createset->queryChild(0);
  7380. HqlExprArray outArgs, setArgs;
  7381. outArgs.append(*LINK(ds));
  7382. outArgs.append(*createAttribute(sequenceAtom, getLocalSequenceNumber()));
  7383. outArgs.append(*createAttribute(namedAtom, createNextStringValue(expr)));
  7384. IHqlExpression * setResult = createValue(no_output, makeVoidType(), outArgs);
  7385. appendToTarget(*setResult);
  7386. transformed.setown(createGetResultFromSetResult(setResult, expr->queryType()));
  7387. }
  7388. }
  7389. break;
  7390. }
  7391. case no_select:
  7392. {
  7393. bool isNew;
  7394. IHqlExpression * row = querySelectorDataset(transformed, isNew);
  7395. if (isNew)
  7396. {
  7397. if (isUsedUnconditionally(expr))
  7398. {
  7399. if (extra->maxActivityDepth != 0)
  7400. {
  7401. node_operator rowOp = row->getOperator();
  7402. if (rowOp == no_selectnth)
  7403. {
  7404. node_operator dsOp = row->queryChild(0)->getOperator();
  7405. if ((dsOp == no_workunit_dataset) || (dsOp == no_inlinetable))
  7406. break;
  7407. }
  7408. if (rowOp == no_createrow || rowOp == no_getresult)
  7409. break;
  7410. if (!isInlineTrivialDataset(row) && !isContextDependent(row) && !transformed->isDataset() && !transformed->isDictionary())
  7411. {
  7412. if (isIndependentOfScope(row))
  7413. return hoist(expr, transformed);
  7414. }
  7415. }
  7416. }
  7417. }
  7418. }
  7419. break;
  7420. case NO_AGGREGATE:
  7421. {
  7422. if (isUsedUnconditionally(expr))
  7423. {
  7424. if (extra->maxActivityDepth != 0)
  7425. {
  7426. IHqlExpression * datasetExpr = transformed->queryChild(0);
  7427. IHqlDataset * rootDataset = queryRootDataset(datasetExpr);
  7428. if (!rootDataset)
  7429. {
  7430. //Something like a+b+c
  7431. rootDataset = datasetExpr->queryDataset()->queryTable();
  7432. if (!rootDataset)
  7433. break;
  7434. }
  7435. //Don't do anything with child datasets....
  7436. IHqlExpression * rootDatasetExpr = queryExpression(rootDataset);
  7437. node_operator rootOp = rootDatasetExpr->getOperator();
  7438. if ((rootOp == no_select) || (rootOp == no_field))
  7439. break;
  7440. if (isIndependentOfScope(datasetExpr) && !isContextDependent(expr))
  7441. {
  7442. return hoist(expr, transformed);
  7443. }
  7444. }
  7445. }
  7446. break;
  7447. }
  7448. case no_globalscope:
  7449. if (isRedundantGlobalScope(transformed))
  7450. return LINK(transformed->queryChild(0));
  7451. break;
  7452. }
  7453. return transformed.getClear();
  7454. }
  7455. void migrateExprToNaturalLevel(WorkflowItem & cur, IWorkUnit * wu, HqlCppTranslator & translator)
  7456. {
  7457. const HqlCppOptions & options = translator.queryOptions();
  7458. HqlExprArray & exprs = cur.queryExprs();
  7459. if (translator.queryOptions().moveUnconditionalActions)
  7460. moveUnconditionalEarlier(exprs);
  7461. translator.checkNormalized(exprs);
  7462. if (options.hoistSimpleGlobal)
  7463. {
  7464. ScalarGlobalTransformer transformer(translator);
  7465. HqlExprArray results;
  7466. transformer.analyseArray(exprs, 0);
  7467. transformer.transformRoot(exprs, results);
  7468. replaceArray(exprs, results);
  7469. translator.checkNormalized(exprs);
  7470. }
  7471. translator.traceExpressions("m0", exprs);
  7472. checkGlobalActionsIndependentOfScope(translator, exprs);
  7473. if (options.workunitTemporaries)
  7474. {
  7475. ExplicitGlobalTransformer transformer(wu, translator);
  7476. transformer.analyseArray(exprs, 0);
  7477. if (transformer.needToTransform())
  7478. {
  7479. HqlExprArray results;
  7480. transformer.transformRoot(exprs, results);
  7481. replaceArray(exprs, results);
  7482. }
  7483. }
  7484. else
  7485. {
  7486. OptGlobalTransformer transformer;
  7487. transformer.analyseArray(exprs, 0);
  7488. if (transformer.needToTransform())
  7489. {
  7490. HqlExprArray results;
  7491. transformer.transformRoot(exprs, results);
  7492. replaceArray(exprs, results);
  7493. }
  7494. }
  7495. translator.checkNormalized(exprs);
  7496. translator.traceExpressions("m1", exprs);
  7497. if (options.allowScopeMigrate) // && !options.minimizeWorkunitTemporaries)
  7498. {
  7499. NewScopeMigrateTransformer transformer(wu, translator);
  7500. HqlExprArray results;
  7501. transformer.analyseArray(exprs, 0);
  7502. transformer.transformRoot(exprs, results);
  7503. replaceArray(exprs, results);
  7504. translator.checkNormalized(exprs);
  7505. }
  7506. translator.traceExpressions("m2", exprs);
  7507. }
  7508. void expandGlobalDatasets(WorkflowArray & array, IWorkUnit * wu, HqlCppTranslator & translator)
  7509. {
  7510. }
  7511. //---------------------------------------------------------------------------
  7512. bool AutoScopeMigrateInfo::addGraph(unsigned graph)
  7513. {
  7514. if (graph == lastGraph)
  7515. return false;
  7516. if (lastGraph)
  7517. manyGraphs = true;
  7518. lastGraph = graph;
  7519. return true;
  7520. }
  7521. bool AutoScopeMigrateInfo::doAutoHoist(IHqlExpression * transformed, bool minimizeWorkunitTemporaries)
  7522. {
  7523. if (useCount == 0)
  7524. return false;
  7525. node_operator op = original->getOperator();
  7526. switch (op)
  7527. {
  7528. case no_fail:
  7529. return false;
  7530. }
  7531. if (firstUseIsConditional && firstUseIsSequential)
  7532. return false;
  7533. if (firstUseIsSequential && !manyGraphs)
  7534. return false;
  7535. // The following *should* generate better code, but there are currently a couple of exceptions (cmaroney29, jholt20) which need investigation
  7536. // if (!manyGraphs)
  7537. // return false;
  7538. if (globalInsideChild && !minimizeWorkunitTemporaries)// && !transformed->isDataset() && !transformed->isDatarow())
  7539. return true;
  7540. if (!manyGraphs)
  7541. return false;
  7542. if (!original->isDataset())
  7543. {
  7544. switch (op)
  7545. {
  7546. case NO_AGGREGATE:
  7547. break;
  7548. default:
  7549. return false;
  7550. }
  7551. }
  7552. if (!isWorthHoisting(transformed, false))
  7553. return false;
  7554. if (isContextDependent(transformed))
  7555. return false;
  7556. return isIndependentOfScope(original);
  7557. }
  7558. static HqlTransformerInfo autoScopeMigrateTransformerInfo("AutoScopeMigrateTransformer");
  7559. AutoScopeMigrateTransformer::AutoScopeMigrateTransformer(IWorkUnit * _wu, HqlCppTranslator & _translator)
  7560. : NewHqlTransformer(autoScopeMigrateTransformerInfo), translator(_translator)
  7561. {
  7562. wu = _wu;
  7563. isRoxie = (translator.getTargetClusterType() == RoxieCluster);
  7564. isConditional = false;
  7565. isSequential = false;
  7566. hasCandidate = false;
  7567. activityDepth = 0;
  7568. curGraph = 1;
  7569. globalTarget = NULL;
  7570. }
  7571. void AutoScopeMigrateTransformer::analyseExpr(IHqlExpression * expr)
  7572. {
  7573. AutoScopeMigrateInfo * extra = queryBodyExtra(expr);
  7574. if (isConditional)
  7575. extra->condUseCount++;
  7576. else
  7577. extra->useCount++;
  7578. bool newGraph = extra->addGraph(curGraph);
  7579. if (!newGraph)
  7580. return;
  7581. if (extra->doAutoHoist(expr, translator.queryOptions().minimizeWorkunitTemporaries))
  7582. {
  7583. hasCandidate = true;
  7584. return;
  7585. }
  7586. unsigned savedDepth = activityDepth;
  7587. doAnalyseExpr(expr);
  7588. activityDepth = savedDepth;
  7589. }
  7590. void AutoScopeMigrateTransformer::doAnalyseConditionalExpr(IHqlExpression * expr, unsigned firstConditional)
  7591. {
  7592. bool wasConditional = isConditional;
  7593. ForEachChild(i, expr)
  7594. {
  7595. if (i == firstConditional)
  7596. isConditional = true;
  7597. analyseExpr(expr->queryChild(i));
  7598. }
  7599. isConditional = wasConditional;
  7600. }
  7601. void AutoScopeMigrateTransformer::doAnalyseExpr(IHqlExpression * expr)
  7602. {
  7603. AutoScopeMigrateInfo * extra = queryBodyExtra(expr);
  7604. if (activityDepth && expr->isDataset())
  7605. {
  7606. if (isWorthHoisting(expr, true) && isIndependentOfScope(expr) && !isContextDependent(expr))
  7607. {
  7608. #ifdef _DEBUG
  7609. isWorthHoisting(expr, true);
  7610. #endif
  7611. extra->globalInsideChild = true;
  7612. hasCandidate = true;
  7613. activityDepth = 0;
  7614. }
  7615. }
  7616. extra->firstUseIsConditional = isConditional;
  7617. extra->firstUseIsSequential = isSequential;
  7618. switch (expr->getOperator())
  7619. {
  7620. //Really the child nodes should be visited, and marked to ensure they are never hoisted, implemented
  7621. //as a prior pass. However long term this transformation should be removed, so not worth doing now.
  7622. case no_allnodes:
  7623. case no_keyedlimit:
  7624. case no_nothor:
  7625. case no_sizeof:
  7626. return;
  7627. case no_sequential:
  7628. return;
  7629. case no_if:
  7630. case no_choose:
  7631. case no_case:
  7632. {
  7633. if (expr->isAction() || (graphDepth == 0))
  7634. {
  7635. doAnalyseConditionalExpr(expr, 1);
  7636. return;
  7637. }
  7638. break;
  7639. }
  7640. case no_newtransform:
  7641. case no_transform:
  7642. if (curGraph)
  7643. {
  7644. activityDepth++;
  7645. NewHqlTransformer::analyseExpr(expr);
  7646. activityDepth--;
  7647. return;
  7648. }
  7649. break;
  7650. case no_map:
  7651. if (expr->isAction() || (graphDepth == 0))
  7652. {
  7653. doAnalyseConditionalExpr(expr, 0);
  7654. return;
  7655. }
  7656. break;
  7657. case no_thor:
  7658. //ignore thor attribute on a dataset..
  7659. if (expr->queryType())
  7660. {
  7661. curGraph++;
  7662. graphDepth++;
  7663. NewHqlTransformer::analyseExpr(expr);
  7664. curGraph++; // don't restore - new pseudo graph to aid cse between global branches separated by graphs
  7665. graphDepth--;
  7666. return;
  7667. }
  7668. break;
  7669. }
  7670. NewHqlTransformer::analyseExpr(expr);
  7671. }
  7672. IHqlExpression * AutoScopeMigrateTransformer::createTransformed(IHqlExpression * expr)
  7673. {
  7674. switch (expr->getOperator())
  7675. {
  7676. case no_allnodes:
  7677. case no_libraryscope:
  7678. case no_nothor:
  7679. case no_sequential:
  7680. return LINK(expr);
  7681. case no_thor:
  7682. {
  7683. IHqlExpression * actions = expr->queryChild(0);
  7684. if (actions)
  7685. {
  7686. //MORE: Simplify this???? or remove the special case all together?
  7687. //OwnedHqlExpr newActions = transform(actions);
  7688. HqlExprArray args;
  7689. unwindCommaCompound(args, actions);
  7690. ForEachItemIn(i, args)
  7691. graphActions.append(*transform(&args.item(i)));
  7692. OwnedHqlExpr newActions = createActionList(graphActions);
  7693. graphActions.kill();
  7694. if (actions == newActions)
  7695. return LINK(expr);
  7696. return createWrapper(no_thor, newActions.getClear());
  7697. }
  7698. break;
  7699. }
  7700. }
  7701. OwnedHqlExpr transformed = NewHqlTransformer::createTransformed(expr);
  7702. updateOrphanedSelectors(transformed, expr);
  7703. AutoScopeMigrateInfo * extra = queryBodyExtra(expr);
  7704. if (extra->doAutoHoist(transformed, translator.queryOptions().minimizeWorkunitTemporaries))
  7705. {
  7706. StringBuffer s;
  7707. s.appendf("AutoGlobal: Spotted %s ", getOpString(expr->getOperator()));
  7708. if (expr->queryName())
  7709. s.append("[").append(expr->queryName()).append("] ");
  7710. s.append("as an item to hoist");
  7711. DBGLOG("%s", s.str());
  7712. if (extra->globalInsideChild)
  7713. {
  7714. StringBuffer nameText;
  7715. if (expr->queryName())
  7716. nameText.append(expr->queryName());
  7717. else
  7718. getExprECL(expr, nameText);
  7719. translator.reportWarning(CategoryEfficiency, SeverityIgnore, queryActiveLocation(expr), HQLWRN_GlobalDatasetFromChildQuery, HQLWRN_GlobalDatasetFromChildQuery_Text, nameText.str());
  7720. }
  7721. GlobalAttributeInfo info("jobtemp::auto","auto", transformed);
  7722. info.extractGlobal(NULL, translator.getTargetClusterType());
  7723. if (translator.targetThor() && extra->globalInsideChild)
  7724. info.preventDiskSpill();
  7725. OwnedHqlExpr getResult, setResult;
  7726. info.checkFew(translator);
  7727. info.splitGlobalDefinition(transformed->queryType(), transformed, wu, setResult, &getResult, isRoxie);
  7728. //If the first use is conditional, then hoist the expression globally (it can't have any dependents)
  7729. //else hoist it within the current graph, otherwise it can get hoisted before globals on datasets that
  7730. //it is dependent on.
  7731. if (extra->firstUseIsConditional)
  7732. globalTarget->append(*createWrapper(no_thor, setResult.getClear()));
  7733. else
  7734. graphActions.append(*setResult.getClear());
  7735. transformed.setown(getResult.getClear());
  7736. }
  7737. return transformed.getClear();
  7738. }
  7739. void AutoScopeMigrateTransformer::transformRoot(const HqlExprArray & in, HqlExprArray & out)
  7740. {
  7741. globalTarget = &out;
  7742. NewHqlTransformer::transformRoot(in, out);
  7743. globalTarget = NULL;
  7744. }
  7745. //---------------------------------------------------------------------------
  7746. static HqlTransformerInfo trivialGraphRemoverInfo("TrivialGraphRemover");
  7747. TrivialGraphRemover::TrivialGraphRemover() : NewHqlTransformer(trivialGraphRemoverInfo)
  7748. {
  7749. hasCandidate = false;
  7750. }
  7751. void TrivialGraphRemover::analyseExpr(IHqlExpression * expr)
  7752. {
  7753. if (hasCandidate || alreadyVisited(expr))
  7754. return;
  7755. if (expr->getOperator() == no_thor)
  7756. {
  7757. if (isTrivialGraph(expr->queryChild(0)))
  7758. hasCandidate = true;
  7759. return;
  7760. }
  7761. NewHqlTransformer::analyseExpr(expr);
  7762. }
  7763. IHqlExpression * TrivialGraphRemover::createTransformed(IHqlExpression * expr)
  7764. {
  7765. switch (expr->getOperator())
  7766. {
  7767. case no_thor:
  7768. {
  7769. IHqlExpression * child = expr->queryChild(0);
  7770. if (child && isTrivialGraph(child))
  7771. return LINK(child);
  7772. return LINK(expr);
  7773. }
  7774. }
  7775. return NewHqlTransformer::createTransformed(expr);
  7776. }
  7777. bool TrivialGraphRemover::isTrivialGraph(IHqlExpression * expr)
  7778. {
  7779. if (!expr)
  7780. return false;
  7781. if (expr->getOperator() == no_setresult)
  7782. {
  7783. IHqlExpression * value = expr->queryChild(0);
  7784. if (value->getOperator() != no_getresult)
  7785. return false;
  7786. return true;
  7787. }
  7788. else if (expr->getOperator() == no_output)
  7789. return isTrivialInlineOutput(expr);
  7790. else
  7791. return false;
  7792. }
  7793. void removeTrivialGraphs(WorkflowItem & curWorkflow)
  7794. {
  7795. HqlExprArray & exprs = curWorkflow.queryExprs();
  7796. TrivialGraphRemover transformer;
  7797. transformer.analyseArray(exprs, 0);
  7798. if (transformer.worthTransforming())
  7799. {
  7800. HqlExprArray simplified;
  7801. transformer.transformRoot(exprs, simplified);
  7802. replaceArray(exprs, simplified);
  7803. }
  7804. }
  7805. //==============================================================================================================
  7806. class FilterCloner
  7807. {
  7808. public:
  7809. FilterCloner(IHqlExpression * _ds) { ds.set(_ds); matched = false; lockTransformMutex(); }
  7810. ~FilterCloner() { unlockTransformMutex(); }
  7811. void addMappings(IHqlExpression * expr, IHqlExpression * addDs);
  7812. IHqlExpression * inheritFilters(IHqlExpression * expr);
  7813. inline bool hasMappings() { return matched; }
  7814. protected:
  7815. void doAddMappings(IHqlExpression * expr);
  7816. bool isMatchingSelector(IHqlExpression * expr);
  7817. void setMapping(IHqlExpression * selector, IHqlExpression * value);
  7818. protected:
  7819. HqlExprAttr ds;
  7820. bool matched;
  7821. };
  7822. void FilterCloner::setMapping(IHqlExpression * selector, IHqlExpression * value)
  7823. {
  7824. selector->setTransformExtra(value);
  7825. matched = true;
  7826. }
  7827. void FilterCloner::doAddMappings(IHqlExpression * expr)
  7828. {
  7829. for (;;)
  7830. {
  7831. switch (expr->getOperator())
  7832. {
  7833. case no_and:
  7834. doAddMappings(expr->queryChild(0));
  7835. expr = expr->queryChild(1);
  7836. continue;
  7837. case no_in:
  7838. case no_notin:
  7839. {
  7840. IHqlExpression * lhs = expr->queryChild(0);
  7841. IHqlExpression * rhs = expr->queryChild(1);
  7842. if (isMatchingSelector(lhs) && !containsActiveDataset(rhs))
  7843. setMapping(lhs, expr);
  7844. break;
  7845. }
  7846. case no_between:
  7847. case no_notbetween:
  7848. {
  7849. IHqlExpression * lhs = expr->queryChild(0);
  7850. if (isMatchingSelector(lhs) && !containsActiveDataset(expr->queryChild(1)) && !containsActiveDataset(expr->queryChild(2)))
  7851. setMapping(lhs, expr);
  7852. break;
  7853. }
  7854. case no_eq:
  7855. case no_ne:
  7856. case no_lt:
  7857. case no_gt:
  7858. case no_ge:
  7859. case no_le:
  7860. {
  7861. IHqlExpression * lhs = expr->queryChild(0);
  7862. IHqlExpression * rhs = expr->queryChild(1);
  7863. if (isMatchingSelector(lhs) && !containsActiveDataset(rhs))
  7864. setMapping(lhs, expr);
  7865. else if (isMatchingSelector(rhs) && !containsActiveDataset(lhs))
  7866. setMapping(rhs, expr);
  7867. break;
  7868. }
  7869. }
  7870. return;
  7871. }
  7872. }
  7873. void FilterCloner::addMappings(IHqlExpression * expr, IHqlExpression * addDs)
  7874. {
  7875. if (!expr) return;
  7876. OwnedHqlExpr replaced = replaceSelector(expr, addDs, ds);
  7877. doAddMappings(replaced);
  7878. }
  7879. bool FilterCloner::isMatchingSelector(IHqlExpression * expr)
  7880. {
  7881. return queryDatasetCursor(expr) == ds;
  7882. }
  7883. IHqlExpression * FilterCloner::inheritFilters(IHqlExpression * expr)
  7884. {
  7885. switch (expr->getOperator())
  7886. {
  7887. case no_and:
  7888. case no_assertkeyed:
  7889. case no_assertstepped:
  7890. {
  7891. HqlExprArray args;
  7892. ForEachChild(i, expr)
  7893. args.append(*inheritFilters(expr->queryChild(i)));
  7894. return cloneOrLink(expr, args);
  7895. }
  7896. case no_eq:
  7897. {
  7898. IHqlExpression * lhs = expr->queryChild(0);
  7899. IHqlExpression * rhs = expr->queryChild(1);
  7900. IHqlExpression * lhsExtra = (IHqlExpression *)lhs->queryTransformExtra();
  7901. if (lhsExtra)
  7902. {
  7903. DBGLOG("Inheriting filter condition");
  7904. IHqlExpression * cond = replaceExpression(lhsExtra, lhs, rhs);
  7905. return createValue(no_and, LINK(expr), cond);
  7906. }
  7907. IHqlExpression * rhsExtra = (IHqlExpression *)rhs->queryTransformExtra();
  7908. if (rhsExtra)
  7909. {
  7910. DBGLOG("Inheriting filter condition");
  7911. IHqlExpression * cond = replaceExpression(rhsExtra, rhs, lhs);
  7912. return createValue(no_and, LINK(expr), cond);
  7913. }
  7914. break;
  7915. }
  7916. case no_in:
  7917. case no_notin:
  7918. case no_between:
  7919. case no_notbetween:
  7920. {
  7921. IHqlExpression * lhs = expr->queryChild(0);
  7922. IHqlExpression * lhsExtra = (IHqlExpression *)lhs->queryTransformExtra();
  7923. if (lhsExtra)
  7924. {
  7925. DBGLOG("Inheriting filter condition");
  7926. return createValue(no_and, LINK(expr), LINK(lhsExtra));
  7927. }
  7928. break;
  7929. }
  7930. }
  7931. return LINK(expr);
  7932. }
  7933. static IHqlExpression * optimizeJoinFilter(IHqlExpression * expr)
  7934. {
  7935. //NB: Not a member function because we use a different transform mutex, and don't want to accidentally interfere with caller's use
  7936. IHqlExpression * index = expr->queryChild(1);
  7937. if (!index->hasAttribute(filteredAtom) && !index->hasAttribute(_filtered_Atom) && (index->getOperator() == no_newkeyindex))
  7938. return LINK(expr);
  7939. if (expr->hasAttribute(keyedAtom))
  7940. return LINK(expr); //MORE!
  7941. OwnedHqlExpr rhs = createSelector(no_right, index, querySelSeq(expr));
  7942. FilterCloner processor(rhs);
  7943. while (index->getOperator() != no_newkeyindex)
  7944. {
  7945. switch (index->getOperator())
  7946. {
  7947. case no_filter:
  7948. //case no_filtered:
  7949. //MORE: This might be useful in the future
  7950. UNIMPLEMENTED;
  7951. }
  7952. index = index->queryChild(0);
  7953. }
  7954. processor.addMappings(queryAttributeChild(index, filteredAtom, 0), queryActiveTableSelector());
  7955. processor.addMappings(queryAttributeChild(index, _filtered_Atom, 0), queryActiveTableSelector());
  7956. if (!processor.hasMappings())
  7957. return LINK(expr);
  7958. HqlExprArray exprs;
  7959. expr->queryChild(2)->unwindList(exprs, no_and);
  7960. bool keyedExplicitly = false;
  7961. ForEachItemIn(i1, exprs)
  7962. {
  7963. IHqlExpression & cur = exprs.item(i1);
  7964. switch (cur.getOperator())
  7965. {
  7966. case no_assertkeyed:
  7967. case no_assertwild:
  7968. keyedExplicitly = true;
  7969. exprs.replace(*processor.inheritFilters(&cur), i1);
  7970. break;
  7971. }
  7972. }
  7973. if (!keyedExplicitly)
  7974. {
  7975. ForEachItemIn(i2, exprs)
  7976. {
  7977. IHqlExpression & cur = exprs.item(i2);
  7978. switch (cur.getOperator())
  7979. {
  7980. case no_assertkeyed:
  7981. case no_assertwild:
  7982. case no_attr:
  7983. case no_attr_link:
  7984. case no_attr_expr:
  7985. break;
  7986. default:
  7987. exprs.replace(*processor.inheritFilters(&cur), i2);
  7988. break;
  7989. }
  7990. }
  7991. }
  7992. HqlExprArray args;
  7993. unwindChildren(args, expr);
  7994. args.replace(*createBalanced(no_and, queryBoolType(), exprs), 2);
  7995. return expr->clone(args);
  7996. }
  7997. static HqlTransformerInfo filteredIndexOptimizerInfo("FilteredIndexOptimizer");
  7998. FilteredIndexOptimizer::FilteredIndexOptimizer(bool _processJoins, bool _processReads)
  7999. : NewHqlTransformer(filteredIndexOptimizerInfo)
  8000. {
  8001. processJoins = _processJoins;
  8002. processReads = _processReads;
  8003. }
  8004. IHqlExpression * FilteredIndexOptimizer::createTransformed(IHqlExpression * expr)
  8005. {
  8006. OwnedHqlExpr transformed = NewHqlTransformer::createTransformed(expr);
  8007. if (processJoins && isKeyedJoin(transformed))
  8008. transformed.setown(optimizeJoinFilter(transformed));
  8009. if (processReads)
  8010. {
  8011. switch (transformed->getOperator())
  8012. {
  8013. case no_compound_indexread:
  8014. case no_compound_indexnormalize:
  8015. case no_compound_indexaggregate:
  8016. case no_compound_indexcount:
  8017. case no_compound_indexgroupaggregate:
  8018. //MORE:
  8019. break;
  8020. }
  8021. }
  8022. return transformed.getClear();
  8023. }
  8024. //==============================================================================================================
  8025. static HqlTransformerInfo DFSLayoutTransformerInfo("DFSLayoutTransformer");
  8026. DFSLayoutTransformer::DFSLayoutTransformer(IErrorReceiver &_errs, ICodegenContextCallback * _ctxCallback, HqlCppOptions const &_options)
  8027. : NewHqlTransformer(DFSLayoutTransformerInfo), errs(_errs), options(_options)
  8028. {
  8029. ctxCallback = _ctxCallback;
  8030. }
  8031. IHqlExpression * DFSLayoutTransformer::createTransformed(IHqlExpression * expr)
  8032. {
  8033. OwnedHqlExpr transformed = NewHqlTransformer::createTransformed(expr);
  8034. int filenameIdx = 0;
  8035. int recordIdx = 0;
  8036. switch (transformed->getOperator())
  8037. {
  8038. case no_table:
  8039. filenameIdx = 0;
  8040. recordIdx = 1;
  8041. break;
  8042. case no_keyindex:
  8043. throwUnexpected(); // They have already been translated into no_newkeyindex by HqlTreeNormalizer::transformKeyIndex
  8044. break;
  8045. case no_newkeyindex:
  8046. // Args are flatfile, record, transform, name. Offset of first payload field indicated by value of __payload_Atom attribute
  8047. filenameIdx = 3;
  8048. recordIdx = 1;
  8049. break;
  8050. default:
  8051. return transformed.getClear();
  8052. }
  8053. LinkedHqlExpr ds = transformed;
  8054. IHqlExpression * filename = ds->queryChild(filenameIdx);
  8055. bool translate = getBoolAttribute(ds, lookupAtom, options.translateDFSlayouts && filename->isConstant());
  8056. if (translate)
  8057. {
  8058. ECLlocation where(ds);
  8059. OwnedHqlExpr folded = foldHqlExpression(errs, filename, nullptr, HFOforcefold);
  8060. StringBuffer fileNameText;
  8061. getStringValue(fileNameText, folded);
  8062. if (fileNameText.length()) // PIPE creates a no_table with a blank filename, it seems
  8063. {
  8064. // Need to work out what to do about numKeyedFields and what happens if it changes
  8065. // If payload fields are added/removed, no problem, just transform as with datasets
  8066. // If keyed fields are added/removed, may be trickier
  8067. OwnedHqlExpr dfsLayout = ctxCallback->lookupDFSlayout(fileNameText, errs, where, ds->hasAttribute(optAtom));
  8068. if (dfsLayout)
  8069. {
  8070. IHqlExpression *wanted = ds->queryRecord();
  8071. // If wanted contains any virtual fields, clone them onto dfsLayout
  8072. if (containsVirtualFields(wanted))
  8073. {
  8074. HqlExprArray newFields;
  8075. unwindChildren(newFields, dfsLayout);
  8076. getVirtualFields(newFields, wanted);
  8077. dfsLayout.setown(dfsLayout->clone(newFields));
  8078. }
  8079. IHqlExpression *payload = dfsLayout->queryAttribute(_payload_Atom);
  8080. unsigned dfsPayload = 0;
  8081. if (payload)
  8082. {
  8083. dfsPayload = (unsigned)getIntValue(payload->queryChild(0));
  8084. dfsLayout.setown(removeAttribute(dfsLayout, _payload_Atom));
  8085. }
  8086. // Check if the layout fetched from DFS matches the ECL layout
  8087. if (recordTypesMatch(dfsLayout, wanted))
  8088. return transformed.getClear(); // No substitution needed
  8089. if (options.reportDFSinfo >= 2)
  8090. {
  8091. VStringBuffer msg( "LOOKUP %s found modified record structure", fileNameText.str());
  8092. errs.reportWarning(CategoryInformation, HQLINFO_DFSlookupSubstituted, msg.str(), str(where.sourcePath), where.lineno, where.column, where.position);
  8093. reportDroppedFields(wanted, dfsLayout, errs, where);
  8094. }
  8095. if (transformed->getOperator() == no_newkeyindex)
  8096. {
  8097. // update the transform used by full keyed join to map fields
  8098. // note it gets updated again if the rhs of the keyed join has translation applied
  8099. IHqlExpression *oldTransform = ds->queryChild(2);
  8100. assertex(oldTransform->getOperator()==no_newtransform);
  8101. OwnedHqlExpr seq = createSelectorSequence();
  8102. OwnedHqlExpr left = createSelector(no_left, oldTransform, seq);
  8103. OwnedHqlExpr self = getSelf(dfsLayout);
  8104. MultiErrorReceiver dummy;
  8105. OwnedHqlExpr transformECLtoDFS = createMappingTransform(self, left, true, dummy, where); // transforms from ECL layout to DFS layout. Don't report missing here!
  8106. NewProjectMapper2 mapper;
  8107. mapper.setMapping(oldTransform);
  8108. OwnedHqlExpr newTransform = mapper.expandFields(transformECLtoDFS, left, nullptr, nullptr);
  8109. ds.setown(replaceChild(ds, 2, newTransform));
  8110. // The payload attribute specifies how many UNKEYED fields there are - so if we added or removed any fields we may need to change it
  8111. if (dfsPayload != numPayloadFields(ds))
  8112. {
  8113. // Note - this doesn't merit a warning - but if we changed the number of keyed it might?
  8114. ds.setown(replaceOwnedAttribute(ds, createExprAttribute(_payload_Atom, createConstant((int) dfsPayload))));
  8115. }
  8116. // NOTE - we assume that the dfs info will always include a fileposition field, even if the ECL definition did not.
  8117. // Hence we always need to set the filepositionAtom indicator to true
  8118. ds.setown(replaceOwnedAttribute(ds, createExprAttribute(filepositionAtom, createConstant(true))));
  8119. }
  8120. OwnedHqlExpr diskread = replaceChild(ds, recordIdx, dfsLayout);
  8121. OwnedHqlExpr seq = createSelectorSequence();
  8122. OwnedHqlExpr left = createSelector(no_left, diskread, seq);
  8123. OwnedHqlExpr self = getSelf(wanted);
  8124. IHqlExpression * transform = createMappingTransform(self, left, true, errs, where); // transforms from dfsLayout to wanted
  8125. HqlExprArray args;
  8126. args.append(*diskread.getClear());
  8127. args.append(*transform);
  8128. args.append(*seq.getClear());
  8129. args.append(*createAttribute(keyedAtom));
  8130. return createDataset(no_hqlproject, args);
  8131. }
  8132. }
  8133. }
  8134. #ifdef TEST_INDEX_PROJECT
  8135. else if (transformed->getOperator()==no_newkeyindex)
  8136. {
  8137. OwnedHqlExpr dfsLayout = LINK(ds->queryRecord());
  8138. OwnedHqlExpr diskread = replaceChild(ds, recordIdx, dfsLayout);
  8139. OwnedHqlExpr seq = createSelectorSequence();
  8140. OwnedHqlExpr left = createSelector(no_left, diskread, seq);
  8141. OwnedHqlExpr self = getSelf(dfsLayout);
  8142. ECLlocation where(ds);
  8143. IHqlExpression * transform = createMappingTransform(self, left, true, errs, where);
  8144. OwnedHqlExpr ret = createDataset(no_hqlproject, diskread.getClear(), createComma(transform, LINK(seq)));
  8145. ret.setown(appendAttribute(ret, keyedAtom));
  8146. DBGLOG("Adding implicit project %p for testing", ret.get());
  8147. return ret.getClear();
  8148. }
  8149. #endif
  8150. return transformed.getClear();
  8151. }
  8152. //==============================================================================================================
  8153. static HqlTransformerInfo KeyedProjectTransformerInfo("KeyedProjectTransformer");
  8154. KeyedProjectTransformer::KeyedProjectTransformer()
  8155. : NewHqlTransformer(KeyedProjectTransformerInfo)
  8156. {
  8157. }
  8158. IHqlExpression * KeyedProjectTransformer::createTransformed(IHqlExpression * expr)
  8159. {
  8160. OwnedHqlExpr transformed = NewHqlTransformer::createTransformed(expr);
  8161. auto op = transformed->getOperator();
  8162. if (op == no_join || op == no_denormalize || op == no_denormalizegroup || op == no_keyeddistribute)
  8163. {
  8164. // Merge a KEYED PROJECT into a join, so that it can become a KEYED JOIN
  8165. // These may have come about through DFS lookup translation, but (at least some of) this transformation is also applicable if user put them in
  8166. OwnedHqlExpr rhs = LINK(transformed->queryChild(1));
  8167. node_operator rhsOp = rhs->getOperator();
  8168. #if 0
  8169. // This code allows example hfrs4 to compile with TEST_INDEX_PROJECT set, but it's not technically correct
  8170. // We should be patching up the child rather than just ignoring the no_compund node.
  8171. if (rhsOp == no_compound)
  8172. {
  8173. rhs.set(LINK(rhs->queryChild(1)));
  8174. rhsOp = rhs->getOperator();
  8175. }
  8176. #endif
  8177. if (rhsOp == no_compound_indexread) // This may not be needed now that the code is moved earlier?
  8178. {
  8179. rhs.set(LINK(rhs->queryChild(0)));
  8180. rhsOp = rhs->getOperator();
  8181. }
  8182. if (op != no_keyeddistribute && queryAttributeChild(transformed, keyedAtom, 0))
  8183. {
  8184. // Full keyed joins need to skip any keyed project that may have been inserted on the KEYED attr
  8185. IHqlExpression *index = queryAttributeChild(transformed, keyedAtom, 0);
  8186. if (index->getOperator()==no_hqlproject && index->hasAttribute(keyedAtom) && !hasUnknownTransform(index))
  8187. transformed.setown(replaceOwnedAttribute(transformed, createExprAttribute(keyedAtom, LINK(index->queryChild(0)))));
  8188. }
  8189. if (rhsOp == no_hqlproject && rhs->hasAttribute(keyedAtom) && !hasUnknownTransform(rhs))
  8190. {
  8191. TableProjectMapper mapper(rhs);
  8192. IHqlExpression * selSeq = querySelSeq(transformed);
  8193. OwnedHqlExpr oldRight = createSelector(no_right, rhs, selSeq);
  8194. OwnedHqlExpr newRight = createSelector(no_right, rhs->queryChild(0), selSeq);
  8195. OwnedHqlExpr translatedFilter = mapper.expandFields(transformed->queryChild(2), oldRight, newRight);
  8196. OwnedHqlExpr expandedTransform;
  8197. if (op != no_keyeddistribute)
  8198. expandedTransform.setown(mapper.expandFields(transformed->queryChild(3), oldRight, newRight));
  8199. if (translatedFilter && (expandedTransform || op == no_keyeddistribute))
  8200. {
  8201. DBGLOG("KeyedProjectTransformer: Merge KEYED PROJECT into JOIN");
  8202. HqlExprArray args;
  8203. args.append(*LINK(transformed->queryChild(0)));
  8204. args.append(*LINK(rhs->queryChild(0)));
  8205. args.append(*translatedFilter.getClear());
  8206. if (op == no_keyeddistribute)
  8207. unwindChildren(args, transformed, 3);
  8208. else
  8209. {
  8210. args.append(*expandedTransform.getClear());
  8211. unwindChildren(args, transformed, 4);
  8212. }
  8213. transformed.setown(transformed->clone(args));
  8214. }
  8215. if (op != no_keyeddistribute && transformed->hasAttribute(keyedAtom) && queryAttributeChild(transformed, keyedAtom, 0))
  8216. {
  8217. // Full keyed joins also need to update the transformation from RHS disk file to INDEX record, in the light of any changes
  8218. // to the RHS since declared. It's not clear that this should be done to user-specified KEYED PROJECTs though...
  8219. IHqlExpression *index = queryAttributeChild(transformed, keyedAtom, 0);
  8220. assertex (index->getOperator() == no_newkeyindex);
  8221. IHqlExpression *oldFlatFile = index->queryChild(0);
  8222. IHqlExpression *newFlatFile = rhs->queryChild(0);
  8223. IHqlExpression *oldTransform = index->queryChild(2);
  8224. assertex(oldTransform->isTransform());
  8225. HqlExprArray args;
  8226. args.append(*LINK(newFlatFile));
  8227. args.append(*LINK(index->queryChild(1)));
  8228. args.append(*mapper.expandFields(oldTransform, oldFlatFile, newFlatFile));
  8229. unwindChildren(args, index, 3);
  8230. transformed.setown(replaceOwnedAttribute(transformed, createExprAttribute(keyedAtom, index->clone(args))));
  8231. }
  8232. }
  8233. }
  8234. return transformed.getClear();
  8235. }
  8236. //==============================================================================================================
  8237. static HqlTransformerInfo localUploadTransformerInfo("LocalUploadTransformer");
  8238. LocalUploadTransformer::LocalUploadTransformer(IWorkUnit * _wu) : NewHqlTransformer(localUploadTransformerInfo)
  8239. {
  8240. wu = _wu;
  8241. }
  8242. IHqlExpression * LocalUploadTransformer::createTransformed(IHqlExpression * expr)
  8243. {
  8244. OwnedHqlExpr transformed = NewHqlTransformer::createTransformed(expr);
  8245. if (transformed->getOperator() == no_table && transformed->hasAttribute(localUploadAtom))
  8246. {
  8247. IHqlExpression * filename = transformed->queryChild(0);
  8248. IHqlExpression * mode = transformed->queryChild(2);
  8249. assertex(filename->getOperator() == no_constant);
  8250. StringBuffer sourceName,localName;
  8251. filename->queryValue()->getStringValue(sourceName);
  8252. getUniqueId(localName.append("local"));
  8253. LocalFileUploadType uploadType = UploadTypeWUResult;
  8254. switch (mode->getOperator())
  8255. {
  8256. case no_csv:
  8257. uploadType = UploadTypeWUResultCsv;
  8258. break;
  8259. case no_xml:
  8260. uploadType = UploadTypeWUResultXml;
  8261. break;
  8262. }
  8263. wu->addLocalFileUpload(uploadType, sourceName, localName, NULL);
  8264. HqlExprArray args;
  8265. args.append(*LINK(expr->queryRecord()));
  8266. args.append(*createAttribute(nameAtom, createConstant(localName.str())));
  8267. args.append(*createAttribute(sequenceAtom, getStoredSequenceNumber()));
  8268. return createDataset(no_workunit_dataset, args);
  8269. }
  8270. return transformed.getClear();
  8271. }
  8272. //==============================================================================================================
  8273. /*
  8274. The following code converts expressions of the form a.b where a and b are datasets into a normalized form including
  8275. an explicit normalize activity. It follows the following rules:
  8276. 1) An filter/project/table on a.b logically has access to fields in a and a.b.
  8277. 2) An operation with a transform (or record defining the output) other than a PROJECT/TABLE, implicitly loses access to fields in a for all
  8278. subsequent operations.
  8279. 3) When a dataset is used in an output or outer outer context (e.g., call parameter) then only the fields in the child dataset are passed.
  8280. This is done by converting f(a.b) to normalize(a, f(LEFT.b)) at the appropriate place.
  8281. These rules mean we maintain the HOLe semantics which allow computed fields to be implemented by projecting fields from the parent dataset,
  8282. but also mean that we avoid problems with parent datasets e.g., join(a.b, c.d)
  8283. To make this efficient we also need to implement the following in the code generator:
  8284. 1) aggregate-normalize(x).
  8285. 2) normalize-source.
  8286. 3) aggregate-normalize-source
  8287. 4) inline processing of normalize.
  8288. After this transform there should be no existing datasets of the form <a.b>[new]. There is a new function to ensure this is correct.
  8289. For the first version, it only generates normalizes around datasets - and assumes that parent fields don't need to be mapped into the denormalize.
  8290. This makes it simpler, but means that parse statements that take a record which could theoretically access parent dataset
  8291. fields won't work. The fix would require a new kind of child normalize that took a no_newtransform, and would require
  8292. analysis of which parent fields are used in the child's no_newtransform.
  8293. */
  8294. inline void getDatasetRange(IHqlExpression * expr, unsigned & first, unsigned & max)
  8295. {
  8296. first = 0;
  8297. switch (getChildDatasetType(expr))
  8298. {
  8299. case childdataset_many_noscope:
  8300. case childdataset_many:
  8301. max = expr->numChildren();
  8302. break;
  8303. case childdataset_if:
  8304. first = 1;
  8305. max = expr->numChildren();
  8306. break;
  8307. default:
  8308. max = getNumChildTables(expr);
  8309. break;
  8310. }
  8311. }
  8312. static HqlTransformerInfo nestedSelectorNormalizerInfo("NestedSelectorNormalizer");
  8313. NestedSelectorNormalizer::NestedSelectorNormalizer() : NewHqlTransformer(nestedSelectorNormalizerInfo)
  8314. {
  8315. spottedCandidate = false;
  8316. }
  8317. void NestedSelectorNormalizer::analyseExpr(IHqlExpression * expr)
  8318. {
  8319. if (alreadyVisited(expr))
  8320. return;
  8321. NewHqlTransformer::analyseExpr(expr);
  8322. if (expr->isDataset())
  8323. {
  8324. bool childrenAreDenormalized = false;
  8325. unsigned first, max;
  8326. getDatasetRange(expr, first, max);
  8327. for (unsigned i=0; i < max; i++)
  8328. {
  8329. if (queryBodyExtra(expr->queryChild(i))->isDenormalized)
  8330. childrenAreDenormalized = true;
  8331. }
  8332. NestedSelectorInfo * extra = queryBodyExtra(expr);
  8333. switch (expr->getOperator())
  8334. {
  8335. case no_select:
  8336. if (isNewSelector(expr))
  8337. {
  8338. childrenAreDenormalized = true;
  8339. spottedCandidate = true;
  8340. }
  8341. break;
  8342. case no_hqlproject:
  8343. case no_usertable:
  8344. break;
  8345. default:
  8346. //Follow test effectively checks whether parent dataset is active beyond this point
  8347. if (expr->queryBody() == expr->queryNormalizedSelector())
  8348. {
  8349. if (childrenAreDenormalized)
  8350. {
  8351. extra->insertDenormalize = true;
  8352. childrenAreDenormalized = false;
  8353. }
  8354. }
  8355. break;
  8356. }
  8357. extra->isDenormalized = childrenAreDenormalized;
  8358. }
  8359. }
  8360. static IHqlExpression * splitSelector(IHqlExpression * expr, SharedHqlExpr & oldDataset)
  8361. {
  8362. assertex(expr->getOperator() == no_select);
  8363. IHqlExpression * ds = expr->queryChild(0);
  8364. if (expr->hasAttribute(newAtom))
  8365. {
  8366. oldDataset.set(ds);
  8367. OwnedHqlExpr left = createSelector(no_left, ds, querySelSeq(expr));
  8368. return createSelectExpr(left.getClear(), LINK(expr->queryChild(1)));
  8369. }
  8370. HqlExprArray args;
  8371. args.append(*splitSelector(ds, oldDataset));
  8372. unwindChildren(args, expr, 1);
  8373. return expr->clone(args);
  8374. }
  8375. IHqlExpression * NestedSelectorNormalizer::createNormalized(IHqlExpression * expr)
  8376. {
  8377. IHqlExpression * root = queryRoot(expr);
  8378. assertex(root && root->getOperator() == no_select && isNewSelector(root));
  8379. OwnedHqlExpr selSeq = createSelectorSequence();
  8380. OwnedHqlExpr oldDataset;
  8381. OwnedHqlExpr newSelector = splitSelector(root, oldDataset);
  8382. OwnedHqlExpr right = createSelector(no_right, expr, selSeq);
  8383. HqlExprArray args;
  8384. args.append(*LINK(oldDataset));
  8385. args.append(*replaceExpression(expr, root, newSelector));
  8386. args.append(*createTransformFromRow(right));
  8387. args.append(*LINK(selSeq));
  8388. OwnedHqlExpr ret = createDataset(no_normalize, args);
  8389. return expr->cloneAllAnnotations(ret);
  8390. }
  8391. IHqlExpression * NestedSelectorNormalizer::createTransformed(IHqlExpression * expr)
  8392. {
  8393. OwnedHqlExpr transformed = NewHqlTransformer::createTransformed(expr);
  8394. NestedSelectorInfo * extra = queryBodyExtra(expr);
  8395. bool denormalizeInputs = false;
  8396. if (extra->insertDenormalize)
  8397. {
  8398. denormalizeInputs = true;
  8399. }
  8400. else
  8401. {
  8402. switch (expr->getOperator())
  8403. {
  8404. case NO_AGGREGATE:
  8405. case no_joined:
  8406. case no_buildindex:
  8407. case no_apply:
  8408. case no_distribution:
  8409. case no_distributer:
  8410. case no_within:
  8411. case no_notwithin:
  8412. case no_output:
  8413. case no_createset:
  8414. case no_soapaction_ds:
  8415. case no_newsoapaction_ds:
  8416. case no_returnresult:
  8417. case no_setgraphresult:
  8418. case no_setgraphloopresult:
  8419. case no_keydiff:
  8420. case no_rowdiff:
  8421. case no_extractresult:
  8422. // case no_setresult:
  8423. case no_blob2id:
  8424. case no_selectnth:
  8425. case no_keypatch:
  8426. case no_assign:
  8427. case no_lt:
  8428. case no_le:
  8429. case no_gt:
  8430. case no_ge:
  8431. case no_ne:
  8432. case no_eq:
  8433. case no_order:
  8434. case no_keyed:
  8435. case no_loopbody:
  8436. case no_rowvalue:
  8437. case no_setmeta:
  8438. case no_typetransfer:
  8439. case no_subgraph:
  8440. denormalizeInputs = true;
  8441. break;
  8442. }
  8443. }
  8444. if (denormalizeInputs)
  8445. {
  8446. bool same = true;
  8447. HqlExprArray args;
  8448. unwindChildren(args, transformed);
  8449. unsigned first, max;
  8450. getDatasetRange(expr, first, max);
  8451. for (unsigned i = first; i < max; i++)
  8452. {
  8453. if (queryBodyExtra(expr->queryChild(i))->isDenormalized)
  8454. {
  8455. args.replace(*createNormalized(&args.item(i)), i);
  8456. same = false;
  8457. }
  8458. }
  8459. if (!same)
  8460. return transformed->clone(args);
  8461. }
  8462. return transformed.getClear();
  8463. }
  8464. //==============================================================================================================
  8465. /*
  8466. Code to spot ambiguous LEFT dataset references....
  8467. */
  8468. static HqlTransformerInfo leftRightSelectorNormalizerInfo("LeftRightSelectorNormalizer");
  8469. LeftRightSelectorNormalizer::LeftRightSelectorNormalizer(bool _allowAmbiguity) : NewHqlTransformer(leftRightSelectorNormalizerInfo)
  8470. {
  8471. allowAmbiguity = _allowAmbiguity;
  8472. isAmbiguous = false;
  8473. }
  8474. void LeftRightSelectorNormalizer::checkAmbiguity(const HqlExprCopyArray & inScope, IHqlExpression * selector)
  8475. {
  8476. node_operator selectOp = selector->getOperator();
  8477. ForEachItemIn(i, inScope)
  8478. {
  8479. IHqlExpression & cur = inScope.item(i);
  8480. if ((&cur != selector) && (cur.getOperator() == selectOp) && (cur.queryRecord() == selector->queryRecord()))
  8481. {
  8482. isAmbiguous = true;
  8483. if (!allowAmbiguity)
  8484. {
  8485. StringBuffer ecl;
  8486. getExprECL(selector, ecl);
  8487. throwError1(HQLERR_AmbiguousLeftRight, ecl.str());
  8488. }
  8489. }
  8490. }
  8491. }
  8492. void LeftRightSelectorNormalizer::analyseExpr(IHqlExpression * expr)
  8493. {
  8494. if (alreadyVisited(expr))
  8495. return;
  8496. IHqlExpression * selSeq = querySelSeq(expr);
  8497. if (selSeq)
  8498. {
  8499. HqlExprCopyArray inScope;
  8500. switch (getChildDatasetType(expr))
  8501. {
  8502. case childdataset_none:
  8503. case childdataset_many_noscope:
  8504. case childdataset_many:
  8505. case childdataset_map:
  8506. case childdataset_dataset_noscope:
  8507. case childdataset_if:
  8508. case childdataset_case:
  8509. case childdataset_dataset:
  8510. case childdataset_evaluate:
  8511. break;
  8512. case childdataset_datasetleft:
  8513. case childdataset_left:
  8514. {
  8515. IHqlExpression * dataset = expr->queryChild(0);
  8516. OwnedHqlExpr left = createSelector(no_left, dataset, selSeq);
  8517. gatherChildTablesUsed(NULL, &inScope, expr, 1);
  8518. checkAmbiguity(inScope, left);
  8519. break;
  8520. }
  8521. case childdataset_leftright:
  8522. {
  8523. OwnedHqlExpr left = createSelector(no_left, expr->queryChild(0), selSeq);
  8524. OwnedHqlExpr right = createSelector(no_right, expr->queryChild(1), selSeq);
  8525. gatherChildTablesUsed(NULL, &inScope, expr, 2);
  8526. checkAmbiguity(inScope, left);
  8527. checkAmbiguity(inScope, right);
  8528. break;
  8529. }
  8530. case childdataset_same_left_right:
  8531. case childdataset_top_left_right:
  8532. case childdataset_nway_left_right:
  8533. {
  8534. IHqlExpression * dataset = expr->queryChild(0);
  8535. OwnedHqlExpr left = createSelector(no_left, dataset, selSeq);
  8536. OwnedHqlExpr right = createSelector(no_right, dataset, selSeq);
  8537. gatherChildTablesUsed(NULL, &inScope, expr, 1);
  8538. checkAmbiguity(inScope, left);
  8539. checkAmbiguity(inScope, right);
  8540. break;
  8541. }
  8542. default:
  8543. UNIMPLEMENTED;
  8544. }
  8545. }
  8546. NewHqlTransformer::analyseExpr(expr);
  8547. }
  8548. IHqlExpression * LeftRightSelectorNormalizer::createTransformed(IHqlExpression * expr)
  8549. {
  8550. if (expr->isAttribute() && expr->queryName() == _selectorSequence_Atom)
  8551. return createDummySelectorSequence();
  8552. return NewHqlTransformer::createTransformed(expr);
  8553. }
  8554. IHqlExpression * LeftRightSelectorNormalizer::createTransformedSelector(IHqlExpression * expr)
  8555. {
  8556. node_operator op = expr->getOperator();
  8557. switch (op)
  8558. {
  8559. case no_left:
  8560. case no_right:
  8561. return transform(expr);
  8562. }
  8563. return NewHqlTransformer::createTransformedSelector(expr);
  8564. }
  8565. //==============================================================================================================
  8566. static HqlTransformerInfo forceLocalTransformerInfo("ForceLocalTransformer");
  8567. ForceLocalTransformer::ForceLocalTransformer(ClusterType _targetClusterType) : NewHqlTransformer(forceLocalTransformerInfo)
  8568. {
  8569. targetClusterType = _targetClusterType;
  8570. insideForceLocal = false;
  8571. allNodesDepth = 0;
  8572. }
  8573. IHqlExpression * ForceLocalTransformer::createTransformed(IHqlExpression * expr)
  8574. {
  8575. node_operator op = expr->getOperator();
  8576. switch (expr->getOperator())
  8577. {
  8578. case no_forcelocal:
  8579. case no_forcenolocal:
  8580. {
  8581. bool wasLocal = insideForceLocal;
  8582. insideForceLocal = (op == no_forcelocal) && (targetClusterType != HThorCluster);
  8583. IHqlExpression * ret = transform(expr->queryChild(0));
  8584. insideForceLocal = wasLocal;
  8585. return ret;
  8586. }
  8587. case no_thisnode:
  8588. if ((targetClusterType != HThorCluster) && (allNodesDepth == 0))
  8589. throwError(HQLERR_ThisNodeNotInsideAllNodes);
  8590. //fall through
  8591. case no_allnodes:
  8592. {
  8593. if (targetClusterType != HThorCluster)
  8594. {
  8595. unsigned oldDepth = allNodesDepth;
  8596. if (op == no_allnodes)
  8597. allNodesDepth++;
  8598. else
  8599. allNodesDepth--;
  8600. IHqlExpression * ret = NewHqlTransformer::createTransformed(expr);
  8601. allNodesDepth = oldDepth;
  8602. return ret;
  8603. }
  8604. else
  8605. return transform(expr->queryChild(0));
  8606. }
  8607. case no_globalscope:
  8608. case no_colon:
  8609. {
  8610. bool wasLocal = insideForceLocal;
  8611. unsigned oldDepth = allNodesDepth;
  8612. insideForceLocal = false;
  8613. allNodesDepth = 0;
  8614. IHqlExpression * ret = NewHqlTransformer::createTransformed(expr);
  8615. insideForceLocal = wasLocal;
  8616. allNodesDepth = oldDepth;
  8617. return ret;
  8618. }
  8619. }
  8620. OwnedHqlExpr ret = NewHqlTransformer::createTransformed(expr);
  8621. if (!insideForceLocal || !localChangesActivity(expr) || expr->hasAttribute(noLocalAtom))
  8622. return ret.getClear();
  8623. return appendLocalAttribute(ret);
  8624. }
  8625. ANewTransformInfo * ForceLocalTransformer::createTransformInfo(IHqlExpression * expr)
  8626. {
  8627. return CREATE_NEWTRANSFORMINFO(ForceLocalTransformInfo, expr);
  8628. }
  8629. IHqlExpression * ForceLocalTransformer::queryAlreadyTransformed(IHqlExpression * expr)
  8630. {
  8631. ForceLocalTransformInfo * extra = queryExtra(expr);
  8632. return extra->localTransformed[boolToInt(insideForceLocal)][boolToInt(insideAllNodes())];
  8633. }
  8634. void ForceLocalTransformer::setTransformed(IHqlExpression * expr, IHqlExpression * transformed)
  8635. {
  8636. ForceLocalTransformInfo * extra = queryExtra(expr);
  8637. extra->localTransformed[boolToInt(insideForceLocal)][boolToInt(insideAllNodes())].set(transformed);
  8638. }
  8639. IHqlExpression * ForceLocalTransformer::queryAlreadyTransformedSelector(IHqlExpression * expr)
  8640. {
  8641. ForceLocalTransformInfo * extra = queryExtra(expr);
  8642. return extra->localTransformedSelector[boolToInt(insideForceLocal)][boolToInt(insideAllNodes())];
  8643. }
  8644. void ForceLocalTransformer::setTransformedSelector(IHqlExpression * expr, IHqlExpression * transformed)
  8645. {
  8646. ForceLocalTransformInfo * extra = queryExtra(expr);
  8647. extra->localTransformedSelector[boolToInt(insideForceLocal)][boolToInt(insideAllNodes())].set(transformed);
  8648. }
  8649. //---------------------------------------------------------------------------
  8650. /*
  8651. This transform is responsible for ensuring that all datasets get converted to link counted datasets. Because all records of the same type get converted it is necessary to
  8652. transform the index and table definitions - otherwise a single record would need to be transformed in multiple different ways, which would require a much more
  8653. complicated transformation. However there are some exceptions:
  8654. a) weird field dataset syntax which dates back to hole days is no transformed.
  8655. b) add a project before a build index to ensure the code is more efficient. (It will be later moved over any intervening sorts).
  8656. c) ensure input and out from a pipe is serialized (so that sizeof(inputrow) returns a sensible value)
  8657. */
  8658. static HqlTransformerInfo hqlLinkedChildRowTransformerInfo("HqlLinkedChildRowTransformer");
  8659. HqlLinkedChildRowTransformer::HqlLinkedChildRowTransformer(bool _implicitLinkedChildRows) : QuickHqlTransformer(hqlLinkedChildRowTransformerInfo, NULL)
  8660. {
  8661. implicitLinkedChildRows = _implicitLinkedChildRows;
  8662. }
  8663. IHqlExpression * HqlLinkedChildRowTransformer::ensureInputSerialized(IHqlExpression * expr)
  8664. {
  8665. LinkedHqlExpr dataset = expr->queryChild(0);
  8666. IHqlExpression * record = dataset->queryRecord();
  8667. OwnedHqlExpr serializedRecord = getSerializedForm(record, diskAtom);
  8668. //If the dataset requires serialization, it is much more efficient to serialize before the sort, than to serialize after.
  8669. if (record == serializedRecord)
  8670. return LINK(expr);
  8671. OwnedHqlExpr selSeq = createSelectorSequence();
  8672. //The expression/transform has references to the in-memory selector, but the selector provided to transform will be serialised.
  8673. //so create a mapping <unserialized> := f(serialized)
  8674. //and then use it to expand references to the unserialized format
  8675. IHqlExpression * selector = dataset->queryNormalizedSelector();
  8676. OwnedHqlExpr mapTransform = createRecordMappingTransform(no_transform, serializedRecord, selector);
  8677. OwnedHqlExpr newDataset = createDatasetF(no_newusertable, LINK(dataset), LINK(serializedRecord), LINK(mapTransform), LINK(selSeq), NULL);
  8678. NewProjectMapper2 mapper;
  8679. mapper.setMapping(mapTransform);
  8680. HqlExprArray oldArgs, expandedArgs, newArgs;
  8681. newArgs.append(*LINK(newDataset));
  8682. unwindChildren(oldArgs, expr, 1);
  8683. mapper.expandFields(expandedArgs, oldArgs, dataset, newDataset, selector);
  8684. //Finally replace any remaining selectors - so that sizeof(ds) gets mapped correctly. Obscure..... but could possibly be used in a pipe,repeat
  8685. ForEachItemIn(i, expandedArgs)
  8686. newArgs.append(*replaceSelector(&expandedArgs.item(i), selector, newDataset->queryNormalizedSelector()));
  8687. return expr->clone(newArgs);
  8688. }
  8689. IHqlExpression * HqlLinkedChildRowTransformer::transformBuildIndex(IHqlExpression * expr)
  8690. {
  8691. return ensureInputSerialized(expr);
  8692. }
  8693. IHqlExpression * HqlLinkedChildRowTransformer::transformPipeThrough(IHqlExpression * expr)
  8694. {
  8695. //serialize input to pipe through, so that if they happen to use sizeof(ds) on the input it will give the
  8696. //non serialized format. No major need to ensure output is not serialized.
  8697. return ensureInputSerialized(expr);
  8698. }
  8699. IHqlExpression * HqlLinkedChildRowTransformer::createTransformedBody(IHqlExpression * expr)
  8700. {
  8701. switch (expr->getOperator())
  8702. {
  8703. case no_field:
  8704. {
  8705. ITypeInfo * type = expr->queryType();
  8706. switch (type->getTypeCode())
  8707. {
  8708. case type_dictionary:
  8709. case type_table:
  8710. case type_groupedtable:
  8711. OwnedHqlExpr transformedRecord = transform(expr->queryRecord());
  8712. if (recordRequiresLinkCount(transformedRecord))
  8713. {
  8714. if (expr->hasAttribute(embeddedAtom) || queryAttribute(type, embeddedAtom) || expr->hasAttribute(countAtom) || expr->hasAttribute(sizeofAtom))
  8715. throwError1(HQLERR_InconsistentEmbedded, str(expr->queryId()));
  8716. }
  8717. if (expr->hasAttribute(embeddedAtom) || queryAttribute(type, embeddedAtom))
  8718. {
  8719. HqlExprArray args;
  8720. transformChildren(expr, args);
  8721. removeAttribute(args, embeddedAtom);
  8722. OwnedITypeInfo newType = transformType(type);
  8723. OwnedITypeInfo cleanType = removeAttribute(newType, embeddedAtom);
  8724. return createField(expr->queryId(), cleanType.getClear(), args);
  8725. }
  8726. if (implicitLinkedChildRows && !expr->hasAttribute(_linkCounted_Atom) && !queryAttribute(type, embeddedAtom))
  8727. {
  8728. //Don't use link counted rows for weird HOLe style dataset attributes
  8729. if (expr->hasAttribute(countAtom) || expr->hasAttribute(sizeofAtom))
  8730. break;
  8731. //add the attribute first so a no linked field doesn't contain a record that requires it
  8732. OwnedHqlExpr modified = appendOwnedOperand(expr, getLinkCountedAttr());
  8733. return transform(modified);
  8734. }
  8735. break;
  8736. }
  8737. break;
  8738. }
  8739. case no_buildindex:
  8740. {
  8741. OwnedHqlExpr transformed = QuickHqlTransformer::createTransformedBody(expr);
  8742. return transformBuildIndex(transformed);
  8743. }
  8744. case no_pipe:
  8745. if (expr->queryRecord())
  8746. {
  8747. return QuickHqlTransformer::createTransformedBody(expr);
  8748. }
  8749. break;
  8750. case no_output:
  8751. //would this be a good idea for output to file? output to pipe?
  8752. if (false)
  8753. {
  8754. IHqlExpression * filename = queryRealChild(expr, 2);
  8755. if (filename)
  8756. {
  8757. OwnedHqlExpr transformed = QuickHqlTransformer::createTransformedBody(expr);
  8758. return ensureInputSerialized(transformed);
  8759. }
  8760. }
  8761. break;
  8762. case no_embedbody:
  8763. if (expr->queryAttribute(languageAtom))
  8764. break;
  8765. //Don't change the type of an embedded C++ body - otherwise result it will become link counted when not expected.
  8766. // Fall into...
  8767. case no_attr:
  8768. return LINK(expr);
  8769. }
  8770. return QuickHqlTransformer::createTransformedBody(expr);
  8771. }
  8772. //---------------------------------------------------------------------------
  8773. HqlScopeTaggerInfo::HqlScopeTaggerInfo(IHqlExpression * _expr) : MergingTransformInfo(_expr)
  8774. {
  8775. if (!onlyTransformOnce() && isIndependentOfScope(_expr))
  8776. {
  8777. //If the node doesn't have any active selectors then it isn't going to be context dependent
  8778. setOnlyTransformOnce(true);
  8779. }
  8780. }
  8781. ANewTransformInfo * HqlScopeTagger::createTransformInfo(IHqlExpression * expr)
  8782. {
  8783. return CREATE_NEWTRANSFORMINFO(HqlScopeTaggerInfo, expr);
  8784. }
  8785. /*
  8786. * This transformation converts references to datasets which are ambiguous. There are two main examples:
  8787. * ds1.childds.x.
  8788. * ECL supports implicit normalization of datasets. i.e., ds1.childds is treated as normalize(ds1, left.childds).
  8789. * Unfortunately this means that exists(ds1.childds(age=10)) has a different meaning depending on its context:
  8790. * exists(ds1(exists(ds1.childds(age=10))) - here childds is iterated within the current row of ds1
  8791. *
  8792. * SELF.x := ds1
  8793. * This either means the entire ds1, or the current row in ds1 = depending on whether dataset is active or not
  8794. * (e.g, due to a TABLE() statement.) This code reports a warning in this case - active(ds1) should be used for
  8795. * the second case.
  8796. *
  8797. * In order to achieve this the entire expression tree needs to be transformed differently within each potentital dataset
  8798. * scope combination.
  8799. Details of the no_select representation:
  8800. no_select
  8801. if the lhs dataset is in scope then no flags are attached. if the lhs is not in scope, then a newAtom attribute is
  8802. attached to select node. That may contain the following attributes:
  8803. global - is an outer level activity
  8804. noTable - no other tables are active at this point e.g., global[1].field;
  8805. relatedTable - I think this could be deprecated in favour of the flag above.
  8806. Datasets
  8807. In other situations where a dataset/datarow is ambiguous whether it is new or in scope then a no_activerow
  8808. is inserted to indicate it is in scope.
  8809. Note:
  8810. new attributes aren't created for subfield e.g, myDataset.level1.level2.
  8811. This means that if (myDataset.level1) is expanded to some expression then a newAtom will be need to be created on the new select
  8812. */
  8813. static HqlTransformerInfo hqlScopeTaggerInfo("HqlScopeTagger");
  8814. HqlScopeTagger::HqlScopeTagger(IErrorReceiver & _errors, ErrorSeverityMapper & _errorMapper)
  8815. : ScopedDependentTransformer(hqlScopeTaggerInfo), errors(_errors), errorMapper(_errorMapper)
  8816. {
  8817. }
  8818. static const char * getECL(IHqlExpression * expr, StringBuffer & s)
  8819. {
  8820. toUserECL(s, expr, false);
  8821. if (s.length() > 2)
  8822. s.setLength(s.length()-2);
  8823. return s.str();
  8824. }
  8825. void HqlScopeTagger::checkActiveRow(IHqlExpression * expr)
  8826. {
  8827. if (!isDatasetActive(expr))
  8828. {
  8829. StringBuffer exprText;
  8830. getECL(expr, exprText);
  8831. elideString(exprText, 20);
  8832. VStringBuffer msg("ROW(%s) - dataset argument is not in scope. Did you mean dataset[1]?", exprText.str());
  8833. reportError(CategoryError, msg);
  8834. }
  8835. }
  8836. void HqlScopeTagger::reportSelectorError(IHqlExpression * selector, IHqlExpression * expr)
  8837. {
  8838. ScopeInfo * scope = innerScope;
  8839. if (innerScope && innerScope->isEmpty() && scopeStack.ordinality() > 1)
  8840. scope = &scopeStack.item(scopeStack.ordinality()-2);
  8841. StringBuffer exprText, datasetName, scopeName;
  8842. StringBuffer msg;
  8843. if (scope && scope->dataset)
  8844. {
  8845. IHqlExpression * topScope = scope->dataset;
  8846. msg.appendf("%s - Table %s is not related to %s",
  8847. getECL(expr, exprText),
  8848. getExprIdentifier(datasetName, selector).str(), getExprIdentifier(scopeName, topScope).str());
  8849. }
  8850. else if (scope && scope->left)
  8851. {
  8852. msg.appendf("%s - no active row for Table %s inside transform (use LEFT?)",
  8853. getECL(expr, exprText),
  8854. getExprIdentifier(datasetName, selector).str());
  8855. }
  8856. else
  8857. {
  8858. msg.appendf("%s - no specified row for Table %s", getECL(expr, exprText),
  8859. getExprIdentifier(datasetName, selector).str());
  8860. }
  8861. reportError(CategoryError, msg);
  8862. }
  8863. IHqlExpression * HqlScopeTagger::transformSelect(IHqlExpression * expr)
  8864. {
  8865. IHqlExpression * ds = expr->queryChild(0);
  8866. if (isDatasetActive(ds))
  8867. {
  8868. if (!innerScope && scopeStack.ordinality() == 0)
  8869. {
  8870. ds = queryDatasetCursor(ds);
  8871. switch (ds->getOperator())
  8872. {
  8873. case no_left:
  8874. case no_right:
  8875. StringBuffer exprText, datasetName;
  8876. VStringBuffer msg("%s - %s not in scope, possibly passed into a global/workflow definition", getECL(expr, exprText), getExprIdentifier(datasetName, ds).str());
  8877. reportError(CategoryError, msg);
  8878. break;
  8879. }
  8880. }
  8881. return Parent::createTransformed(expr); // this will call transformSelector() on lhs since new is not present
  8882. }
  8883. IHqlExpression * cursor = queryDatasetCursor(ds);
  8884. if (cursor->isDataset())
  8885. {
  8886. if (expr->isDictionary())
  8887. {
  8888. StringBuffer exprText;
  8889. VStringBuffer msg("dictionary %s must be explicitly NORMALIZED", getECL(expr, exprText));
  8890. reportError(CategoryError, msg);
  8891. }
  8892. else if (!expr->isDataset())
  8893. {
  8894. if (!isDatasetARow(ds))
  8895. reportSelectorError(ds, expr);
  8896. }
  8897. }
  8898. pushScope(expr);
  8899. OwnedHqlExpr newDs = transformNewDataset(ds, false);
  8900. popScope();
  8901. IHqlExpression * field = expr->queryChild(1);
  8902. if (ds->isDataset())
  8903. {
  8904. if (!expr->isDataset() && !expr->isDatarow() && !expr->isDictionary())
  8905. {
  8906. //If the left is a dataset, and the right isn't a dataset or a datarow then this doesn't make sense - it is an illegal
  8907. return createSelectExpr(newDs.getClear(), LINK(field));
  8908. }
  8909. }
  8910. //MORE: What about child datasets - should really be tagged as
  8911. //if (!isNewDataset && field->isDataset() && !containsSelf(ds) && !isDatasetActive(ds)) isNew = true;
  8912. if ((newDs->getOperator() == no_select) && newDs->isDatarow())
  8913. return createSelectExpr(newDs.getClear(), LINK(field));
  8914. return createNewSelectExpr(newDs.getClear(), LINK(field));
  8915. }
  8916. IHqlExpression * HqlScopeTagger::transformSelectorsAttr(IHqlExpression * expr)
  8917. {
  8918. HqlExprArray args;
  8919. HqlExprArray transformedArgs;
  8920. unwindChildren(args, expr);
  8921. ForEachItemIn(i, args)
  8922. {
  8923. IHqlExpression & cur = args.item(i);
  8924. assertex(cur.getOperator() == no_select);
  8925. //Only retain selectors for datasets which are in scope.
  8926. if (isDatasetActive(cur.queryChild(0)))
  8927. transformedArgs.append(*transformSelector(&cur));
  8928. }
  8929. return expr->clone(transformedArgs);
  8930. }
  8931. IHqlExpression * HqlScopeTagger::transformNewDataset(IHqlExpression * expr, bool isActiveOk)
  8932. {
  8933. node_operator op = expr->getOperator();
  8934. switch (op)
  8935. {
  8936. case no_activerow:
  8937. {
  8938. IHqlExpression * arg0 = expr->queryChild(0);
  8939. checkActiveRow(arg0);
  8940. OwnedHqlExpr transformedArg = transformSelector(arg0);
  8941. return ensureActiveRow(transformedArg);
  8942. }
  8943. }
  8944. OwnedHqlExpr transformed = transform(expr);
  8945. switch (op)
  8946. {
  8947. //MORE: I'm still not quite sure the active tagging of rows is right to have these as exceptions...
  8948. case no_left:
  8949. case no_right:
  8950. case no_matchattr:
  8951. return transformed.getClear();
  8952. case no_select:
  8953. if (isDatasetActive(expr))
  8954. {
  8955. IHqlExpression * ds = expr->queryChild(0);
  8956. if (!isAlwaysActiveRow(ds))
  8957. {
  8958. StringBuffer exprText;
  8959. VStringBuffer msg("%s - Need to use active(dataset) to refer to the current row of an active dataset", getECL(expr, exprText));
  8960. reportError(CategoryError, msg);
  8961. }
  8962. }
  8963. return transformed.getClear();
  8964. }
  8965. if (isDatasetActive(expr))
  8966. {
  8967. if (!isActiveOk)
  8968. {
  8969. StringBuffer exprText;
  8970. VStringBuffer msg("%s - Need to use active(dataset) to refer to the current row of an active dataset", getECL(expr, exprText));
  8971. reportError(CategoryError, msg);
  8972. }
  8973. return ensureActiveRow(transformed->queryNormalizedSelector());
  8974. }
  8975. switch (op)
  8976. {
  8977. case no_if:
  8978. case no_chooseds:
  8979. {
  8980. HqlExprArray args;
  8981. args.append(*transform(expr->queryChild(0)));
  8982. args.append(*transformNewDataset(expr->queryChild(1), false));
  8983. ForEachChildFrom(i, expr, 2)
  8984. args.append(*transformNewDataset(expr->queryChild(i), false));
  8985. return expr->clone(args);
  8986. }
  8987. case no_addfiles:
  8988. case no_projectrow:
  8989. return transformAmbiguousChildren(expr);
  8990. case no_case:
  8991. case no_map:
  8992. throwUnexpected(); // should have been converted to no_if by now...
  8993. default:
  8994. return transformed.getClear();
  8995. }
  8996. }
  8997. IHqlExpression * HqlScopeTagger::transformAmbiguous(IHqlExpression * expr, bool isActiveOk)
  8998. {
  8999. ITypeInfo * type = expr->queryType();
  9000. type_t tc = type_void;
  9001. if (type)
  9002. tc = type->getTypeCode();
  9003. switch (tc)
  9004. {
  9005. case type_void:
  9006. return transformAmbiguousChildren(expr);
  9007. case type_table:
  9008. case type_groupedtable:
  9009. {
  9010. pushScope(expr);
  9011. OwnedHqlExpr ret = transformNewDataset(expr, isActiveOk);
  9012. popScope();
  9013. return ret.getClear();
  9014. }
  9015. }
  9016. return transform(expr);
  9017. }
  9018. IHqlExpression * HqlScopeTagger::transformAmbiguousChildren(IHqlExpression * expr)
  9019. {
  9020. unsigned max = expr->numChildren();
  9021. if (max == 0)
  9022. return LINK(expr);
  9023. bool same = true;
  9024. HqlExprArray args;
  9025. args.ensure(max);
  9026. for(unsigned i=0; i < max; i++)
  9027. {
  9028. IHqlExpression * cur = expr->queryChild(i);
  9029. IHqlExpression * tr = transformAmbiguous(cur, false);
  9030. args.append(*tr);
  9031. if (cur != tr)
  9032. same = false;
  9033. }
  9034. if (same)
  9035. return LINK(expr);
  9036. return expr->clone(args);
  9037. }
  9038. IHqlExpression * HqlScopeTagger::transformCall(IHqlExpression * expr)
  9039. {
  9040. unsigned max = expr->numChildren();
  9041. bool same = true;
  9042. HqlExprArray args;
  9043. args.ensure(max);
  9044. for(unsigned i=0; i < max; i++)
  9045. {
  9046. IHqlExpression * cur = expr->queryChild(i);
  9047. IHqlExpression * tr = transformAmbiguous(cur, false);
  9048. args.append(*tr);
  9049. if (cur != tr)
  9050. same = false;
  9051. }
  9052. IHqlExpression * funcdef = expr->queryFunctionDefinition();
  9053. OwnedHqlExpr newFuncDef = transform(funcdef);
  9054. if (same && funcdef == newFuncDef)
  9055. return LINK(expr);
  9056. return createReboundFunction(newFuncDef, args);
  9057. }
  9058. IHqlExpression * HqlScopeTagger::transformSizeof(IHqlExpression * expr)
  9059. {
  9060. IHqlExpression * arg = expr->queryChild(0)->queryNormalizedSelector();
  9061. //Sizeof (dataset.somefield(<new>)) - convert to sizeof(record.somefield), so the argument doesn't get hoisted incorrectly, and don't get a scope error
  9062. OwnedHqlExpr newArg;
  9063. if (arg->getOperator() == no_select)
  9064. {
  9065. IHqlExpression * ds = arg->queryChild(0);
  9066. IHqlExpression * cursor = queryDatasetCursor(ds);
  9067. if (!isDatasetActive(cursor) && cursor->isDataset())
  9068. newArg.setown(createSelectExpr(LINK(ds->queryRecord()), LINK(arg->queryChild(1))));
  9069. }
  9070. if (!newArg)
  9071. newArg.setown(transformAmbiguous(arg, true));
  9072. HqlExprArray args;
  9073. args.append(*newArg.getClear());
  9074. return completeTransform(expr, args);
  9075. }
  9076. IHqlExpression * HqlScopeTagger::transformWithin(IHqlExpression * dataset, IHqlExpression * scope)
  9077. {
  9078. while (dataset->getOperator() == no_related)
  9079. dataset = dataset->queryChild(0);
  9080. if (dataset->getOperator() != no_select)
  9081. {
  9082. StringBuffer exprText;
  9083. VStringBuffer msg("%s - dataset filtered by WITHIN is too complex", getECL(dataset, exprText));
  9084. reportError(CategoryError, msg);
  9085. return transform(dataset);
  9086. }
  9087. IHqlExpression * ds = dataset->queryChild(0);
  9088. IHqlExpression * field = dataset->queryChild(1);
  9089. if (ds->queryNormalizedSelector() == scope)
  9090. {
  9091. OwnedHqlExpr newDs = transform(ds);
  9092. return createSelectExpr(newDs.getClear(), LINK(field));
  9093. }
  9094. OwnedHqlExpr newDs = transformWithin(ds, scope);
  9095. return createNewSelectExpr(newDs.getClear(), LINK(field));
  9096. }
  9097. IHqlExpression * HqlScopeTagger::transformRelated(IHqlExpression * expr)
  9098. {
  9099. IHqlExpression * ds = expr->queryChild(0);
  9100. IHqlExpression * scope = expr->queryChild(1);
  9101. if (!isDatasetActive(scope))
  9102. {
  9103. StringBuffer exprText;
  9104. VStringBuffer msg("dataset \"%s\" used in WITHIN is not in scope", getECL(scope, exprText));
  9105. reportError(CategoryError, msg);
  9106. }
  9107. //Check the ds is a table
  9108. IHqlDataset * scopeDs = scope->queryDataset();
  9109. if (scopeDs != scopeDs->queryTable())
  9110. {
  9111. StringBuffer exprText;
  9112. VStringBuffer msg("dataset \"%s\" used as parameter to WITHIN is too complex", getECL(expr, exprText));
  9113. reportError(CategoryError, msg);
  9114. }
  9115. return transformWithin(ds, scope->queryNormalizedSelector());
  9116. }
  9117. IHqlExpression * HqlScopeTagger::createTransformed(IHqlExpression * expr)
  9118. {
  9119. IHqlExpression * body = expr->queryBody(true);
  9120. if (expr != body)
  9121. {
  9122. switch (expr->getAnnotationKind())
  9123. {
  9124. case annotate_meta:
  9125. {
  9126. unsigned max = errorMapper.processMetaAnnotation(expr);
  9127. OwnedHqlExpr transformedBody = transform(body);
  9128. errorMapper.restoreLocalOnWarnings(max);
  9129. if (body == transformedBody)
  9130. return LINK(expr);
  9131. return expr->cloneAnnotation(transformedBody);
  9132. }
  9133. break;
  9134. case annotate_symbol:
  9135. {
  9136. ErrorSeverityMapper::SymbolScope saved(errorMapper, expr);
  9137. OwnedHqlExpr transformedBody = transform(body);
  9138. if (body == transformedBody)
  9139. return LINK(expr);
  9140. return expr->cloneAnnotation(transformedBody);
  9141. }
  9142. case annotate_location:
  9143. {
  9144. break;
  9145. }
  9146. }
  9147. OwnedHqlExpr transformedBody = transform(body);
  9148. if (body == transformedBody)
  9149. return LINK(expr);
  9150. return expr->cloneAnnotation(transformedBody);
  9151. }
  9152. switch (expr->getOperator())
  9153. {
  9154. case no_left:
  9155. case no_right:
  9156. case no_self:
  9157. case no_top:
  9158. return LINK(expr);
  9159. case no_activerow:
  9160. checkActiveRow(expr->queryChild(0));
  9161. break;
  9162. case no_select:
  9163. return transformSelect(expr);
  9164. case no_externalcall:
  9165. case no_rowvalue:
  9166. // case no_addfiles:
  9167. // case no_libraryscopeinstance:??
  9168. return transformAmbiguousChildren(expr);
  9169. case no_call:
  9170. return transformCall(expr);
  9171. case no_offsetof:
  9172. case no_sizeof:
  9173. return transformSizeof(expr);
  9174. case no_attr_expr:
  9175. if (expr->queryName() == _selectors_Atom)
  9176. return transformSelectorsAttr(expr);
  9177. return transformAmbiguousChildren(expr);
  9178. case no_datasetfromrow:
  9179. {
  9180. IHqlExpression * ds = expr->queryChild(0);
  9181. if (ds->isDataset() && !isDatasetActive(ds))
  9182. {
  9183. StringBuffer exprText;
  9184. VStringBuffer msg("dataset %s mistakenly interpreted as a datarow, possibly due to missing dataset() in parameter type", getECL(ds, exprText));
  9185. reportError(CategoryError, msg);
  9186. }
  9187. return transformAmbiguousChildren(expr);
  9188. }
  9189. case no_temptable:
  9190. if (expr->queryChild(0)->isDatarow())
  9191. return transformAmbiguousChildren(expr);
  9192. break;
  9193. case no_related:
  9194. return transformRelated(expr);
  9195. case no_eq:
  9196. case no_ne:
  9197. case no_lt:
  9198. case no_le:
  9199. case no_gt:
  9200. case no_ge:
  9201. case no_order:
  9202. //MORE: Should check this doesn't make the comparison invalid.
  9203. return transformAmbiguousChildren(expr);
  9204. case no_assign:
  9205. {
  9206. IHqlExpression * lhs = expr->queryChild(0);
  9207. IHqlExpression * rhs = expr->queryChild(1);
  9208. OwnedHqlExpr newRhs = transformAmbiguous(rhs, false);
  9209. if (lhs->isDatarow() && newRhs->isDataset())
  9210. {
  9211. StringBuffer exprText;
  9212. VStringBuffer msg("dataset expression (%s) assigned to field '%s' with type row", getECL(rhs, exprText), str(lhs->queryChild(1)->queryName()));
  9213. reportError(CategoryError, msg.str());
  9214. }
  9215. if (rhs == newRhs)
  9216. return LINK(expr);
  9217. HqlExprArray children;
  9218. children.append(*LINK(expr->queryChild(0)));
  9219. children.append(*newRhs.getClear());
  9220. return completeTransform(expr, children);
  9221. }
  9222. break;
  9223. case no_evaluate:
  9224. throwUnexpected();
  9225. case no_projectrow:
  9226. {
  9227. OwnedHqlExpr transformed = Parent::createTransformed(expr);
  9228. if (transformed->queryChild(0)->isDataset())
  9229. reportError(CategoryError, "PROJECT() row argument resolved to a dataset. Missing DATASET() from parameter type?");
  9230. return transformed.getClear();
  9231. }
  9232. case no_merge:
  9233. {
  9234. HqlExprArray children;
  9235. transformChildren(expr, children);
  9236. IHqlExpression * sorted = queryAttribute(sortedAtom, children);
  9237. if (!sorted || queryAttribute(_implicitSorted_Atom, children))
  9238. {
  9239. IHqlExpression * dataset = &children.item(0);
  9240. removeAttribute(children, _implicitSorted_Atom);
  9241. if (sorted)
  9242. children.zap(*sorted);
  9243. HqlExprArray sorts;
  9244. OwnedHqlExpr order = getExistingSortOrder(dataset, expr->hasAttribute(localAtom), true);
  9245. if (order)
  9246. unwindChildren(sorts, order);
  9247. ForEachItemInRev(i, sorts)
  9248. {
  9249. if (sorts.item(i).isAttribute())
  9250. {
  9251. reportError(CategorySyntax, HQLWRN_MergeBadSortOrder_Text);
  9252. sorts.remove(i);
  9253. }
  9254. }
  9255. children.append(*createExprAttribute(sortedAtom, sorts));
  9256. }
  9257. return expr->clone(children);
  9258. }
  9259. }
  9260. return Parent::createTransformed(expr);
  9261. }
  9262. void HqlScopeTagger::reportError(WarnErrorCategory category, const char * msg)
  9263. {
  9264. IHqlExpression * location = errorMapper.queryActiveSymbol();
  9265. //Make this an error when we are confident...
  9266. int startLine= location ? location->getStartLine() : 0;
  9267. int startColumn = location ? location->getStartColumn() : 0;
  9268. ISourcePath * sourcePath = location ? location->querySourcePath() : NULL;
  9269. ErrorSeverity severity = queryDefaultSeverity(category);
  9270. Owned<IError> err = createError(category, severity, ERR_ASSERT_WRONGSCOPING, msg, str(sourcePath), startLine, startColumn, 0);
  9271. errors.report(err); // will throw immediately if it is an error.
  9272. }
  9273. //---------------------------------------------------------------------------------------------------------------------
  9274. SharedTableInfo * ImplicitAliasTransformInfo::uses(IHqlExpression * tableBody) const
  9275. {
  9276. ForEachItemIn(i, sharedTables)
  9277. {
  9278. SharedTableInfo & cur = sharedTables.item(i);
  9279. if (cur.dataset == tableBody)
  9280. return &cur;
  9281. }
  9282. return NULL;
  9283. }
  9284. void ImplicitAliasTransformInfo::add(SharedTableInfo * table)
  9285. {
  9286. sharedTables.append(*LINK(table));
  9287. }
  9288. void ImplicitAliasTransformInfo::addAmbiguity(SharedTableInfo * table)
  9289. {
  9290. containsAmbiguity = true;
  9291. merge(table);
  9292. }
  9293. void ImplicitAliasTransformInfo::merge(SharedTableInfo * table)
  9294. {
  9295. ForEachItemIn(i, sharedTables)
  9296. {
  9297. SharedTableInfo & cur = sharedTables.item(i);
  9298. if (cur.dataset == table->dataset)
  9299. {
  9300. if (cur.depth < table->depth)
  9301. sharedTables.replace(*LINK(table), i);
  9302. return;
  9303. }
  9304. }
  9305. add(table);
  9306. }
  9307. void ImplicitAliasTransformInfo::inherit(const ImplicitAliasTransformInfo * other)
  9308. {
  9309. ForEachItemIn(i, other->sharedTables)
  9310. merge(&other->sharedTables.item(i));
  9311. }
  9312. static HqlTransformerInfo implicitAliasTransformerInfo("ImplicitAliasTransformer");
  9313. ImplicitAliasTransformer::ImplicitAliasTransformer() : NewHqlTransformer(implicitAliasTransformerInfo)
  9314. {
  9315. seenShared = true;
  9316. seenAmbiguity = false;
  9317. }
  9318. void ImplicitAliasTransformer::analyseExpr(IHqlExpression * _expr)
  9319. {
  9320. IHqlExpression * body = _expr->queryBody();
  9321. if (alreadyVisited(body))
  9322. {
  9323. if ((pass == 0) && body->isDataset())
  9324. {
  9325. switch (body->getOperator())
  9326. {
  9327. case no_rows:
  9328. break;
  9329. default:
  9330. seenShared = true;
  9331. queryExtra(body)->shared.setown(new SharedTableInfo(body, 0));
  9332. break;
  9333. }
  9334. }
  9335. return;
  9336. }
  9337. NewHqlTransformer::analyseExpr(body);
  9338. if (pass == 0)
  9339. return;
  9340. ImplicitAliasTransformInfo * extra = queryExtra(body);
  9341. if (extra->shared)
  9342. extra->add(extra->shared);
  9343. switch (body->getOperator())
  9344. {
  9345. case no_activerow:
  9346. case no_filepos:
  9347. case no_file_logicalname:
  9348. case no_offsetof:
  9349. case no_joined:
  9350. case no_colon:
  9351. case no_globalscope:
  9352. case no_attr:
  9353. return;
  9354. case no_select:
  9355. {
  9356. bool isNew;
  9357. IHqlExpression * ds = querySelectorDataset(body, isNew);
  9358. if (isNew)
  9359. {
  9360. ImplicitAliasTransformInfo * dsExtra = queryExtra(ds->queryBody());
  9361. extra->inherit(dsExtra);
  9362. }
  9363. return;
  9364. }
  9365. }
  9366. IHqlExpression * dataset = NULL;
  9367. switch (getChildDatasetType(body))
  9368. {
  9369. case childdataset_none:
  9370. case childdataset_many_noscope:
  9371. case childdataset_many:
  9372. case childdataset_map:
  9373. case childdataset_dataset_noscope:
  9374. case childdataset_if:
  9375. case childdataset_case:
  9376. case childdataset_evaluate:
  9377. case childdataset_left:
  9378. case childdataset_leftright:
  9379. case childdataset_same_left_right:
  9380. case childdataset_nway_left_right:
  9381. break;
  9382. case childdataset_dataset:
  9383. case childdataset_datasetleft:
  9384. case childdataset_top_left_right:
  9385. dataset = body->queryChild(0)->queryBody();
  9386. break;
  9387. default:
  9388. UNIMPLEMENTED;
  9389. }
  9390. ForEachChild(i, body)
  9391. {
  9392. IHqlExpression * cur = body->queryChild(i);
  9393. ImplicitAliasTransformInfo * childExtra = queryExtra(cur->queryBody());
  9394. //If this is one of the arguments to an operation which has an active top dataset,
  9395. //check to see if any of the contained expressions reference this item
  9396. if (dataset && (i != 0))
  9397. {
  9398. SharedTableInfo * match = childExtra->uses(dataset);
  9399. if (match)
  9400. {
  9401. seenAmbiguity = true;
  9402. SharedTableInfo * nested = createAmbiguityInfo(match->dataset, match->depth+1);
  9403. extra->addAmbiguity(nested);
  9404. }
  9405. // dbglogExpr(_expr);
  9406. // DBGLOG("Implicit nested table ambiguity spotted in expression");
  9407. }
  9408. extra->inherit(childExtra);
  9409. }
  9410. }
  9411. SharedTableInfo * ImplicitAliasTransformer::createAmbiguityInfo(IHqlExpression * dataset, unsigned depth)
  9412. {
  9413. ForEachItemIn(i, ambiguousTables)
  9414. {
  9415. SharedTableInfo & cur = ambiguousTables.item(i);
  9416. if ((cur.dataset == dataset) && (depth == cur.depth))
  9417. return &cur;
  9418. }
  9419. ambiguousTables.append(*new SharedTableInfo(dataset, depth));
  9420. return &ambiguousTables.tos();
  9421. }
  9422. ANewTransformInfo * ImplicitAliasTransformer::createTransformInfo(IHqlExpression * expr)
  9423. {
  9424. return CREATE_NEWTRANSFORMINFO(ImplicitAliasTransformInfo, expr);
  9425. }
  9426. IHqlExpression * ImplicitAliasTransformer::createTransformed(IHqlExpression * expr)
  9427. {
  9428. IHqlExpression * body = expr->queryBody();
  9429. if (expr != body)
  9430. {
  9431. OwnedHqlExpr newBody = transform(body);
  9432. return expr->cloneAllAnnotations(newBody);
  9433. }
  9434. OwnedHqlExpr transformed = NewHqlTransformer::createTransformed(expr);
  9435. updateOrphanedSelectors(transformed, expr);
  9436. ImplicitAliasTransformInfo * extra = queryExtra(body);
  9437. if (!extra->containsAmbiguity)
  9438. return transformed.getClear();
  9439. SharedTableInfo * match = extra->uses(body->queryChild(0)->queryBody());
  9440. assertex(match && match->depth != 0);
  9441. if (!match->uid)
  9442. match->uid.setown(createUniqueId());
  9443. OwnedHqlExpr aliased;
  9444. IHqlExpression * dataset = transformed->queryChild(0)->queryBody();
  9445. IHqlExpression * uid = match->uid;
  9446. aliased.setown(createDataset(no_dataset_alias, LINK(dataset), LINK(uid)));
  9447. //Replace dataset with an aliased variety, and remap all the selectors
  9448. HqlExprArray args;
  9449. args.append(*LINK(aliased));
  9450. replaceSelectors(args, transformed, 1, dataset->queryNormalizedSelector(), aliased->queryNormalizedSelector());
  9451. return transformed->clone(args);
  9452. }
  9453. void ImplicitAliasTransformer::process(HqlExprArray & exprs)
  9454. {
  9455. analyseArray(exprs, 0);
  9456. if (!seenShared)
  9457. return;
  9458. analyseArray(exprs, 1);
  9459. if (hasAmbiguity())
  9460. {
  9461. // DBGLOG("Implicit nested dataset ambiguity spotted in expression");
  9462. HqlExprArray transformed;
  9463. transformRoot(exprs, transformed);
  9464. replaceArray(exprs, transformed);
  9465. }
  9466. }
  9467. //---------------------------------------------------------------------------------------------------------------------
  9468. SharedTableInfo * LeftRightTransformInfo::uses(IHqlExpression * tableBody) const
  9469. {
  9470. ForEachItemIn(i, sharedTables)
  9471. {
  9472. SharedTableInfo & cur = sharedTables.item(i);
  9473. if (cur.dataset == tableBody)
  9474. return &cur;
  9475. }
  9476. return NULL;
  9477. }
  9478. void LeftRightTransformInfo::add(SharedTableInfo * table)
  9479. {
  9480. sharedTables.append(*LINK(table));
  9481. }
  9482. bool LeftRightTransformInfo::noteUsed(IHqlExpression * seq)
  9483. {
  9484. if (!seqs.contains(*seq))
  9485. seqs.append(*seq);
  9486. if (seqs.ordinality() > 1)
  9487. {
  9488. shared.setown(new SharedTableInfo(original, 0));
  9489. return true;
  9490. }
  9491. return false;
  9492. }
  9493. void LeftRightTransformInfo::addAmbiguity(SharedTableInfo * table)
  9494. {
  9495. containsAmbiguity = true;
  9496. merge(table);
  9497. }
  9498. void LeftRightTransformInfo::merge(SharedTableInfo * table)
  9499. {
  9500. ForEachItemIn(i, sharedTables)
  9501. {
  9502. SharedTableInfo & cur = sharedTables.item(i);
  9503. if (cur.dataset == table->dataset)
  9504. {
  9505. if (cur.depth < table->depth)
  9506. sharedTables.replace(*LINK(table), i);
  9507. return;
  9508. }
  9509. }
  9510. add(table);
  9511. }
  9512. void LeftRightTransformInfo::inherit(const LeftRightTransformInfo * other)
  9513. {
  9514. ForEachItemIn(i, other->sharedTables)
  9515. merge(&other->sharedTables.item(i));
  9516. }
  9517. static HqlTransformerInfo LeftRightTransformerInfo("LeftRightTransformer");
  9518. LeftRightTransformer::LeftRightTransformer() : NewHqlTransformer(LeftRightTransformerInfo)
  9519. {
  9520. seenShared = true;
  9521. }
  9522. void LeftRightTransformer::analyseExpr(IHqlExpression * _expr)
  9523. {
  9524. IHqlExpression * body = _expr->queryBody();
  9525. if (alreadyVisited(body))
  9526. return;
  9527. NewHqlTransformer::analyseExpr(body);
  9528. if (pass == 0)
  9529. {
  9530. //First pass gathers a list of selectors that are potentially ambiguous if the sequence was removed
  9531. IHqlExpression * left = NULL;
  9532. IHqlExpression * right = NULL;
  9533. switch (getChildDatasetType(body))
  9534. {
  9535. case childdataset_left:
  9536. case childdataset_datasetleft:
  9537. left = body->queryChild(0);
  9538. break;
  9539. case childdataset_same_left_right:
  9540. case childdataset_nway_left_right:
  9541. case childdataset_top_left_right:
  9542. left = body->queryChild(0);
  9543. right = body->queryChild(0);
  9544. break;
  9545. case childdataset_leftright:
  9546. left = body->queryChild(0);
  9547. right = body->queryChild(1);
  9548. break;
  9549. }
  9550. if (left)
  9551. {
  9552. LeftRightTransformInfo * extra = queryExtra(body);
  9553. IHqlExpression * selSeq = querySelSeq(body);
  9554. OwnedHqlExpr seq = createDummySelectorSequence();
  9555. extra->rawLeft.setown(createSelector(no_left, left, seq));
  9556. incUsage(extra->rawLeft, selSeq);
  9557. if (right)
  9558. {
  9559. //no_left is used deliberately in the following to avoid complications where right matches
  9560. //but left doesn't, causing the depths to be messed up.
  9561. extra->rawRight.setown(createSelector(no_left, right, seq));
  9562. if (extra->rawLeft != extra->rawRight)
  9563. incUsage(extra->rawRight, selSeq);
  9564. else
  9565. extra->rawRight.clear();
  9566. }
  9567. }
  9568. }
  9569. else
  9570. {
  9571. //Second pass - for each expression, gather a list of selectors that would actually be ambiguous.
  9572. LeftRightTransformInfo * extra = queryExtra(body);
  9573. IHqlExpression * rawLeft = extra->rawLeft;
  9574. IHqlExpression * rawRight = extra->rawRight;
  9575. //If LEFT is potentially ambiguous, then add it to the list of selectors used by this expression
  9576. if (rawLeft)
  9577. {
  9578. LeftRightTransformInfo * leftExtra = queryExtra(rawLeft);
  9579. if (leftExtra->shared)
  9580. extra->add(leftExtra->shared);
  9581. }
  9582. //Ditto for right
  9583. if (rawRight)
  9584. {
  9585. LeftRightTransformInfo * rightExtra = queryExtra(rawRight);
  9586. if (rightExtra->shared)
  9587. extra->add(rightExtra->shared);
  9588. }
  9589. switch (body->getOperator())
  9590. {
  9591. case no_activerow:
  9592. case no_filepos:
  9593. case no_file_logicalname:
  9594. case no_offsetof:
  9595. case no_joined:
  9596. case no_colon:
  9597. case no_globalscope:
  9598. case no_attr:
  9599. return;
  9600. case no_select:
  9601. {
  9602. bool isNew;
  9603. IHqlExpression * ds = querySelectorDataset(body, isNew);
  9604. if (isNew)
  9605. {
  9606. LeftRightTransformInfo * dsExtra = queryExtra(ds->queryBody());
  9607. extra->inherit(dsExtra);
  9608. }
  9609. return;
  9610. }
  9611. }
  9612. ForEachChild(i, body)
  9613. {
  9614. IHqlExpression * cur = body->queryChild(i);
  9615. LeftRightTransformInfo * childExtra = queryExtra(cur->queryBody());
  9616. //If this is one of the arguments to an operation which has an active top dataset,
  9617. //check to see if any of the contained expressions reference this item
  9618. if ((i != 0) && rawLeft)
  9619. {
  9620. SharedTableInfo * matchLeft = childExtra->uses(rawLeft);
  9621. SharedTableInfo * matchRight = rawRight ? childExtra->uses(rawRight) : NULL;
  9622. if (matchLeft || matchRight)
  9623. {
  9624. unsigned leftDepth = matchLeft ? matchLeft->depth : 0;
  9625. unsigned rightDepth = matchRight ? matchRight->depth : 0;
  9626. unsigned depth = leftDepth > rightDepth ? leftDepth : rightDepth;
  9627. SharedTableInfo * nested = createAmbiguityInfo(rawLeft, depth+1);
  9628. extra->addAmbiguity(nested);
  9629. if (rawRight)
  9630. {
  9631. SharedTableInfo * nested = createAmbiguityInfo(rawRight, depth+1);
  9632. extra->addAmbiguity(nested);
  9633. }
  9634. }
  9635. }
  9636. extra->inherit(childExtra);
  9637. }
  9638. }
  9639. }
  9640. void LeftRightTransformer::incUsage(IHqlExpression * expr, IHqlExpression * seq)
  9641. {
  9642. //MORE: Needs to keep track of the sequences that were used with it, so know if needs disambiguating.
  9643. LeftRightTransformInfo * extra = queryExtra(expr);
  9644. if (extra->noteUsed(seq))
  9645. seenShared = true;
  9646. }
  9647. SharedTableInfo * LeftRightTransformer::createAmbiguityInfo(IHqlExpression * dataset, unsigned depth)
  9648. {
  9649. ForEachItemIn(i, ambiguousTables)
  9650. {
  9651. SharedTableInfo & cur = ambiguousTables.item(i);
  9652. if ((cur.dataset == dataset) && (depth == cur.depth))
  9653. return &cur;
  9654. }
  9655. ambiguousTables.append(*new SharedTableInfo(dataset, depth));
  9656. return &ambiguousTables.tos();
  9657. }
  9658. ANewTransformInfo * LeftRightTransformer::createTransformInfo(IHqlExpression * expr)
  9659. {
  9660. return CREATE_NEWTRANSFORMINFO(LeftRightTransformInfo, expr);
  9661. }
  9662. IHqlExpression * LeftRightTransformer::createTransformed(IHqlExpression * expr)
  9663. {
  9664. IHqlExpression * body = expr->queryBody();
  9665. if (expr != body)
  9666. {
  9667. OwnedHqlExpr newBody = transform(body);
  9668. return expr->cloneAllAnnotations(newBody);
  9669. }
  9670. OwnedHqlExpr transformed = NewHqlTransformer::createTransformed(expr);
  9671. updateOrphanedSelectors(transformed, expr);
  9672. LeftRightTransformInfo * extra = queryExtra(body);
  9673. SharedTableInfo * matchLeft = extra->rawLeft ? extra->uses(extra->rawLeft) : NULL;
  9674. if (matchLeft)
  9675. {
  9676. childDatasetType dsType = getChildDatasetType(expr);
  9677. IHqlExpression * left = transformed->queryChild(0);
  9678. IHqlExpression * right = hasRight(dsType) ? (hasSameLeftRight(dsType) ? left : transformed->queryChild(1)) : NULL;
  9679. IHqlExpression * oldSelSeq = querySelSeq(expr);
  9680. OwnedHqlExpr newSelSeq = createSelectorSequence(matchLeft->depth);
  9681. OwnedHqlExpr oldLeft = createSelector(no_left, left, oldSelSeq);
  9682. OwnedHqlExpr newLeft = createSelector(no_left, left, newSelSeq);
  9683. //Replace dataset with an aliased variety, and remap all the selectors
  9684. HqlExprArray mapped;
  9685. replaceSelectors(mapped, transformed, 1, oldLeft, newLeft);
  9686. if (right)
  9687. {
  9688. OwnedHqlExpr oldRight = createSelector(no_right, right, oldSelSeq);
  9689. OwnedHqlExpr newRight = createSelector(no_right, right, newSelSeq);
  9690. unsigned firstArg = (hasSameLeftRight(dsType) ? 0 : 1);
  9691. replaceSelectors(mapped, firstArg, oldRight, newRight);
  9692. }
  9693. HqlExprArray args;
  9694. args.append(*LINK(left));
  9695. appendArray(args, mapped);
  9696. args.zap(*oldSelSeq);
  9697. args.append(*LINK(newSelSeq));
  9698. transformed.setown(transformed->clone(args));
  9699. }
  9700. return transformed.getClear();
  9701. }
  9702. void LeftRightTransformer::process(HqlExprArray & exprs)
  9703. {
  9704. analyseArray(exprs, 0);
  9705. if (!seenShared)
  9706. return;
  9707. analyseArray(exprs, 1);
  9708. HqlExprArray transformed;
  9709. transformRoot(exprs, transformed);
  9710. replaceArray(exprs, transformed);
  9711. }
  9712. //---------------------------------------------------------------------------------------------------------------------
  9713. /*
  9714. Common up expressions so that all references to the same expression have identical symbols, annotations.
  9715. Generally it improves the code a lot - especially when macros are used. However there are occasional problems....
  9716. a) ut.CleanCompany(ds.x) and ut.cleanCompany(left.x).
  9717. If a component one of them is commoned up with another occurrence, then you can get different named symbols within the expanded function. When we
  9718. then test to see if something is sorted it can incorrectly mismatch (see busheader.xhql dnb_combined_append)
  9719. */
  9720. static void unwindAnnotations(HqlExprCopyArray & unwound, IHqlExpression * expr)
  9721. {
  9722. if (expr->getAnnotationKind() == annotate_none)
  9723. return;
  9724. unwindAnnotations(unwound, expr->queryBody(true));
  9725. unwound.append(*expr);
  9726. }
  9727. IHqlExpression * AnnotationTransformInfo::cloneAnnotations(IHqlExpression * newBody)
  9728. {
  9729. if (annotations.ordinality() == 0)
  9730. return LINK(newBody);
  9731. return annotations.item(0).cloneAllAnnotations(newBody);
  9732. #if 0
  9733. //This code is not currently used - saved for future reference
  9734. LinkedHqlExpr ret = newBody;
  9735. #if 1
  9736. ForEachItemIn(i, annotations)
  9737. ret.setown(annotations.item(i).cloneAllAnnotations(ret));
  9738. #else
  9739. //Code saved once we start removing duplicate annotations (e.g., locations)
  9740. HqlExprCopyArray toApply;
  9741. ForEachItemIn(i, annotations)
  9742. unwindAnnotations(toApply, &annotations.item(i));
  9743. ForEachItemIn(i2, toApply)
  9744. {
  9745. IHqlExpression & curAnnotate = toApply.item(i2);
  9746. ret.setown(curAnnotate.cloneAnnotation(ret));
  9747. }
  9748. #endif
  9749. return ret.getClear();
  9750. #endif
  9751. }
  9752. void AnnotationTransformInfo::noteAnnotation(IHqlExpression * annotation)
  9753. {
  9754. //MORE: Need more intelligence to see if this is a subset of what we already have..
  9755. annotations.append(*annotation);
  9756. }
  9757. static HqlTransformerInfo annotationNormalizerInfo("AnnotationNormalizerTransformer");
  9758. AnnotationNormalizerTransformer::AnnotationNormalizerTransformer()
  9759. : NewHqlTransformer(annotationNormalizerInfo)
  9760. {
  9761. }
  9762. ANewTransformInfo * AnnotationNormalizerTransformer::createTransformInfo(IHqlExpression * expr)
  9763. {
  9764. return CREATE_NEWTRANSFORMINFO(AnnotationTransformInfo, expr);
  9765. }
  9766. void AnnotationNormalizerTransformer::analyseExpr(IHqlExpression * expr)
  9767. {
  9768. if (alreadyVisited(expr))
  9769. return;
  9770. IHqlExpression * body = expr->queryBody();
  9771. node_operator op = body->getOperator();
  9772. if (expr != body)
  9773. {
  9774. if ((op != no_select) || isNewSelector(body))
  9775. queryLocationIndependentExtra(body)->noteAnnotation(expr);
  9776. //Note: expr already tested if expr == body...
  9777. if (alreadyVisited(body))
  9778. return;
  9779. }
  9780. switch (op)
  9781. {
  9782. case no_attr_expr:
  9783. analyseChildren(body);
  9784. return;
  9785. }
  9786. NewHqlTransformer::analyseExpr(body);
  9787. }
  9788. IHqlExpression * AnnotationNormalizerTransformer::createTransformed(IHqlExpression * expr)
  9789. {
  9790. node_operator op = expr->getOperator();
  9791. IHqlExpression * body = expr->queryBody();
  9792. switch (op)
  9793. {
  9794. case no_list:
  9795. {
  9796. if (body->numChildren() == 0)
  9797. return LINK(body);
  9798. break;
  9799. }
  9800. case no_constant:
  9801. // case no_null:
  9802. {
  9803. //AnnotationTransformInfo * extra = queryLocationIndependentExtra(body);
  9804. //Don't common up the location information for this, otherwise it gets silly! Possibly worth removing altogether if ambiguous?
  9805. //MORE: This should probably depend on whether there is more than one annotation on the constant.
  9806. return LINK(body);
  9807. }
  9808. }
  9809. if (expr != body)
  9810. return transform(body);
  9811. OwnedHqlExpr transformed = NewHqlTransformer::createTransformed(expr);
  9812. return queryLocationIndependentExtra(body)->cloneAnnotations(transformed);
  9813. }
  9814. AnnotationTransformInfo * AnnotationNormalizerTransformer::queryLocationIndependentExtra(IHqlExpression * expr)
  9815. {
  9816. return static_cast<AnnotationTransformInfo *>(queryTransformExtra(queryLocationIndependent(expr)));
  9817. }
  9818. void normalizeAnnotations(HqlCppTranslator & translator, HqlExprArray & exprs)
  9819. {
  9820. //First iterate through the expressions and call queryLocationIndependent() to avoid nested transforms (which are less efficient)
  9821. ForEachItemIn(iInit, exprs)
  9822. queryLocationIndependent(&exprs.item(iInit));
  9823. translator.traceExpressions("before annotation normalize", exprs);
  9824. AnnotationNormalizerTransformer normalizer;
  9825. HqlExprArray transformed;
  9826. normalizer.analyseArray(exprs, 0);
  9827. normalizer.transformRoot(exprs, transformed);
  9828. replaceArray(exprs, transformed);
  9829. }
  9830. //---------------------------------------------------------------------------
  9831. static HqlTransformerInfo containsCompoundTransformerInfo("ContainsCompoundTransformer");
  9832. ContainsCompoundTransformer::ContainsCompoundTransformer()
  9833. : QuickHqlTransformer(containsCompoundTransformerInfo, NULL)
  9834. {
  9835. containsCompound = false;
  9836. }
  9837. //NB: This cannot be short circuited, because it is also gathering information about whether or
  9838. void ContainsCompoundTransformer::doAnalyseBody(IHqlExpression * expr)
  9839. {
  9840. if (containsCompound)
  9841. return;
  9842. switch (expr->getOperator())
  9843. {
  9844. case no_compound:
  9845. if (!expr->isAction())
  9846. {
  9847. containsCompound = true;
  9848. return;
  9849. }
  9850. break;
  9851. case no_colon:
  9852. case no_cluster:
  9853. case no_sequential:
  9854. case no_allnodes:
  9855. case no_thisnode:
  9856. //Need to recursively handle these
  9857. containsCompound = true;
  9858. return;
  9859. case no_record:
  9860. case no_field:
  9861. case no_attr:
  9862. case no_attr_link:
  9863. case no_left:
  9864. case no_right:
  9865. case no_self:
  9866. case no_top:
  9867. case no_workunit_dataset:
  9868. case no_assertwild:
  9869. case no_getresult:
  9870. case no_getgraphresult:
  9871. case no_activerow:
  9872. case no_newkeyindex:
  9873. return;
  9874. case no_list:
  9875. if (expr->isConstant())
  9876. return;
  9877. break;
  9878. case no_select:
  9879. // if (expr->hasAttribute(newAtom))
  9880. analyse(expr->queryChild(0));
  9881. return;
  9882. case no_assign:
  9883. analyse(expr->queryChild(1));
  9884. return;
  9885. }
  9886. QuickHqlTransformer::doAnalyseBody(expr);
  9887. }
  9888. bool containsCompound(const HqlExprArray & exprs)
  9889. {
  9890. ContainsCompoundTransformer spotter;
  9891. spotter.analyseArray(exprs);
  9892. return spotter.containsCompound;
  9893. }
  9894. bool containsCompound(IHqlExpression * expr)
  9895. {
  9896. ContainsCompoundTransformer spotter;
  9897. spotter.analyse(expr);
  9898. return spotter.containsCompound;
  9899. }
  9900. //---------------------------------------------------------------------------------------------------------------------
  9901. static HqlTransformerInfo nestedCompoundTransformerInfo("NestedCompoundTransformer");
  9902. NestedCompoundTransformer::NestedCompoundTransformer(HqlCppTranslator & _translator)
  9903. : HoistingHqlTransformer(nestedCompoundTransformerInfo, CTFnoteifactions), translator(_translator), translatorOptions(_translator.queryOptions())
  9904. {
  9905. }
  9906. //For the moment allow simple external calls in scalar setting
  9907. //to make logging much easier...
  9908. static bool isSimpleSideeffect(IHqlExpression * expr)
  9909. {
  9910. switch (expr->getOperator())
  9911. {
  9912. case no_externalcall:
  9913. case no_attr:
  9914. case no_attr_expr:
  9915. case no_attr_link:
  9916. return true;
  9917. case no_comma:
  9918. case no_compound:
  9919. case no_parallel:
  9920. {
  9921. ForEachChild(i, expr)
  9922. {
  9923. if (!isSimpleSideeffect(expr->queryChild(i)))
  9924. return false;
  9925. }
  9926. return true;
  9927. }
  9928. case no_if:
  9929. {
  9930. ForEachChildFrom(i, expr, 1)
  9931. {
  9932. if (!isSimpleSideeffect(expr->queryChild(i)))
  9933. return false;
  9934. }
  9935. return true;
  9936. }
  9937. }
  9938. return false;
  9939. }
  9940. static bool isCalloutSideeffect(IHqlExpression * expr)
  9941. {
  9942. if (!expr->queryType()->isScalar())
  9943. return false;
  9944. return isSimpleSideeffect(expr->queryChild(0));
  9945. }
  9946. IHqlExpression * NestedCompoundTransformer::createTransformed(IHqlExpression * expr)
  9947. {
  9948. if (expr->isConstant())
  9949. return LINK(expr);
  9950. IHqlExpression * ret = queryTransformAnnotation(expr);
  9951. if (ret)
  9952. return ret;
  9953. node_operator op = expr->getOperator();
  9954. switch (op)
  9955. {
  9956. case no_compound:
  9957. if (isUsedUnconditionally(expr) && !expr->isAction() && !isCalloutSideeffect(expr))
  9958. {
  9959. IHqlExpression * sideEffect = expr->queryChild(0);
  9960. IHqlExpression * value = expr->queryChild(1);
  9961. if (!isIndependentOfScope(sideEffect))
  9962. {
  9963. StringBuffer s;
  9964. if (sideEffect->queryName())
  9965. s.appendf(" '%s'", str(sideEffect->queryName()));
  9966. else if (value->queryName())
  9967. s.appendf(" '%s'", str(value->queryName()));
  9968. else
  9969. s.append(" ").append(getOpString(sideEffect->getOperator()));
  9970. IHqlExpression * location = queryLocation(sideEffect);
  9971. if (!location)
  9972. location = queryLocation(value);
  9973. if (!location)
  9974. location = queryActiveLocation();
  9975. if (!isSimpleSideeffect(sideEffect))
  9976. {
  9977. //MORE: This should possibly be an error, but there are still false positives e.g., OUTPUT(ds1.childds)
  9978. //so needs to stay a warning.
  9979. // translator.ERRORAT1(location, HQLERR_GlobalSideEffectDependent, s.str());
  9980. translator.WARNINGAT1(CategoryUnexpected, location, HQLWRN_GlobalSideEffectDependent, s.str());
  9981. }
  9982. break;
  9983. }
  9984. if (!translatorOptions.workunitTemporaries)
  9985. {
  9986. StringBuffer s;
  9987. if (expr->queryName())
  9988. s.append(expr->queryName()).append(": ");
  9989. getExprECL(sideEffect, s);
  9990. throwError1(HQLERR_LibrariesCannotContainSideEffects, s.str());
  9991. }
  9992. appendToTarget(*transform(sideEffect));
  9993. return transform(value);
  9994. }
  9995. break;
  9996. }
  9997. return HoistingHqlTransformer::createTransformed(expr);
  9998. }
  9999. //---------------------------------------------------------------------------
  10000. class LocationInfo : public CInterface
  10001. {
  10002. public:
  10003. HqlExprArray matches;
  10004. };
  10005. static HqlTransformerInfo duplicateCodeSpotterInfo("DuplicateCodeSpotter");
  10006. class DuplicateCodeSpotter : public QuickHqlTransformer
  10007. {
  10008. public:
  10009. DuplicateCodeSpotter() : QuickHqlTransformer(duplicateCodeSpotterInfo, NULL) {}
  10010. inline bool checkExpr(IHqlExpression * expr)
  10011. {
  10012. if (!expr->isDataset())
  10013. return false;
  10014. switch (expr->getOperator())
  10015. {
  10016. case no_join:
  10017. break;
  10018. default:
  10019. return false;
  10020. }
  10021. return true;
  10022. }
  10023. virtual void doAnalyse(IHqlExpression * expr)
  10024. {
  10025. if (checkExpr(expr))
  10026. {
  10027. IHqlExpression * location = queryLocation(expr);
  10028. if (location)
  10029. {
  10030. OwnedHqlExpr attr = createLocationAttr(location->querySourcePath(), location->getStartLine(), location->getStartColumn(), 0);
  10031. Linked<LocationInfo> info;
  10032. Shared<LocationInfo> * match = map.getValue(attr);
  10033. if (match)
  10034. info.set(*match);
  10035. else
  10036. {
  10037. info.setown(new LocationInfo);
  10038. map.setValue(attr, info);
  10039. }
  10040. IHqlExpression * body = expr->queryBody();
  10041. if (!info->matches.contains(*body))
  10042. {
  10043. ForEachItemIn(i, info->matches)
  10044. {
  10045. debugFindFirstDifference(body, &info->matches.item(i));
  10046. }
  10047. info->matches.append(*LINK(body));
  10048. }
  10049. }
  10050. }
  10051. QuickHqlTransformer::doAnalyse(expr);
  10052. }
  10053. MapOwnedToOwned<IHqlExpression, LocationInfo> map;
  10054. };
  10055. void spotPotentialDuplicateCode(HqlExprArray & exprs)
  10056. {
  10057. DuplicateCodeSpotter spotter;
  10058. spotter.analyseArray(exprs);
  10059. }
  10060. //---------------------------------------------------------------------------
  10061. static bool isUniqueAttributeName(IAtom * name)
  10062. {
  10063. const char * nameText = str(name);
  10064. unsigned len = strlen(nameText);
  10065. if (len > 3)
  10066. {
  10067. if ((nameText[len-2] == '_') && (nameText[len-1] == '_') && isdigit((unsigned char)nameText[len-3]))
  10068. return true;
  10069. }
  10070. return false;
  10071. }
  10072. static bool containsUpperCase(const char * s)
  10073. {
  10074. for (;;)
  10075. {
  10076. unsigned char next = *s++;
  10077. if (!next)
  10078. return false;
  10079. if (isupper(next))
  10080. return true;
  10081. }
  10082. }
  10083. static IIdAtom * simplifySymbolName(IIdAtom * name, bool commonUniqueNameAttributes)
  10084. {
  10085. if (!commonUniqueNameAttributes)
  10086. return NULL;
  10087. //Rename all attributes __x__1234__ to __x__
  10088. const char * nameText = str(lower(name));
  10089. size_t nameLen = strlen(nameText);
  10090. size_t len = nameLen;
  10091. if (len > 3)
  10092. {
  10093. if ((nameText[len-2] == '_') && (nameText[len-1] == '_') && isdigit((unsigned char)nameText[len-3]))
  10094. {
  10095. len -= 3;
  10096. while (len && isdigit((unsigned char)nameText[len-1]))
  10097. len--;
  10098. //Shouldn't be possible...
  10099. if (len == 0)
  10100. len = nameLen;
  10101. }
  10102. }
  10103. if (nameLen != len)
  10104. {
  10105. StringAttr truncName;
  10106. truncName.set(nameText, len);
  10107. return createIdAtom(truncName);
  10108. }
  10109. return NULL;
  10110. }
  10111. static IIdAtom * lowerCaseSymbolName(IIdAtom * name)
  10112. {
  10113. if (containsUpperCase(str(name)))
  10114. return createIdAtom(str(lower(name)));
  10115. return name;
  10116. }
  10117. static bool exprIsSelfConstant(IHqlExpression * expr)
  10118. {
  10119. if (expr->isConstant())
  10120. return true;
  10121. switch (expr->getOperator())
  10122. {
  10123. case no_select:
  10124. {
  10125. IHqlExpression * selector = expr->queryChild(0);
  10126. while (selector->getOperator() == no_select)
  10127. selector = selector->queryChild(0);
  10128. return (selector->getOperator() == no_selfref);
  10129. }
  10130. }
  10131. if (expr->isDataset() && (getNumChildTables(expr) == 0))
  10132. return false;
  10133. ForEachChild(i, expr)
  10134. {
  10135. IHqlExpression * cur = expr->queryChild(i);
  10136. if (!exprIsSelfConstant(cur))
  10137. return false;
  10138. }
  10139. return true;
  10140. }
  10141. static IAtom * queryPatUseModule(IHqlExpression * expr)
  10142. {
  10143. IHqlExpression * moduleAttr = expr->queryAttribute(moduleAtom);
  10144. if (moduleAttr)
  10145. return moduleAttr->queryChild(0)->queryBody()->queryName();
  10146. return NULL;
  10147. }
  10148. static IAtom * queryPatUseName(IHqlExpression * expr)
  10149. {
  10150. IHqlExpression * nameAttr = expr->queryAttribute(nameAtom);
  10151. return nameAttr->queryChild(0)->queryBody()->queryName();
  10152. }
  10153. void HqlTreeNormalizerInfo::noteSymbol(IHqlExpression * _symbol)
  10154. {
  10155. if (!symbol || isUniqueAttributeName(symbol->queryName()))
  10156. symbol = _symbol;
  10157. }
  10158. static HqlTransformerInfo hqlTreeNormalizerInfo("HqlTreeNormalizer");
  10159. HqlTreeNormalizer::HqlTreeNormalizer(HqlCppTranslator & _translator) : NewHqlTransformer(hqlTreeNormalizerInfo), translator(_translator)
  10160. {
  10161. seenForceLocal = false;
  10162. seenLocalUpload = false;
  10163. const HqlCppOptions & translatorOptions = translator.queryOptions();
  10164. options.assertSortedDistributed = translatorOptions.assertSortedDistributed;
  10165. options.removeAsserts = !translatorOptions.checkAsserts;
  10166. options.commonUniqueNameAttributes = translatorOptions.commonUniqueNameAttributes;
  10167. options.sortIndexPayload = translatorOptions.sortIndexPayload;
  10168. options.allowSections = translatorOptions.allowSections;
  10169. options.normalizeExplicitCasts = translatorOptions.normalizeExplicitCasts;
  10170. options.ensureRecordsHaveSymbols = translatorOptions.ensureRecordsHaveSymbols;
  10171. options.outputRowsAsDatasets = translator.targetRoxie();
  10172. options.constantFoldNormalize = translatorOptions.constantFoldNormalize;
  10173. options.allowActivityForKeyedJoin = translatorOptions.allowActivityForKeyedJoin;
  10174. options.implicitSubSort = translatorOptions.implicitBuildIndexSubSort;
  10175. options.forceAllDatasetsParallel = translatorOptions.forceAllDatasetsParallel;
  10176. errorProcessor = &translator.queryErrorProcessor();
  10177. nextSequenceValue = 1;
  10178. }
  10179. ANewTransformInfo * HqlTreeNormalizer::createTransformInfo(IHqlExpression * expr)
  10180. {
  10181. return CREATE_NEWTRANSFORMINFO(HqlTreeNormalizerInfo, expr);
  10182. }
  10183. HqlTreeNormalizerInfo * HqlTreeNormalizer::queryLocationIndependentExtra(IHqlExpression * expr)
  10184. {
  10185. return static_cast<HqlTreeNormalizerInfo *>(queryTransformExtra(queryLocationIndependent(expr)));
  10186. }
  10187. void HqlTreeNormalizer::convertRecordToAssigns(HqlExprArray & assigns, IHqlExpression * oldRecord, IHqlExpression * targetSelector, bool canOmit, bool convertTempTable)
  10188. {
  10189. ForEachChild(idx, oldRecord)
  10190. {
  10191. IHqlExpression * oldField = oldRecord->queryChild(idx);
  10192. OwnedHqlExpr newField = transform(oldField);
  10193. switch (oldField->getOperator())
  10194. {
  10195. case no_ifblock:
  10196. convertRecordToAssigns(assigns, oldField->queryChild(1), targetSelector, canOmit, convertTempTable);
  10197. break;
  10198. case no_record:
  10199. convertRecordToAssigns(assigns, oldField, targetSelector, canOmit, convertTempTable);
  10200. break;
  10201. case no_field:
  10202. {
  10203. IHqlExpression * oldFieldRecord = oldField->queryRecord();
  10204. IHqlExpression * value = queryRealChild(oldField, 0);
  10205. OwnedHqlExpr newTargetSelector = createSelectExpr(LINK(targetSelector), LINK(newField));
  10206. if (oldFieldRecord && !oldField->isDataset() && !value)
  10207. {
  10208. if (convertTempTable)
  10209. convertRecordToAssigns(assigns, oldFieldRecord, newTargetSelector, canOmit, convertTempTable);
  10210. else
  10211. {
  10212. IHqlExpression * newRecord = newField->queryRecord();
  10213. OwnedHqlExpr newSelf = getSelf(newRecord);
  10214. HqlExprArray newAssigns;
  10215. convertRecordToAssigns(newAssigns, oldFieldRecord, newSelf, canOmit, convertTempTable);
  10216. IHqlExpression * transform = createValue(no_newtransform, makeTransformType(newRecord->getType()), newAssigns);
  10217. IHqlExpression * newValue = createRow(no_createrow, transform);
  10218. assigns.append(*createAssign(LINK(newTargetSelector), newValue));
  10219. }
  10220. }
  10221. else
  10222. {
  10223. assertex(value || canOmit);
  10224. if (value && (!convertTempTable || exprIsSelfConstant(value)))
  10225. assigns.append(*createAssign(LINK(newTargetSelector), transform(value)));
  10226. if (oldFieldRecord && convertTempTable)
  10227. assigns.append(*createExprAttribute(defaultAtom, createExprAttribute(defaultAtom, LINK(newTargetSelector)), convertRecordToAssigns(oldFieldRecord, canOmit, convertTempTable)));
  10228. }
  10229. break;
  10230. }
  10231. case no_attr:
  10232. case no_attr_link:
  10233. case no_attr_expr:
  10234. break;
  10235. default:
  10236. UNIMPLEMENTED;
  10237. }
  10238. }
  10239. }
  10240. IHqlExpression * HqlTreeNormalizer::convertRecordToAssigns(IHqlExpression * oldRecord, bool canOmit, bool convertTempTable)
  10241. {
  10242. OwnedHqlExpr newRecord = transform(oldRecord);
  10243. HqlExprArray assigns;
  10244. OwnedHqlExpr self = getSelf(newRecord);
  10245. convertRecordToAssigns(assigns, oldRecord, self, canOmit, convertTempTable);
  10246. return createValue(no_transform, makeTransformType(newRecord->getType()), assigns);
  10247. }
  10248. // Problems occur if a record used in a select fields is also used for some other linked purpose.
  10249. // Thankfully the only one so far is BUILDINDEX(index)
  10250. IHqlExpression * HqlTreeNormalizer::convertSelectToProject(IHqlExpression * newRecord, IHqlExpression * expr)
  10251. {
  10252. OwnedHqlExpr newDataset = transform(expr->queryChild(0));
  10253. IHqlExpression * oldRecord = expr->queryChild(1);
  10254. if (oldRecord->getOperator() == no_null)
  10255. return newDataset.getClear();
  10256. HqlExprArray assigns;
  10257. OwnedHqlExpr self = getSelf(newRecord);
  10258. convertRecordToAssigns(assigns, oldRecord, self, false, false);
  10259. OwnedHqlExpr newTransform = createValue(no_newtransform, makeTransformType(LINK(newRecord->queryRecordType())), assigns);
  10260. newTransform.setown(newRecord->cloneAllAnnotations(newTransform));
  10261. HqlExprArray args;
  10262. args.append(*newDataset.getClear());
  10263. args.append(*LINK(newRecord));
  10264. args.append(*newTransform.getClear());
  10265. unsigned numChildren = expr->numChildren();
  10266. for (unsigned idx = 2; idx < numChildren; idx++)
  10267. args.append(*transform(expr->queryChild(idx)));
  10268. node_operator op = isAggregateDataset(expr) ? no_newaggregate : no_newusertable;
  10269. OwnedHqlExpr project = createDataset(op, args);
  10270. return expr->cloneAllAnnotations(project);
  10271. }
  10272. IHqlExpression * HqlTreeNormalizer::removeDefaultsFromExpr(IHqlExpression * expr, unsigned recordChildIndex, node_operator newOp)
  10273. {
  10274. IHqlExpression * oldRecord = expr->queryChild(recordChildIndex);
  10275. OwnedHqlExpr newRecord = transform(oldRecord);
  10276. HqlExprArray assigns;
  10277. OwnedHqlExpr self = getSelf(newRecord);
  10278. convertRecordToAssigns(assigns, oldRecord, self, false, false);
  10279. IHqlExpression * newTransform = createValue(no_newtransform, makeTransformType(LINK(newRecord->queryRecordType())), assigns);
  10280. HqlExprArray args;
  10281. args.append(*transform(expr->queryChild(0)));
  10282. for (unsigned i= 1; i < recordChildIndex; i++)
  10283. args.append(*transform(expr->queryChild(i)));
  10284. args.append(*LINK(newRecord));
  10285. args.append(*newTransform);
  10286. unsigned numChildren = expr->numChildren();
  10287. for (unsigned idx = recordChildIndex+1; idx < numChildren; idx++)
  10288. args.append(*transform(expr->queryChild(idx)));
  10289. OwnedHqlExpr project;
  10290. if (expr->isDataset())
  10291. project.setown(createDataset(newOp, args));
  10292. else if (expr->isDatarow())
  10293. project.setown(createRow(newOp, args));
  10294. else
  10295. project.setown(createValue(newOp, makeVoidType(), args));
  10296. return expr->cloneAllAnnotations(project);
  10297. }
  10298. ITypeInfo * HqlTreeNormalizer::transformType(ITypeInfo * type)
  10299. {
  10300. switch (type->queryModifier())
  10301. {
  10302. case typemod_original:
  10303. {
  10304. switch (type->getTypeCode())
  10305. {
  10306. case type_record:
  10307. case type_transform:
  10308. case type_row:
  10309. case type_table:
  10310. case type_groupedtable:
  10311. //Strip all original annotations - they cause branches to not be commoned up
  10312. return transformType(type->queryTypeBase());
  10313. }
  10314. //But keep annotations used for typedef information, they should probably work differently
  10315. break;
  10316. }
  10317. case typemod_none:
  10318. {
  10319. //Ensure all records with the same format get the same original modifier
  10320. Owned<ITypeInfo> transformedType = NewHqlTransformer::transformType(type);
  10321. if (type->getTypeCode() == type_record)
  10322. {
  10323. IHqlExpression * record = queryExpression(type);
  10324. if (record && record->getOperator() == no_record)
  10325. {
  10326. OwnedHqlExpr transformedRecord = transform(record);
  10327. if (transformedRecord->queryBody() != transformedRecord)
  10328. return makeOriginalModifier(LINK(transformedType), LINK(transformedRecord));
  10329. }
  10330. }
  10331. return transformedType.getClear();
  10332. }
  10333. break;
  10334. }
  10335. return NewHqlTransformer::transformType(type);
  10336. }
  10337. bool isVoidOrDatasetOrList(IHqlExpression * expr)
  10338. {
  10339. ITypeInfo * type = expr->queryType();
  10340. switch (type->getTypeCode())
  10341. {
  10342. case type_void:
  10343. case type_table:
  10344. case type_row:
  10345. case type_groupedtable:
  10346. case type_set:
  10347. return true;
  10348. default:
  10349. return false;
  10350. }
  10351. }
  10352. inline IHqlExpression * createColon(IHqlExpression * l, HqlExprArray & actions)
  10353. {
  10354. HqlExprArray args;
  10355. args.append(*l);
  10356. ForEachItemIn(i, actions)
  10357. args.append(OLINK(actions.item(i)));
  10358. return createWrapper(no_colon, args);
  10359. }
  10360. void HqlTreeNormalizer::analyseExpr(IHqlExpression * expr)
  10361. {
  10362. IHqlExpression * body = expr->queryBody();
  10363. node_operator op = body->getOperator();
  10364. if ((op == no_record) && (expr != body))
  10365. {
  10366. IHqlExpression * symbol = queryNamedSymbol(expr);
  10367. if (symbol)
  10368. queryLocationIndependentExtra(body)->noteSymbol(expr);
  10369. }
  10370. if (alreadyVisited(body))
  10371. return;
  10372. //Record a list of all USE(name[,name]) so we know what needs fixing up, and all patterns with explicit defines already
  10373. switch (op)
  10374. {
  10375. case no_pat_use:
  10376. if (body->hasAttribute(nameAtom))
  10377. forwardReferences.append(*LINK(body));
  10378. break;
  10379. case no_pat_instance:
  10380. if (body->queryChild(0)->getOperator() == no_define)
  10381. defines.append(*LINK(body));
  10382. break;
  10383. case no_libraryscopeinstance:
  10384. analyseExpr(body->queryDefinition());
  10385. break;
  10386. case no_transform:
  10387. case no_call:
  10388. case no_externalcall:
  10389. {
  10390. IHqlExpression * record = queryOriginalRecord(body->queryType());
  10391. if (record)
  10392. analyseExpr(record);
  10393. break;
  10394. }
  10395. case no_attr_expr:
  10396. case no_record:
  10397. case no_ifblock:
  10398. case no_select:
  10399. analyseChildren(body);
  10400. return;
  10401. case no_field:
  10402. {
  10403. IHqlExpression * record = queryOriginalRecord(body->queryType());
  10404. if (record)
  10405. analyseExpr(record);
  10406. analyseChildren(body);
  10407. break;
  10408. }
  10409. case no_if:
  10410. if (expr->isDataset())
  10411. {
  10412. IHqlExpression * left = expr->queryChild(1);
  10413. IHqlExpression * right = expr->queryChild(2);
  10414. if (right && (isGrouped(left) != isGrouped(right)))
  10415. translator.reportError(expr, ERR_GROUPING_MISMATCH, "Branches of the condition have different grouping");
  10416. break;
  10417. }
  10418. }
  10419. Parent::analyseExpr(body);
  10420. }
  10421. IHqlExpression * HqlTreeNormalizer::makeRecursiveName(IAtom * searchModule, IAtom * searchName)
  10422. {
  10423. //If this symbol is already has a user define, use that instead of creating our own,
  10424. //because I don't cope very well with multiple defines on the same pattern instance.
  10425. ForEachItemIn(i, defines)
  10426. {
  10427. IHqlExpression & cur = defines.item(i);
  10428. IHqlExpression * moduleExpr = cur.queryChild(2);
  10429. IAtom * module = moduleExpr ? moduleExpr->queryBody()->queryName() : NULL;
  10430. IAtom * name = cur.queryChild(1)->queryBody()->queryName();
  10431. if (name == searchName && module == searchModule)
  10432. return LINK(cur.queryChild(0)->queryChild(1));
  10433. }
  10434. StringBuffer s;
  10435. s.append("$").append(searchModule).append(".").append(searchName);
  10436. return createConstant(s.str());
  10437. }
  10438. IHqlExpression * HqlTreeNormalizer::queryTransformPatternDefine(IHqlExpression * expr)
  10439. {
  10440. if (expr->queryChild(0)->getOperator() == no_define)
  10441. return NULL;
  10442. IHqlExpression * moduleExpr = expr->queryChild(2);
  10443. IAtom * module = moduleExpr ? moduleExpr->queryBody()->queryName() : NULL;
  10444. IAtom * name = expr->queryChild(1)->queryBody()->queryName();
  10445. ForEachItemIn(i, forwardReferences)
  10446. {
  10447. IHqlExpression * cur = &forwardReferences.item(i);
  10448. if ((name == queryPatUseName(cur)) && (module == queryPatUseModule(cur)))
  10449. {
  10450. IHqlExpression * base = transform(expr->queryChild(0));
  10451. HqlExprArray args;
  10452. args.append(*createValue(no_define, base->getType(), base, makeRecursiveName(module, name)));
  10453. unwindChildren(args, expr, 1);
  10454. return createValue(expr->getOperator(), transformType(expr->queryType()), args);
  10455. }
  10456. }
  10457. return NULL;
  10458. }
  10459. IHqlExpression * HqlTreeNormalizer::transformActionList(IHqlExpression * expr)
  10460. {
  10461. HqlExprArray args;
  10462. ForEachChild(i, expr)
  10463. {
  10464. IHqlExpression * cur = expr->queryChild(i);
  10465. if (cur->getOperator() != no_setmeta)
  10466. {
  10467. OwnedHqlExpr transformed = transform(cur);
  10468. if ((transformed->getOperator() != no_null) || !transformed->isAction())
  10469. args.append(*transformed.getClear());
  10470. }
  10471. }
  10472. return expr->clone(args);
  10473. }
  10474. IHqlExpression * HqlTreeNormalizer::transformCaseToIfs(IHqlExpression * expr)
  10475. {
  10476. unsigned max = numRealChildren(expr);
  10477. OwnedHqlExpr testVar = transform(expr->queryChild(0));
  10478. OwnedHqlExpr elseExpr = transform(expr->queryChild(max-1));
  10479. for (unsigned idx = max-2; idx != 0; idx--)
  10480. {
  10481. IHqlExpression * cur = expr->queryChild(idx);
  10482. IHqlExpression * curValue = cur->queryChild(0);
  10483. Owned<ITypeInfo> type = ::getPromotedECLType(testVar->queryType(), curValue->queryType());
  10484. OwnedHqlExpr castCurValue = ensureExprType(curValue, type);
  10485. OwnedHqlExpr test = createBoolExpr(no_eq, ensureExprType(testVar, type), transform(castCurValue));
  10486. if (options.constantFoldNormalize)
  10487. test.setown(foldConstantOperator(test, 0, NULL));
  10488. OwnedHqlExpr trueExpr = transform(cur->queryChild(1));
  10489. elseExpr.setown(createIf(test.getClear(), trueExpr.getClear(), elseExpr.getClear()));
  10490. if (options.constantFoldNormalize)
  10491. elseExpr.setown(foldConstantOperator(elseExpr, 0, NULL));
  10492. }
  10493. return elseExpr.getClear();
  10494. }
  10495. IHqlExpression * HqlTreeNormalizer::transformCaseToChoose(IHqlExpression * expr)
  10496. {
  10497. //For the moment only convert datasets to choose format. (Partly to test implementation.)
  10498. //Datarows are unlikely to benefit, and will cause additional work.
  10499. //Converting actions has implications for needing new activity kinds, and support in thor.
  10500. if (!expr->isDataset())
  10501. return transformCaseToIfs(expr);
  10502. unsigned max = numRealChildren(expr);
  10503. HqlExprArray branches;
  10504. OwnedHqlExpr testVar = transform(expr->queryChild(0));
  10505. HqlExprArray caseArgs;
  10506. caseArgs.append(*LINK(testVar));
  10507. bool isNullMapping = true;
  10508. for (unsigned i1=1; i1 < max-1; i1++)
  10509. {
  10510. IHqlExpression * mapto = expr->queryChild(i1);
  10511. OwnedHqlExpr key = transform(mapto->queryChild(0));
  10512. OwnedHqlExpr branch = transform(mapto->queryChild(1));
  10513. unsigned matchIndex = branches.find(*branch);
  10514. if (matchIndex == NotFound)
  10515. {
  10516. matchIndex = branches.ordinality();
  10517. branches.append(*branch.getClear());
  10518. }
  10519. OwnedHqlExpr value = getSizetConstant(matchIndex+1);
  10520. //MORE: Could calculate a delta and add/subtract it from testVar
  10521. if (!key->queryValue() || !matchesConstantValue(key, matchIndex+1))
  10522. isNullMapping = false;
  10523. caseArgs.append(*createValue(no_mapto, value->getType(), key.getClear(), LINK(value)));
  10524. }
  10525. caseArgs.append(*getSizetConstant(max-1));
  10526. HqlExprArray args;
  10527. if (isNullMapping)
  10528. args.append(*LINK(testVar));
  10529. else
  10530. args.append(*createValue(no_case, LINK(sizetType), caseArgs));
  10531. appendArray(args, branches);
  10532. args.append(*transform(expr->queryChild(max-1)));
  10533. if (expr->isDataset())
  10534. return createDataset(no_chooseds, args);
  10535. return createAction(no_choose, args);
  10536. }
  10537. IHqlExpression * HqlTreeNormalizer::transformEvaluate(IHqlExpression * expr)
  10538. {
  10539. //Evaluate causes chaos - so translate it to a different form.
  10540. //following cases supported so far:
  10541. //EVALUATE(LEFT/RIGHT, g()) -> g(LEFT)
  10542. //EVAlUATE(x, field) -> x.field;
  10543. //EVALUATE(t[n], e) -> table(t,{f1 := e})[n].f1;
  10544. IHqlExpression * ds = expr->queryChild(0);
  10545. IHqlExpression * attr = expr->queryChild(1);
  10546. OwnedHqlExpr transformed;
  10547. OwnedHqlExpr activeTable = getActiveTableSelector();
  10548. if ((attr->getOperator() == no_select) && (attr->queryChild(0) == activeTable))
  10549. {
  10550. //EVAlUATE(x, field) -> x.field;
  10551. transformed.setown(createSelectExpr(LINK(ds), LINK(attr->queryChild(1))));
  10552. }
  10553. else if (attr->isConstant())
  10554. transformed.set(attr);
  10555. else
  10556. {
  10557. switch (ds->getOperator())
  10558. {
  10559. case no_left:
  10560. case no_right:
  10561. //EVALUATE(LEFT/RIGHT, g()) -> g(LEFT)
  10562. //May change too many datasets?
  10563. transformed.setown(replaceSelector(attr, activeTable, ds));
  10564. break;
  10565. case no_select:
  10566. //EVALUATE(x.y, g()) -> EVALUATE(x, g(y))
  10567. //May change too many datasets?
  10568. transformed.setown(createValue(no_evaluate, attr->getType(), LINK(ds->queryChild(0)), replaceSelector(attr, activeTable, ds->queryChild(1))));
  10569. break;
  10570. case no_selectnth:
  10571. {
  10572. IHqlExpression * baseDs = ds->queryChild(0);
  10573. if ((attr->getOperator() == no_select) && (attr->queryChild(0)->queryNormalizedSelector() == baseDs->queryNormalizedSelector()))
  10574. {
  10575. //Special case a select same as field...
  10576. transformed.setown(createSelectExpr(LINK(ds), LINK(attr->queryChild(1))));
  10577. }
  10578. else
  10579. {
  10580. //EVALUATE(t[n], e) -> table(t,{f1 := e})[n].f1;
  10581. OwnedHqlExpr field = createFieldFromValue(valueId, expr);
  10582. IHqlExpression * aggregateRecord = createRecord(field);
  10583. IHqlExpression * newAttr = replaceSelector(attr, activeTable, baseDs);
  10584. IHqlExpression * assign = createAssign(createSelectExpr(getSelf(aggregateRecord), LINK(field)), newAttr);
  10585. IHqlExpression * transform = createValue(no_newtransform, makeTransformType(aggregateRecord->getType()), assign);
  10586. IHqlExpression * project = createDataset(no_newusertable, LINK(baseDs), createComma(aggregateRecord, transform));
  10587. project = createRow(no_selectnth, project, LINK(ds->queryChild(1)));
  10588. transformed.setown(createSelectExpr(project, LINK(field)));
  10589. }
  10590. break;
  10591. }
  10592. default:
  10593. UNIMPLEMENTED;
  10594. }
  10595. }
  10596. return transform(transformed);
  10597. }
  10598. IHqlExpression * HqlTreeNormalizer::transformMap(IHqlExpression * expr)
  10599. {
  10600. unsigned max = numRealChildren(expr);
  10601. OwnedHqlExpr elseExpr = transform(expr->queryChild(max-1));
  10602. for (unsigned idx = max-1; idx-- != 0; )
  10603. {
  10604. IHqlExpression * cur = expr->queryChild(idx);
  10605. elseExpr.setown(createIf(transform(cur->queryChild(0)), transform(cur->queryChild(1)), elseExpr.getClear()));
  10606. if (options.constantFoldNormalize)
  10607. elseExpr.setown(foldConstantOperator(elseExpr, 0, NULL));
  10608. }
  10609. return elseExpr.getClear();
  10610. }
  10611. IHqlExpression * HqlTreeNormalizer::transformTempRow(IHqlExpression * expr)
  10612. {
  10613. ECLlocation dummyLocation(0, 0, 0, NULL);
  10614. OwnedHqlExpr createRow = convertTempRowToCreateRow(translator.queryErrorProcessor(), dummyLocation, expr);
  10615. return transform(createRow);
  10616. }
  10617. IHqlExpression * HqlTreeNormalizer::transformTempTable(IHqlExpression * expr)
  10618. {
  10619. ECLlocation dummyLocation(0, 0, 0, NULL);
  10620. OwnedHqlExpr inlineTable = convertTempTableToInlineTable(translator.queryErrorProcessor(), dummyLocation, expr);
  10621. if (expr != inlineTable)
  10622. return transform(inlineTable);
  10623. IHqlExpression * oldValues = expr->queryChild(0);
  10624. IHqlExpression * oldRecord = expr->queryChild(1);
  10625. OwnedHqlExpr values = normalizeListCasts(oldValues);
  10626. OwnedHqlExpr newRecord = transform(oldRecord);
  10627. node_operator valueOp = values->getOperator();
  10628. if ((valueOp != no_recordlist) && (valueOp != no_list))
  10629. {
  10630. if (queryRealChild(expr, 2))
  10631. return Parent::createTransformed(expr);
  10632. HqlExprArray children;
  10633. children.append(*transform(oldValues));
  10634. children.append(*LINK(newRecord));
  10635. children.append(*convertRecordToAssigns(oldRecord, true, true));
  10636. return expr->clone(children);
  10637. }
  10638. //should have already been handled by convertTempTableToInlineTable();
  10639. throwUnexpected();
  10640. }
  10641. IHqlExpression * HqlTreeNormalizer::transformNewKeyIndex(IHqlExpression * expr)
  10642. {
  10643. IHqlExpression * ds = expr->queryChild(0);
  10644. //If dataset is already null, then do standard
  10645. if (ds->getOperator() == no_null)
  10646. return completeTransform(expr);
  10647. //Before we do anything replace the dataset with a null dataset. This ensures we do the minimum transformation on the rest of the tree
  10648. OwnedHqlExpr newDs = createDataset(no_null, LINK(ds->queryRecord()));
  10649. HqlExprArray args;
  10650. args.append(*ds->cloneAllAnnotations(newDs));
  10651. args.append(*LINK(expr->queryChild(1)));
  10652. args.append(*quickFullReplaceExpression(expr->queryChild(2), ds->queryNormalizedSelector(), newDs));
  10653. unwindChildren(args, expr, 3);
  10654. OwnedHqlExpr ret = expr->clone(args);
  10655. return transform(ret);
  10656. }
  10657. IHqlExpression * HqlTreeNormalizer::transformKeyIndex(IHqlExpression * expr)
  10658. {
  10659. //Before we do anything replace the dataset with a null dataset. This ensures we do the minimum transformation on the rest of the tree
  10660. IHqlExpression * ds = expr->queryChild(0);
  10661. OwnedHqlExpr newDs = createDataset(no_null, LINK(ds->queryRecord()));
  10662. HqlExprArray args;
  10663. args.append(*ds->cloneAllAnnotations(newDs));
  10664. args.append(*quickFullReplaceExpression(expr->queryChild(1), ds->queryNormalizedSelector(), newDs));
  10665. unwindChildren(args, expr, 2);
  10666. OwnedHqlExpr normalized = expr->clone(args);
  10667. //Now convert from the no_keyindex format to the no_newkeyindex format.
  10668. //force the 1st argument to be processed..
  10669. OwnedHqlExpr transformed = completeTransform(normalized);
  10670. HqlExprArray assigns;
  10671. OwnedHqlExpr self = getSelf(transformed);
  10672. convertRecordToAssigns(assigns, normalized->queryChild(1), self, true, false); // fpos may not have a value...
  10673. args.kill();
  10674. unwindChildren(args, transformed);
  10675. args.add(*createValue(no_newtransform, makeTransformType(transformed->queryChild(1)->getType()), assigns), 2);
  10676. OwnedHqlExpr ret = createDataset(no_newkeyindex, args);
  10677. //MORE: This would be the place to add a FILTERED() attribute derived from any filters applied to the dataset
  10678. return expr->cloneAllAnnotations(ret);
  10679. }
  10680. IHqlExpression * HqlTreeNormalizer::transformMerge(IHqlExpression * expr)
  10681. {
  10682. HqlExprArray children;
  10683. transformChildren(expr, children);
  10684. HqlExprArray args;
  10685. reorderAttributesToEnd(args, children);
  10686. return expr->clone(args);
  10687. }
  10688. IHqlExpression * HqlTreeNormalizer::transformPatNamedUse(IHqlExpression * expr)
  10689. {
  10690. OwnedHqlExpr define = makeRecursiveName(queryPatUseModule(expr), queryPatUseName(expr));
  10691. HqlExprArray args;
  10692. ForEachChild(i, expr)
  10693. {
  10694. IHqlExpression * cur = queryRealChild(expr, i);
  10695. if (cur)
  10696. args.append(*LINK(cur));
  10697. }
  10698. args.append(*define.getClear());
  10699. ITypeInfo * type = expr->queryType();
  10700. return createValue(no_pat_use, transformType(type), args);
  10701. }
  10702. IHqlExpression * HqlTreeNormalizer::transformPatCheckIn(IHqlExpression * expr)
  10703. {
  10704. OwnedHqlExpr set = transform(expr->queryChild(1));
  10705. //because this is a check pattern, we are free to remove any instance tags - they can't be used for matching.
  10706. while (set->getOperator() == no_pat_instance)
  10707. set.set(set->queryChild(0));
  10708. if (set->getOperator() == no_pat_set)
  10709. {
  10710. IHqlExpression * notAttr = expr->queryAttribute(notAtom);
  10711. if (!notAttr)
  10712. return LINK(set);
  10713. HqlExprArray args;
  10714. unwindChildren(args, set);
  10715. if (args.find(*notAttr) == NotFound)
  10716. args.append(*LINK(notAttr));
  10717. else
  10718. args.zap(*notAttr);
  10719. return set->clone(args);
  10720. }
  10721. HqlExprArray values, newValues;
  10722. set->unwindList(values, no_pat_or);
  10723. ForEachItemIn(idx, values)
  10724. {
  10725. IHqlExpression * cur = &values.item(idx);
  10726. while (cur->getOperator() == no_pat_instance)
  10727. cur = cur->queryChild(0);
  10728. if (cur->getOperator() != no_pat_const)
  10729. return NULL;
  10730. IValue * value = cur->queryChild(0)->queryValue();
  10731. if (!value)
  10732. return NULL;
  10733. ITypeInfo * type = value->queryType();
  10734. if (type->getStringLen() != 1)
  10735. return NULL;
  10736. switch (type->getTypeCode())
  10737. {
  10738. case type_string:
  10739. newValues.append(*createConstant((int)*(const byte *)value->queryValue()));
  10740. break;
  10741. case type_unicode:
  10742. newValues.append(*createConstant((int)*(const UChar *)value->queryValue()));
  10743. break;
  10744. case type_utf8:
  10745. newValues.append(*createConstant((int)rtlUtf8Char((const char *)value->queryValue())));
  10746. break;
  10747. default:
  10748. return NULL;
  10749. }
  10750. }
  10751. if (expr->hasAttribute(notAtom))
  10752. newValues.append(*createAttribute(notAtom));
  10753. return createValue(no_pat_set, makePatternType(), newValues);
  10754. }
  10755. IHqlExpression * HqlTreeNormalizer::transformTable(IHqlExpression * untransformed)
  10756. {
  10757. //Convert DATASET('xx', rec, PIPE('z'))
  10758. //DATASET('xx', rec, THOR) | PIPE('z')
  10759. OwnedHqlExpr transformed = completeTransform(untransformed);
  10760. IHqlExpression * mode = transformed->queryChild(2);
  10761. if (mode->getOperator() != no_pipe)
  10762. return transformed.getClear();
  10763. IHqlExpression * filename = transformed->queryChild(0);
  10764. StringBuffer s;
  10765. if (getStringValue(s, filename, NULL).length() == 0)
  10766. return transformed.getClear();
  10767. OwnedHqlExpr modeThor = createValue(no_thor);
  10768. IHqlExpression * diskRead = replaceChild(transformed, 2, modeThor);
  10769. HqlExprArray args;
  10770. args.append(*diskRead);
  10771. unwindChildren(args, mode);
  10772. return createDataset(no_pipe, args);
  10773. }
  10774. IHqlExpression * HqlTreeNormalizer::optimizeAssignSkip(HqlExprArray & children, IHqlExpression * expr, IHqlExpression * cond, unsigned depth)
  10775. {
  10776. if (!containsSkip(expr))
  10777. return LINK(expr);
  10778. switch (expr->getOperator())
  10779. {
  10780. case no_skip:
  10781. children.append(*createValue(no_skip, makeVoidType(), LINK(cond)));
  10782. return NULL;
  10783. case no_cast:
  10784. case no_implicitcast:
  10785. {
  10786. bool same= true;
  10787. HqlExprArray args;
  10788. ForEachChild(i, expr)
  10789. {
  10790. IHqlExpression * cur = expr->queryChild(i);
  10791. IHqlExpression * ret = optimizeAssignSkip(children, cur, cond, depth);
  10792. if (!ret)
  10793. return NULL;
  10794. args.append(*ret);
  10795. if (cur != ret)
  10796. same = false;
  10797. }
  10798. if (same)
  10799. return LINK(expr);
  10800. return expr->clone(args);
  10801. }
  10802. //could try and handle map/case/choose, but less common, and more complicated.
  10803. case no_if:
  10804. {
  10805. //For the moment only hoist SKIPS within a single level of IF() conditions.
  10806. //Multi level rarely occur, and don't significantly improve the code
  10807. if (depth != 0)
  10808. return LINK(expr);
  10809. IHqlExpression * thisCond = expr->queryChild(0);
  10810. IHqlExpression * left = expr->queryChild(1);
  10811. IHqlExpression * right = expr->queryChild(2);
  10812. if (!right)
  10813. return LINK(expr);
  10814. OwnedHqlExpr leftCond = extendConditionOwn(no_and, LINK(cond), LINK(thisCond));
  10815. OwnedHqlExpr inverseCond = createValue(no_not, makeBoolType(), LINK(thisCond));
  10816. OwnedHqlExpr rightCond = extendConditionOwn(no_and, LINK(cond), LINK(inverseCond));
  10817. OwnedHqlExpr newLeft = optimizeAssignSkip(children, left, leftCond, depth+1);
  10818. OwnedHqlExpr newRight = optimizeAssignSkip(children, right, rightCond, depth+1);
  10819. if (!newLeft && !newRight)
  10820. return NULL;
  10821. //if cond is true, then it will skip => no need to check the condition
  10822. if (!newLeft)
  10823. return LINK(newRight);
  10824. if (!newRight)
  10825. return LINK(newLeft);
  10826. if (left == newLeft && right == newRight)
  10827. return LINK(expr);
  10828. HqlExprArray args;
  10829. unwindChildren(args, expr);
  10830. args.replace(*newLeft.getClear(), 1);
  10831. args.replace(*newRight.getClear(), 2);
  10832. return expr->clone(args);
  10833. }
  10834. default:
  10835. return LINK(expr);
  10836. }
  10837. }
  10838. bool HqlTreeNormalizer::transformTransform(HqlExprArray & children, IHqlExpression * expr)
  10839. {
  10840. bool same = true;
  10841. ForEachChild(i, expr)
  10842. {
  10843. IHqlExpression * cur = expr->queryChild(i);
  10844. switch (cur->getOperator())
  10845. {
  10846. case no_assignall:
  10847. transformTransform(children, cur);
  10848. same = false; // assign all is removed and assigns expanded in its place
  10849. break;
  10850. case no_assign:
  10851. {
  10852. OwnedHqlExpr assign = transform(cur);
  10853. if (cur->getInfoFlags() & HEFcontainsSkip)
  10854. {
  10855. IHqlExpression * rhs = assign->queryChild(1);
  10856. OwnedHqlExpr newRhs = optimizeAssignSkip(children, rhs, NULL, 0);
  10857. if (rhs != newRhs)
  10858. {
  10859. IHqlExpression * lhs = assign->queryChild(0);
  10860. if (!newRhs)
  10861. newRhs.setown(createNullExpr(rhs));
  10862. assign.setown(createAssign(LINK(lhs), newRhs.getClear()));
  10863. }
  10864. }
  10865. if (assign != cur)
  10866. same = false;
  10867. children.append(*assign.getClear());
  10868. break;
  10869. }
  10870. default:
  10871. {
  10872. IHqlExpression * next = transform(cur);
  10873. children.append(*next);
  10874. if (next != cur)
  10875. same = false;
  10876. }
  10877. break;
  10878. }
  10879. }
  10880. return same;
  10881. }
  10882. IHqlExpression * HqlTreeNormalizer::transformTransform(IHqlExpression * expr)
  10883. {
  10884. HqlExprArray children;
  10885. IHqlExpression * oldRecord = queryOriginalRecord(expr);
  10886. OwnedHqlExpr newRecord = transform(oldRecord);
  10887. bool same = transformTransform(children, expr);
  10888. if ((oldRecord != newRecord) || !same)
  10889. {
  10890. ITypeInfo * newRecordType = createRecordType(newRecord);
  10891. OwnedHqlExpr ret = createValue(expr->getOperator(), makeTransformType(newRecordType), children);
  10892. return expr->cloneAllAnnotations(ret);
  10893. }
  10894. return LINK(expr);
  10895. }
  10896. IHqlExpression * HqlTreeNormalizer::transformIfAssert(node_operator newOp, IHqlExpression * expr)
  10897. {
  10898. unsigned max = expr->numChildren();
  10899. HqlExprArray children;
  10900. bool same = transformChildren(expr, children);
  10901. if ((expr->hasAttribute(assertAtom) || (options.assertSortedDistributed && (newOp != no_assertgrouped))) && !options.removeAsserts)
  10902. {
  10903. OwnedHqlExpr ret = createDataset(newOp, children);
  10904. return expr->cloneAllAnnotations(ret);
  10905. }
  10906. if (!same)
  10907. return expr->clone(children);
  10908. return LINK(expr);
  10909. }
  10910. IHqlExpression * HqlTreeNormalizer::transformExecuteWhen(IHqlExpression * expr)
  10911. {
  10912. OwnedHqlExpr transformedAction = transform(expr->queryChild(1));
  10913. if ((transformedAction->getOperator() == no_setmeta) ||
  10914. ((transformedAction->getOperator() == no_null) && transformedAction->isAction()))
  10915. return transform(expr->queryChild(0));
  10916. HqlExprArray children;
  10917. if (translator.queryOptions().convertWhenExecutedToCompound && !expr->queryChild(2))
  10918. {
  10919. //For the moment, for maximal compatibility, convert no_executewhen to a no_compound
  10920. children.append(*transformedAction.getClear());
  10921. children.append(*transform(expr->queryChild(0)));
  10922. OwnedHqlExpr ret = createCompound(children);
  10923. return expr->cloneAllAnnotations(ret);
  10924. }
  10925. //Need to create a unique id to differentiate the different side effects.
  10926. transformChildren(expr, children);
  10927. assertex(!expr->hasAttribute(_uid_Atom));
  10928. children.append(*createUniqueId());
  10929. return expr->clone(children);
  10930. }
  10931. IHqlExpression * HqlTreeNormalizer::transformWithinFilter(IHqlExpression * expr)
  10932. {
  10933. OwnedHqlExpr ds = transform(expr->queryChild(0));
  10934. HqlExprArray children;
  10935. children.append(*LINK(ds));
  10936. ForEachChildFrom(i, expr, 1)
  10937. {
  10938. IHqlExpression * filter = expr->queryChild(i);
  10939. if (filter->getOperator() == no_within)
  10940. {
  10941. IHqlExpression * scope = filter->queryChild(0);
  10942. while (scope->getOperator() == no_filter)
  10943. {
  10944. ForEachChildFrom(i2, scope, 1)
  10945. children.append(*transform(scope->queryChild(i2)));
  10946. scope = scope->queryChild(0);
  10947. }
  10948. ds.setown(createDataset(no_related, LINK(ds), transform(scope)));
  10949. }
  10950. else
  10951. children.append(*transform(filter));
  10952. }
  10953. if (children.ordinality() == 1)
  10954. return ds.getClear();
  10955. children.replace(*ds.getClear(), 0);
  10956. return expr->clone(children);
  10957. }
  10958. IHqlExpression * HqlTreeNormalizer::validateKeyedJoin(IHqlExpression * expr)
  10959. {
  10960. //Transform join(x, local(key), ....) to join(x, key, ...., local);
  10961. HqlExprArray children;
  10962. transformChildren(expr, children);
  10963. unsigned prevChildren = children.ordinality();
  10964. IHqlExpression * rhs = &children.item(1);
  10965. for (;;)
  10966. {
  10967. node_operator op = rhs->getOperator();
  10968. if (op == no_forcelocal)
  10969. children.append(*createLocalAttribute());
  10970. else if (op == no_forcenolocal)
  10971. children.append(*createAttribute(noLocalAtom));
  10972. else if ((op == no_section) || (op == no_sectioninput))
  10973. {
  10974. //remove the section
  10975. }
  10976. else
  10977. break;
  10978. rhs = rhs->queryChild(0);
  10979. }
  10980. if (prevChildren != children.ordinality())
  10981. {
  10982. if (isKey(rhs))
  10983. children.replace(*LINK(rhs), 1);
  10984. else
  10985. children.trunc(prevChildren);
  10986. }
  10987. //Now check that a join marked as keyed has a key as the rhs.
  10988. IHqlExpression * keyed = expr->queryAttribute(keyedAtom);
  10989. if (!keyed || keyed->queryChild(0) || isKey(rhs))
  10990. return expr->clone(children);
  10991. if (options.allowActivityForKeyedJoin)
  10992. {
  10993. children.append(*createAttribute(_complexKeyed_Atom));
  10994. return expr->clone(children);
  10995. }
  10996. StringBuffer s;
  10997. if (expr->queryName())
  10998. s.append(" (").append(expr->queryName()).append(")");
  10999. throwError1(HQLERR_RhsKeyedNotKey, s.str());
  11000. return NULL;
  11001. }
  11002. //A bit of a nasty dependency - this should match the capabilities of the code in hqlsource for finding selectors
  11003. static void gatherPotentialSelectors(HqlExprArray & args, IHqlExpression * expr)
  11004. {
  11005. node_operator op = expr->getOperator();
  11006. switch (op)
  11007. {
  11008. case no_and:
  11009. case no_or:
  11010. case no_eq:
  11011. case no_ne:
  11012. case no_gt:
  11013. case no_lt:
  11014. case no_ge:
  11015. case no_le:
  11016. gatherPotentialSelectors(args, expr->queryChild(0));
  11017. gatherPotentialSelectors(args, expr->queryChild(1));
  11018. break;
  11019. case no_if:
  11020. case no_case:
  11021. case no_map:
  11022. case no_mapto:
  11023. {
  11024. ForEachChild(i, expr)
  11025. gatherPotentialSelectors(args, expr->queryChild(i));
  11026. break;
  11027. }
  11028. case no_assertkeyed:
  11029. case no_assertstepped:
  11030. case no_not:
  11031. case no_between:
  11032. case no_notbetween:
  11033. case no_cast:
  11034. case no_implicitcast:
  11035. case no_notin:
  11036. case no_in:
  11037. case no_add:
  11038. case no_sub:
  11039. case no_substring:
  11040. gatherPotentialSelectors(args, expr->queryChild(0));
  11041. break;
  11042. case no_select:
  11043. {
  11044. IHqlExpression * selector = expr->queryNormalizedSelector();
  11045. if (!args.contains(*selector))
  11046. args.append(*LINK(selector));
  11047. break;
  11048. }
  11049. }
  11050. }
  11051. IHqlExpression * HqlTreeNormalizer::transformChildrenNoAnnotations(IHqlExpression * expr)
  11052. {
  11053. HqlExprArray args;
  11054. ForEachChild(i, expr)
  11055. {
  11056. OwnedHqlExpr newChild = transform(expr->queryChild(i)->queryBody());
  11057. args.append(*LINK(newChild->queryBody()));
  11058. }
  11059. return completeTransform(expr, args);
  11060. }
  11061. //The following symbol removal code works, but I'm not sure I want to do it at the moment because of the changes to the HOLe queries
  11062. //Remove as many named symbols as we can - try and keep for datasets and statements so can go in the tree.
  11063. IHqlExpression * HqlTreeNormalizer::createTransformed(IHqlExpression * expr)
  11064. {
  11065. IHqlExpression * body = expr->queryBody(true);
  11066. node_operator op = expr->getOperator();
  11067. if (expr != body)
  11068. {
  11069. OwnedHqlExpr transformedBody;
  11070. try
  11071. {
  11072. transformedBody.setown(transform(body));
  11073. }
  11074. catch (IException * e)
  11075. {
  11076. if (dynamic_cast<IError *>(e))
  11077. throw;
  11078. IHqlExpression * location = queryLocation(expr);
  11079. if (location)
  11080. {
  11081. IError * error = annotateExceptionWithLocation(e, location);
  11082. e->Release();
  11083. throw error;
  11084. }
  11085. throw;
  11086. }
  11087. //Don't retain any annotations on records - except for a symbol which maybe added in the createTransform()
  11088. //code. Otherwise expressions that would otherwise be commoned up are treated as different.
  11089. if (op == no_record)
  11090. return transformedBody.getClear();
  11091. switch (expr->getAnnotationKind())
  11092. {
  11093. case annotate_warning:
  11094. case annotate_parsemeta:
  11095. return transformedBody.getClear();
  11096. case annotate_javadoc:
  11097. return expr->cloneAnnotation(transformedBody);
  11098. case annotate_meta:
  11099. {
  11100. HqlExprArray preservedMeta;
  11101. IHqlExpression * cur;
  11102. bool changed = false;
  11103. for (unsigned i=0; (cur = expr->queryAnnotationParameter(i)) != 0; i++)
  11104. {
  11105. IAtom * name = cur->queryName();
  11106. bool keep = true;
  11107. if (name == deprecatedAtom)
  11108. keep = false;
  11109. else if (!options.allowSections && (name == sectionAtom))
  11110. keep = false;
  11111. if (keep)
  11112. preservedMeta.append(*LINK(cur));
  11113. else
  11114. changed = true;
  11115. }
  11116. if (changed)
  11117. {
  11118. if (preservedMeta.ordinality() == 0)
  11119. return transformedBody.getClear();
  11120. return createMetaAnnotation(transformedBody.getClear(), preservedMeta);
  11121. }
  11122. break; // default action
  11123. }
  11124. case annotate_symbol:
  11125. {
  11126. if (hasNamedSymbol(transformedBody))
  11127. return transformedBody.getClear();
  11128. IIdAtom * id = expr->queryId();
  11129. IIdAtom * simpleId = simplifySymbolName(id, options.commonUniqueNameAttributes);
  11130. if (simpleId)
  11131. return cloneSymbol(expr, simpleId, transformedBody, NULL, NULL);
  11132. break;
  11133. }
  11134. } // switch(kind)
  11135. if (body == transformedBody)
  11136. return LINK(expr);
  11137. return expr->cloneAnnotation(transformedBody);
  11138. }
  11139. //This option is purely for regression testing, to ensure attributes are not lost, and no infinite recursion occurs.
  11140. if (options.forceAllDatasetsParallel)
  11141. {
  11142. if (expr->isDataset() && !expr->hasAttribute(parallelAtom))
  11143. {
  11144. switch (op)
  11145. {
  11146. case no_colon:
  11147. case no_rows:
  11148. case no_select:
  11149. case no_call:
  11150. case no_externalcall:
  11151. case no_matchattr:
  11152. break;
  11153. default:
  11154. {
  11155. OwnedHqlExpr parallel = appendAttribute(expr, parallelAtom);
  11156. return transform(parallel);
  11157. }
  11158. }
  11159. }
  11160. }
  11161. //MORE: Types of all pattern attributes should also be normalized. Currently they aren't which causes discrepancies between types
  11162. //for ghoogle.hql. It could conceivably cause problems later on.
  11163. if (forwardReferences.ordinality())
  11164. {
  11165. if (op == no_pat_use && expr->hasAttribute(nameAtom))
  11166. return transformPatNamedUse(expr);
  11167. if (op == no_pat_instance)
  11168. {
  11169. OwnedHqlExpr ret = queryTransformPatternDefine (expr);
  11170. if (ret)
  11171. return ret.getClear();
  11172. }
  11173. }
  11174. IHqlExpression * sideEffects = expr->queryAttribute(_sideEffect_Atom);
  11175. if (sideEffects)
  11176. {
  11177. HqlExprArray args;
  11178. unwindChildren(args, expr);
  11179. args.zap(*sideEffects);
  11180. OwnedHqlExpr next = createCompound(LINK(sideEffects->queryChild(0)), expr->clone(args));
  11181. return transform(next);
  11182. }
  11183. if (!options.constantFoldNormalize)
  11184. return createTransformedBody(expr);
  11185. switch (op)
  11186. {
  11187. case no_if:
  11188. {
  11189. OwnedHqlExpr cond = transform(expr->queryChild(0));
  11190. IValue * condValue = cond->queryValue();
  11191. if (condValue)
  11192. {
  11193. unsigned idx = condValue->getBoolValue() ? 1 : 2;
  11194. IHqlExpression * branch = expr->queryChild(idx);
  11195. if (branch)
  11196. return transform(branch);
  11197. assertex(expr->isAction());
  11198. return createValue(no_null, makeVoidType());
  11199. }
  11200. break;
  11201. }
  11202. case no_choose:
  11203. case no_chooseds:
  11204. {
  11205. OwnedHqlExpr cond = transform(expr->queryChild(0));
  11206. IValue * condValue = cond->queryValue();
  11207. if (condValue)
  11208. {
  11209. unsigned idx = (unsigned)condValue->getIntValue();
  11210. if (idx != 0)
  11211. {
  11212. IHqlExpression * branch = queryRealChild(expr, idx);
  11213. if (branch)
  11214. return transform(branch);
  11215. }
  11216. IHqlExpression * defaultExpr = queryLastNonAttribute(expr);
  11217. return transform(defaultExpr);
  11218. }
  11219. break;
  11220. }
  11221. case no_and:
  11222. {
  11223. IHqlExpression * left = expr->queryChild(0);
  11224. IHqlExpression * right = expr->queryChild(1);
  11225. OwnedHqlExpr simpleRight = transformSimpleConst(right);
  11226. if (simpleRight->queryValue())
  11227. {
  11228. if (simpleRight->queryValue()->getBoolValue())
  11229. return transform(left);
  11230. return simpleRight.getClear();
  11231. }
  11232. OwnedHqlExpr newLeft = transform(left);
  11233. IValue * leftValue = newLeft->queryValue();
  11234. if (leftValue)
  11235. {
  11236. if (!leftValue->getBoolValue())
  11237. return newLeft.getClear();
  11238. return transform(right);
  11239. }
  11240. break;
  11241. }
  11242. case no_or:
  11243. {
  11244. IHqlExpression * left = expr->queryChild(0);
  11245. IHqlExpression * right = expr->queryChild(1);
  11246. OwnedHqlExpr simpleRight = transformSimpleConst(right);
  11247. if (simpleRight->queryValue())
  11248. {
  11249. if (!simpleRight->queryValue()->getBoolValue())
  11250. return transform(left);
  11251. return simpleRight.getClear();
  11252. }
  11253. OwnedHqlExpr newLeft = transform(left);
  11254. IValue * leftValue = newLeft->queryValue();
  11255. if (leftValue)
  11256. {
  11257. if (leftValue->getBoolValue())
  11258. return newLeft.getClear();
  11259. return transform(right);
  11260. }
  11261. break;
  11262. }
  11263. case no_attr:
  11264. if (expr->queryName() == _original_Atom)
  11265. return LINK(expr);
  11266. break;
  11267. }
  11268. OwnedHqlExpr transformed = createTransformedBody(expr);
  11269. return foldConstantOperator(transformed, 0, NULL);
  11270. }
  11271. IHqlExpression * HqlTreeNormalizer::createTransformedBody(IHqlExpression * expr)
  11272. {
  11273. node_operator op = expr->getOperator();
  11274. switch (op)
  11275. {
  11276. case no_constant:
  11277. return LINK(expr); // avoid creating an array in default code...
  11278. case no_case:
  11279. if (isVoidOrDatasetOrList(expr) || expr->isDictionary())
  11280. return transformCaseToChoose(expr);
  11281. break;
  11282. case no_map:
  11283. if (isVoidOrDatasetOrList(expr) || expr->isDictionary())
  11284. return transformMap(expr);
  11285. break;
  11286. case no_transform:
  11287. //optimize location of skips
  11288. return transformTransform(expr);
  11289. case no_getresult:
  11290. case no_newtransform:
  11291. {
  11292. IHqlExpression * record = queryOriginalRecord(expr);
  11293. if (record)
  11294. ::Release(transform(record));
  11295. LinkedHqlExpr cleaned = expr;
  11296. //remove any no_assignall children... could really do for no_transform as well... would reduce clarity of graph ecl
  11297. if ((op == no_newtransform) && queryChildOperator(no_assignall, expr))
  11298. {
  11299. HqlExprArray args;
  11300. ForEachChild(i, expr)
  11301. expr->queryChild(i)->unwindList(args, no_assignall);
  11302. cleaned.setown(expr->clone(args));
  11303. }
  11304. return Parent::createTransformed(cleaned);
  11305. }
  11306. case no_usertable:
  11307. case no_selectfields:
  11308. {
  11309. OwnedHqlExpr newRecord = transform(expr->queryChild(1));
  11310. return convertSelectToProject(newRecord, expr);
  11311. }
  11312. case no_parse:
  11313. return removeDefaultsFromExpr(expr, 3, no_newparse);
  11314. case no_xmlparse:
  11315. return removeDefaultsFromExpr(expr, 2, no_newxmlparse);
  11316. case no_soapcall:
  11317. return removeDefaultsFromExpr(expr, 2, no_newsoapcall);
  11318. case no_soapcall_ds:
  11319. return removeDefaultsFromExpr(expr, 3, no_newsoapcall_ds);
  11320. case no_soapaction_ds:
  11321. return removeDefaultsFromExpr(expr, 3, no_newsoapaction_ds);
  11322. #ifdef OPTIMIZE_IMPLICIT_CAST
  11323. //Following is a good idea, but makes some things worse because of the way we currently spot table invariants.
  11324. case no_implicitcast:
  11325. {
  11326. OwnedHqlExpr transformed = NewHqlTransformer::createTransformed(expr);
  11327. return ensureExprType(transformed->queryChild(0), transformed->queryType());
  11328. }
  11329. #endif
  11330. case no_record:
  11331. {
  11332. OwnedHqlExpr transformed = completeTransform(expr);
  11333. if (transformed->hasAttribute(lookupAtom))
  11334. {
  11335. StringBuffer filename;
  11336. IHqlExpression *dfsAttr = transformed->queryAttribute(lookupAtom);
  11337. if (dfsAttr)
  11338. getStringValue(filename, dfsAttr->queryChild(0));
  11339. translator.reportError(expr, ECODETEXT(HQLERR_DFSlookupFailure), filename.str());
  11340. }
  11341. if (transformed->hasAttribute(packedAtom))
  11342. transformed.setown(getPackedRecord(transformed));
  11343. if (options.ensureRecordsHaveSymbols)
  11344. {
  11345. //Ensure all records only have a single unique name, and transform it here so that record types also map to that unique name
  11346. IHqlExpression * recordSymbol = queryLocationIndependentExtra(expr)->symbol;
  11347. if (recordSymbol)
  11348. {
  11349. IIdAtom * id = recordSymbol->queryId();
  11350. IIdAtom * simpleId = simplifySymbolName(id, options.commonUniqueNameAttributes);
  11351. IIdAtom * newid = simpleId ? simpleId : id;
  11352. IIdAtom * lowerId = lowerCaseSymbolName(newid);
  11353. return createSymbol(lowerId, transformed.getClear(), ob_private);
  11354. }
  11355. }
  11356. return transformed.getClear();
  11357. }
  11358. case no_left:
  11359. case no_right:
  11360. case no_top:
  11361. case no_self:
  11362. {
  11363. IHqlExpression * record = expr->queryChild(0);
  11364. //If no record argument then make sure the type is transformed
  11365. if (!record)
  11366. break;
  11367. HqlExprArray children;
  11368. //Ensure that the first parameter to one of these nodes is the body of the record, not a named symbol
  11369. OwnedHqlExpr transformedRecord = transform(record);
  11370. children.append(*LINK(transformedRecord->queryBody()));
  11371. return completeTransform(expr, children);
  11372. }
  11373. case no_field:
  11374. {
  11375. //Remove the default values...
  11376. HqlExprArray children;
  11377. bool same = true;
  11378. ForEachChild(idx, expr)
  11379. {
  11380. IHqlExpression * cur = expr->queryChild(idx);
  11381. if (cur->isAttribute())
  11382. {
  11383. IHqlExpression * transformed = transform(cur);
  11384. children.append(*transformed);
  11385. if (cur != transformed)
  11386. same = false;
  11387. }
  11388. else
  11389. same = false;
  11390. }
  11391. ITypeInfo * type = expr->queryType();
  11392. OwnedITypeInfo newType = transformType(type);
  11393. IIdAtom * id = expr->queryId();
  11394. IIdAtom * newId = lowerCaseSymbolName(id);
  11395. if ((type != newType) || (newId != id))
  11396. return createField(newId, newType.getClear(), children);
  11397. if (same)
  11398. return LINK(expr);
  11399. return expr->clone(children);
  11400. }
  11401. case no_trim:
  11402. //TRIM(x,RIGHT) should be represented the same way as TRIM(x) - and it's more efficient
  11403. if ((expr->numChildren() == 2) && (expr->queryChild(1)->queryName() == rightAtom))
  11404. {
  11405. HqlExprArray children;
  11406. children.append(*transform(expr->queryChild(0)));
  11407. return expr->clone(children);
  11408. }
  11409. break;
  11410. case no_pat_pattern:
  11411. return LINK(expr->queryChild(1));
  11412. case no_temptable:
  11413. return transformTempTable(expr);
  11414. case no_temprow:
  11415. return transformTempRow(expr);
  11416. case no_keyindex:
  11417. return transformKeyIndex(expr);
  11418. case no_newkeyindex:
  11419. // seenIndex = true;
  11420. return transformNewKeyIndex(expr);
  11421. case no_table:
  11422. if (expr->hasAttribute(localUploadAtom))
  11423. seenLocalUpload = true;
  11424. return transformTable(expr);
  11425. case no_pat_checkin:
  11426. if (expr->queryChild(0)->getOperator() == no_pat_anychar)
  11427. {
  11428. IHqlExpression * transformed = transformPatCheckIn(expr);
  11429. if (transformed)
  11430. return transformed;
  11431. }
  11432. break;
  11433. case no_denormalize:
  11434. case no_denormalizegroup:
  11435. {
  11436. OwnedHqlExpr transformed = validateKeyedJoin(expr);
  11437. //Explicitly add a left outer flag to a denormalize if no other join type is specified.
  11438. //Do here rather than in parser so crc for persists isn't changed.
  11439. if (!transformed->hasAttribute(innerAtom) &&
  11440. !transformed->hasAttribute(leftonlyAtom) && !transformed->hasAttribute(leftouterAtom) &&
  11441. !transformed->hasAttribute(rightonlyAtom) && !transformed->hasAttribute(rightouterAtom) &&
  11442. !transformed->hasAttribute(fullonlyAtom) && !transformed->hasAttribute(fullouterAtom))
  11443. {
  11444. return appendOwnedOperand(transformed, createAttribute(leftouterAtom));
  11445. }
  11446. return transformed.getClear();
  11447. }
  11448. case no_colon:
  11449. {
  11450. OwnedHqlExpr transformed = Parent::createTransformed(expr);
  11451. LinkedHqlExpr value = transformed->queryChild(0);
  11452. bool same = true;
  11453. bool needToPreserveOriginal = false;
  11454. HqlExprArray actions, scheduleActions;
  11455. unwindChildren(actions, transformed, 1);
  11456. ForEachItemInRev(i, actions)
  11457. {
  11458. IHqlExpression & cur = actions.item(i);
  11459. IHqlExpression * replacement = NULL;
  11460. switch (cur.getOperator())
  11461. {
  11462. case no_global:
  11463. {
  11464. HqlExprArray scopeArgs;
  11465. scopeArgs.append(*LINK(value));
  11466. unwindChildren(scopeArgs, &cur);
  11467. replacement = createWrapper(no_globalscope, scopeArgs);
  11468. break;
  11469. }
  11470. case no_persist:
  11471. {
  11472. needToPreserveOriginal = true;
  11473. same = false;
  11474. break;
  11475. }
  11476. case no_attr:
  11477. case no_attr_expr:
  11478. case no_attr_link:
  11479. if (cur.queryName() == defineAtom)
  11480. replacement = createValue(no_define, transformed->getType(), LINK(value), LINK(cur.queryChild(0)));
  11481. break;
  11482. //Separate scheduled items into a separate no_colon.
  11483. case no_when:
  11484. case no_priority:
  11485. scheduleActions.append(OLINK(cur));
  11486. actions.remove(i);
  11487. same = false;
  11488. break;
  11489. }
  11490. if (replacement)
  11491. {
  11492. value.setown(replacement);
  11493. actions.remove(i);
  11494. same = false;
  11495. }
  11496. }
  11497. if (same)
  11498. return transformed.getClear();
  11499. if (needToPreserveOriginal)
  11500. actions.append(*createAttribute(_original_Atom, LINK(expr->queryChild(0))));
  11501. OwnedHqlExpr result;
  11502. if (actions.ordinality() == 0)
  11503. result.set(value);
  11504. else
  11505. result.setown(createColon(LINK(value), actions));
  11506. if (scheduleActions.ordinality())
  11507. result.setown(createColon(result.getClear(), scheduleActions));
  11508. return result.getClear();
  11509. }
  11510. case no_evaluate:
  11511. return transformEvaluate(expr);
  11512. case no_selectnth:
  11513. {
  11514. IHqlExpression * ds = expr->queryChild(0);
  11515. if (isGrouped(ds))
  11516. {
  11517. OwnedHqlExpr newChild = createDataset(no_group, LINK(ds));
  11518. OwnedHqlExpr mapped = replaceChild(expr, 0, newChild);
  11519. return transform(mapped);
  11520. }
  11521. break;
  11522. }
  11523. case no_assert_ds:
  11524. if (options.removeAsserts)
  11525. return transform(expr->queryChild(0));
  11526. break;
  11527. case no_section:
  11528. case no_sectioninput:
  11529. if (!options.allowSections)
  11530. return transform(expr->queryChild(0));
  11531. break;
  11532. case no_type:
  11533. return transformAlienType(expr);
  11534. case no_param:
  11535. {
  11536. //no_param may be retained by library call definitions + they need the type transforming for consistency
  11537. ITypeInfo * type = expr->queryType();
  11538. OwnedITypeInfo newType = transformType(type);
  11539. if (type != newType)
  11540. {
  11541. //Attributes shouldn't need transforming, but simplest
  11542. HqlExprArray attrs;
  11543. transformChildren(expr, attrs);
  11544. return createParameter(expr->queryId(), (unsigned)expr->querySequenceExtra(), newType.getClear(), attrs);
  11545. }
  11546. break;
  11547. }
  11548. case no_libraryscope:
  11549. {
  11550. OwnedHqlExpr ret = transformScope(expr);
  11551. if (translator.targetHThor())
  11552. return appendOwnedOperand(ret, createAttribute(_noStreaming_Atom));
  11553. return ret.getClear();
  11554. }
  11555. case no_virtualscope:
  11556. return transformScope(expr);
  11557. case no_libraryscopeinstance:
  11558. {
  11559. IHqlExpression * oldFunction = expr->queryDefinition();
  11560. OwnedHqlExpr newFunction = transform(oldFunction);
  11561. HqlExprArray children;
  11562. bool same = true;
  11563. ForEachChild(i, expr)
  11564. {
  11565. LinkedHqlExpr cur = expr->queryChild(i);
  11566. if (cur->getOperator() == no_virtualscope)
  11567. {
  11568. cur.setown(checkCreateConcreteModule(NULL, cur, cur->queryAttribute(_location_Atom)));
  11569. assertex(cur->getOperator() != no_virtualscope);
  11570. same = false;
  11571. }
  11572. else if (cur->getOperator() == no_purevirtual)
  11573. {
  11574. IAtom * name = cur->queryName();
  11575. throwError1(HQLERR_LibraryMemberArgNotDefined, name ? str(name) : "");
  11576. }
  11577. IHqlExpression * transformed = transform(cur);
  11578. children.append(*transformed);
  11579. if (cur != transformed)
  11580. same = false;
  11581. }
  11582. if (same && (oldFunction == newFunction))
  11583. return LINK(expr);
  11584. return createLibraryInstance(newFunction.getClear(), children);
  11585. }
  11586. case no_transformascii:
  11587. case no_transformebcdic:
  11588. {
  11589. HqlExprArray children;
  11590. transformChildren(expr, children);
  11591. OwnedHqlExpr transformed = createDataset(no_hqlproject, children);
  11592. return transform(transformed);
  11593. }
  11594. case no_join:
  11595. {
  11596. OwnedHqlExpr transformed = validateKeyedJoin(expr);
  11597. if (isSelfJoin(expr))
  11598. {
  11599. HqlExprArray children;
  11600. unwindChildren(children, transformed);
  11601. children.replace(*createAttribute(_selfJoinPlaceholder_Atom), 1); // replace the 1st dataset with an attribute so parameters are still in the same place.
  11602. return createDataset(no_selfjoin, children);
  11603. }
  11604. if (isKeyedJoin(transformed) && translator.targetRoxie() && !hasOrderedAttribute(expr))
  11605. return appendOwnedOperand(transformed, createAttribute(_ordered_Atom));
  11606. return transformed.getClear();
  11607. }
  11608. case no_projectrow:
  11609. {
  11610. //Work around a problem where left is ambiguous - either outer LEFT, or left within this ROW
  11611. //Not a full solution since PROJECT(PROJECT(LEFT),t(LEFT)) where project(LEFT) doesn't change types
  11612. //would suffer from the same problem.
  11613. //Remove as many instances of PROJECT(row, transform) as we can since ROW(transform) is handled more efficient.
  11614. HqlExprArray children;
  11615. OwnedHqlExpr ds = transform(expr->queryChild(0));
  11616. node_operator dsOp = ds->getOperator();
  11617. if (dsOp == no_left)
  11618. // if (isAlwaysActiveRow(ds))
  11619. {
  11620. //MORE: The call to replaceExpression below isn't actually correct unless selectors are unique
  11621. //this optimization may have to move elsewhere.
  11622. OwnedHqlExpr newTransform = transform(expr->queryChild(1));
  11623. OwnedHqlExpr newSel = transform(querySelSeq(expr));
  11624. OwnedHqlExpr myLeft = createSelector(no_left, ds, newSel);
  11625. OwnedHqlExpr replaced = quickFullReplaceExpression(newTransform, myLeft, ds);
  11626. return createRow(no_createrow, LINK(replaced));
  11627. }
  11628. children.append(*ds.getClear());
  11629. return completeTransform(expr, children);
  11630. }
  11631. case no_sorted:
  11632. return transformIfAssert(no_assertsorted, expr);
  11633. case no_grouped:
  11634. return transformIfAssert(no_assertgrouped, expr);
  11635. case no_distributed:
  11636. //remove distributed(x)
  11637. if (expr->hasAttribute(unknownAtom))
  11638. return transform(expr->queryChild(0));
  11639. return transformIfAssert(no_assertdistributed, expr);
  11640. #if defined(MAP_PROJECT_TO_USERTABLE)
  11641. case no_hqlproject:
  11642. if (!isCountProject(expr))
  11643. {
  11644. HqlExprArray children;
  11645. transformChildren(expr, children);
  11646. IHqlExpression * ds = &children.item(0);
  11647. OwnedHqlExpr left = createSelector(no_left, ds, querySelSeq(expr));
  11648. OwnedHqlExpr mapped = replaceExpression(&children.item(1), left, ds->queryNormalizedSelector());
  11649. children.add(*LINK(mapped->queryRecord()), 1);
  11650. HqlExprArray assigns;
  11651. unwindChildren(assigns, mapped);
  11652. children.replace(*createValue(no_newtransform, mapped->getType(), assigns), 2);
  11653. OwnedHqlExpr transformed = createDataset(no_newusertable, children);
  11654. return transform(transformed);
  11655. }
  11656. break;
  11657. #endif
  11658. case no_comma:
  11659. case no_compound:
  11660. if (expr->queryChild(0)->getOperator() == no_setmeta)
  11661. return transform(expr->queryChild(1));
  11662. if ((op == no_compound) && expr->isAction())
  11663. {
  11664. HqlExprArray args;
  11665. expr->unwindList(args, no_compound);
  11666. OwnedHqlExpr compound = createAction(no_actionlist, args);
  11667. return transform(compound);
  11668. }
  11669. break;
  11670. case no_actionlist:
  11671. case no_orderedactionlist:
  11672. return transformActionList(expr);
  11673. case no_forcelocal:
  11674. case no_forcenolocal:
  11675. case no_allnodes:
  11676. case no_thisnode:
  11677. seenForceLocal = true;
  11678. break;
  11679. case no_enth:
  11680. {
  11681. HqlExprArray children;
  11682. bool same = transformChildren(expr, children);
  11683. IHqlExpression * denom = queryRealChild(expr, 2);
  11684. if (!denom && !expr->queryAttribute(localAtom))
  11685. {
  11686. children.add(*createValue(no_count, LINK(defaultIntegralType), LINK(&children.item(0))), 2);
  11687. same = false;
  11688. }
  11689. if (!same)
  11690. return expr->clone(children);
  11691. return LINK(expr);
  11692. }
  11693. case no_assertconstant:
  11694. {
  11695. IHqlExpression * child = expr->queryChild(0);
  11696. OwnedHqlExpr ret = transform(child);
  11697. OwnedHqlExpr folded = foldHqlExpression(translator.queryErrorProcessor(), ret, NULL, HFOforcefold);
  11698. if (!folded->isConstant())
  11699. {
  11700. StringBuffer s;
  11701. getExprECL(child, s);
  11702. translator.ERRORAT1(expr->queryChild(1), HQLERR_ExpectedConstant, s.str());
  11703. }
  11704. return folded.getClear();
  11705. }
  11706. case no_assertconcrete:
  11707. {
  11708. ECLlocation errpos;
  11709. errpos.extractLocationAttr(expr->queryChild(1));
  11710. reportAbstractModule(translator.queryErrorProcessor(), expr->queryChild(0), errpos);
  11711. throw MakeStringException(HQLERR_ErrorAlreadyReported, "%s", "");
  11712. }
  11713. case no_pat_instance:
  11714. {
  11715. OwnedHqlExpr child = transform(expr->queryChild(0));
  11716. if (child->getOperator() == no_pat_instance && child->hasAttribute(tempAtom))
  11717. return createValue(no_pat_instance, child->getType(), LINK(child->queryChild(0)));
  11718. //default action
  11719. break;
  11720. }
  11721. case no_if:
  11722. {
  11723. //Parameters are being used a lot to select between two items in inside a function/module
  11724. //so much better if we trim the tree earlier....
  11725. IValue * value = expr->queryChild(0)->queryValue();
  11726. if (value && !expr->isAction())
  11727. {
  11728. unsigned branch = value->getBoolValue() ? 1 : 2;
  11729. IHqlExpression * arg = expr->queryChild(branch);
  11730. if (arg)
  11731. return transform(arg);
  11732. }
  11733. break;
  11734. }
  11735. case no_stored:
  11736. {
  11737. HqlExprArray children;
  11738. OwnedHqlExpr name = transform(expr->queryChild(0));
  11739. children.append(*getLowerCaseConstantExpr(name));
  11740. return completeTransform(expr, children);
  11741. }
  11742. case no_merge:
  11743. return transformMerge(expr);
  11744. //yuk: Sets of datasets need special casing because their type isn't implicitly calculated from their inputs.
  11745. case no_datasetlist:
  11746. case no_rowset:
  11747. {
  11748. HqlExprArray children;
  11749. transformChildren(expr, children);
  11750. OwnedITypeInfo setType = makeSetType(children.item(0).getType());
  11751. return createValue(op, setType.getClear(), children);
  11752. }
  11753. case no_rowsetrange:
  11754. {
  11755. HqlExprArray children;
  11756. transformChildren(expr, children);
  11757. OwnedITypeInfo setType = children.item(0).getType();
  11758. return createValue(op, setType.getClear(), children);
  11759. }
  11760. case no_buildindex:
  11761. {
  11762. //Normalize the index build by splitting out the sort here, so that constant percolating
  11763. //is also done on these parameters
  11764. OwnedHqlExpr transformed = Parent::createTransformed(expr);
  11765. for (;;)
  11766. {
  11767. IHqlExpression * ret = normalizeIndexBuild(transformed, options.sortIndexPayload, !translator.targetThor(), options.implicitSubSort);
  11768. if (!ret)
  11769. return LINK(transformed);
  11770. transformed.setown(ret);
  11771. }
  11772. }
  11773. case no_keyed:
  11774. {
  11775. HqlExprArray args;
  11776. bool same = transformChildren(expr, args);
  11777. IHqlExpression * ds = &args.item(0);
  11778. if ((ds->getOperator() == no_section) || (ds->getOperator() == no_sectioninput))
  11779. {
  11780. args.replace(*LINK(ds->queryChild(0)), 0);
  11781. same = false;
  11782. }
  11783. if (!same)
  11784. return expr->clone(args);
  11785. return LINK(expr);
  11786. }
  11787. case no_eclcrc:
  11788. {
  11789. OwnedHqlExpr arg = transform(expr->queryChild(0)->queryChild(0));
  11790. return createConstant(expr->queryType()->castFrom(true, (__int64)getExpressionCRC(arg)));
  11791. }
  11792. case no_cast:
  11793. if (options.normalizeExplicitCasts)
  11794. {
  11795. Owned<ITypeInfo> type = transformType(expr->queryType());
  11796. OwnedHqlExpr arg = transform(expr->queryChild(0));
  11797. return createValue(no_implicitcast, type.getClear(), arg.getClear());
  11798. }
  11799. break;
  11800. #if 0
  11801. //This code adds a assertsorted activity after an nary-join, but I don't think it is actually correct, so removed. I may revisit.
  11802. case no_nwayjoin:
  11803. if (expr->hasAttribute(assertAtom) && !removeAsserts)
  11804. {
  11805. OwnedHqlExpr transformed = NewHqlTransformer::createTransformed(expr);;
  11806. IHqlExpression * ds = transformed->queryChild(0);
  11807. IHqlExpression * selSeq = querySelSeq(transformed);
  11808. OwnedHqlExpr left = createSelector(no_left, ds, selSeq);
  11809. IHqlExpression * sortOrder = transformed->queryChild(3);
  11810. HqlExprArray args;
  11811. args.append(*LINK(transformed));
  11812. //MORE: Need fixing once join can have a different output format
  11813. args.append(*replaceSelector(sortOrder, left, transformed->queryNormalizedSelector()));
  11814. return createDataset(no_assertsorted, args);
  11815. }
  11816. break;
  11817. #endif
  11818. case no_attr:
  11819. case no_attr_link:
  11820. case no_attr_expr:
  11821. {
  11822. IAtom * name = expr->queryName();
  11823. if ((name == _uid_Atom) && (expr->numChildren() > 0))
  11824. {
  11825. //Make sure we ignore any line number information on the parameters mangled with the uid - otherwise
  11826. //they may create too many unique ids.
  11827. IHqlExpression * normalForm = queryLocationIndependent(expr);
  11828. if (normalForm != expr)
  11829. return transform(normalForm);
  11830. return ::createUniqueId();
  11831. }
  11832. #ifdef USE_SELSEQ_UID
  11833. if (name == _selectorSequence_Atom)
  11834. {
  11835. //Ensure parameterised sequences generate a unique sequence number...
  11836. //Not sure the following is really necessary, but will reduce in memory tree size....
  11837. //also saves complications from having weird attributes in the tree
  11838. if (expr->numChildren() > 0)
  11839. {
  11840. //Make sure we ignore any line number information on the parameters mangled with the uid - otherwise
  11841. //they may create too many unique ids.
  11842. OwnedHqlExpr transformed = Parent::createTransformed(expr);
  11843. IHqlExpression * normalForm = queryLocationIndependent(transformed);
  11844. OwnedHqlExpr ret;
  11845. if (normalForm != expr)
  11846. {
  11847. IHqlExpression * mapped = queryAlreadyTransformed(normalForm);
  11848. if (!mapped)
  11849. {
  11850. ret.setown(createSelectorSequence());
  11851. setTransformed(normalForm, ret);
  11852. }
  11853. else
  11854. ret.set(mapped);
  11855. }
  11856. else
  11857. ret.setown(createSelectorSequence());
  11858. return ret.getClear();
  11859. }
  11860. }
  11861. #endif
  11862. //If a named symbol is used as an argument to maxlength you can end up with a situation
  11863. //where records are identical except for that named symbol.
  11864. //If f(a + b) is then optimized to b this can lead to incompatible selectors, since
  11865. //a and b are "compatible", but not identical.
  11866. //This then causes chaos, so strip them as a precaution... but it is only a partial solution.
  11867. if (name == maxLengthAtom || name == maxCountAtom)
  11868. return transformChildrenNoAnnotations(expr);
  11869. break;
  11870. }
  11871. case no_call:
  11872. {
  11873. IHqlExpression * oldFuncdef = expr->queryFunctionDefinition();
  11874. if (oldFuncdef->getOperator() == no_delayedselect)
  11875. {
  11876. IHqlExpression * module = oldFuncdef->queryChild(1);
  11877. ECLlocation errpos(module);
  11878. //errpos.extractLocationAttr(expr->queryChild(1));
  11879. reportAbstractModule(translator.queryErrorProcessor(), module, errpos);
  11880. throw MakeStringException(HQLERR_ErrorAlreadyReported, "%s", "");
  11881. }
  11882. if (oldFuncdef->getOperator() == no_param)
  11883. return LINK(expr);
  11884. assertex(oldFuncdef->getOperator() == no_funcdef);
  11885. return transformCall(expr);
  11886. }
  11887. case no_externalcall:
  11888. //Yuk.... Because we ensure that all records have a name, we need to make sure that external functions that return records
  11889. //also have there return value normalized - otherwise (jtolbert2.xhql) you can create an ambiguity
  11890. //We could also want to do this for user functions - but better would be to have a different node type.
  11891. if (options.ensureRecordsHaveSymbols)
  11892. {
  11893. if (expr->queryRecord())
  11894. return transformExternalCall(expr);
  11895. }
  11896. break;
  11897. case no_external:
  11898. {
  11899. ITypeInfo * type = expr->queryType();
  11900. OwnedITypeInfo newType = transformType(type);
  11901. HqlExprArray args;
  11902. bool same = transformChildren(expr, args);
  11903. if (same && (type == newType))
  11904. return LINK(expr);
  11905. return createExternalReference(expr->queryId(), newType.getClear(), args);
  11906. }
  11907. case no_outputscalar:
  11908. if (options.outputRowsAsDatasets && expr->queryChild(0)->isDatarow())
  11909. {
  11910. HqlExprArray args;
  11911. bool same = transformChildren(expr, args);
  11912. args.replace(*createDatasetFromRow(LINK(&args.item(0))), 0);
  11913. return createValue(no_output, makeVoidType(), args);
  11914. }
  11915. break;
  11916. case no_nameof:
  11917. {
  11918. OwnedHqlExpr newChild = transform(expr->queryChild(0));
  11919. switch (newChild->getOperator())
  11920. {
  11921. case no_newkeyindex:
  11922. return LINK(newChild->queryChild(3));
  11923. case no_table:
  11924. return LINK(newChild->queryChild(0));
  11925. default:
  11926. throwError(HQLERR_CannotDeduceNameDataset);
  11927. }
  11928. break;
  11929. }
  11930. case no_typedef:
  11931. {
  11932. HqlExprArray children;
  11933. transformChildren(expr, children);
  11934. OwnedITypeInfo newType = transformType(expr->queryType());
  11935. return createValue(op, newType.getClear(), children);
  11936. }
  11937. case no_assertkeyed:
  11938. {
  11939. //Ensure assertkeyed is tagged with the selectors of each of the fields that are keyed, otherwise
  11940. //when expressions are constant folded, the information about keyed fields is lost.
  11941. HqlExprArray children;
  11942. transformChildren(expr, children);
  11943. HqlExprArray args;
  11944. gatherPotentialSelectors(args, expr);
  11945. OwnedHqlExpr selectors = createExprAttribute(_selectors_Atom, args);
  11946. children.append(*transform(selectors));
  11947. return expr->clone(children);
  11948. }
  11949. case no_sequence:
  11950. return getSizetConstant(nextSequenceValue++);
  11951. case no_filter:
  11952. return transformWithinFilter(expr);
  11953. case no_executewhen:
  11954. return transformExecuteWhen(expr);
  11955. case no_funcdef:
  11956. {
  11957. HqlExprArray children;
  11958. if (transformChildren(expr, children))
  11959. return LINK(expr);
  11960. return createFunctionDefinition(expr->queryId(), children);
  11961. }
  11962. case no_debug_option_value:
  11963. {
  11964. if (!matchesConstantString(expr->queryChild(0), "targetClusterType", true))
  11965. return getDebugValueExpr(translator.wu(), expr);
  11966. break;
  11967. }
  11968. case no_loop:
  11969. {
  11970. OwnedHqlExpr transformed = NewHqlTransformer::createTransformed(expr);
  11971. IHqlExpression * loopCond = queryRealChild(transformed, 3);
  11972. if (loopCond)
  11973. {
  11974. //Create a firstCond attribute so that the condition for whether to execute the loop
  11975. //the first time will be efficiently optimized.
  11976. IHqlExpression * dataset = transformed->queryChild(0);
  11977. IHqlExpression * filter = queryRealChild(transformed, 2);
  11978. IHqlExpression * rowsid = transformed->queryAttribute(_rowsid_Atom);
  11979. IHqlExpression * selSeq = querySelSeq(transformed);
  11980. IHqlExpression * counter = queryAttributeChild(transformed, _countProject_Atom, 0);
  11981. OwnedHqlExpr left = createSelector(no_left, dataset, selSeq);
  11982. OwnedHqlExpr rowsExpr = createDataset(no_rows, LINK(left), LINK(rowsid));
  11983. OwnedHqlExpr initialLoopDataset = LINK(dataset);
  11984. if (filter)
  11985. {
  11986. //If there is a loop filter then the global condition is applied to dataset filtered by that.
  11987. OwnedHqlExpr mappedFilter = replaceSelector(filter, left, dataset);
  11988. initialLoopDataset.setown(createDataset(no_filter, initialLoopDataset.getClear(), LINK(mappedFilter)));
  11989. }
  11990. OwnedHqlExpr firstCond = quickFullReplaceExpression(loopCond, rowsExpr, initialLoopDataset);
  11991. if (counter)
  11992. {
  11993. //Whether to evaluate the 1st time round the loop requires COUNTER=1
  11994. OwnedHqlExpr one = createConstant(createIntValue(1, counter->getType()));
  11995. firstCond.setown(quickFullReplaceExpression(firstCond, counter, one));
  11996. }
  11997. return appendOwnedOperand(transformed, createExprAttribute(_loopFirst_Atom, firstCond.getClear()));
  11998. }
  11999. return transformed.getClear();
  12000. }
  12001. break;
  12002. }
  12003. ITypeInfo * type = expr->queryType();
  12004. bool checkType = false;
  12005. if (type)
  12006. {
  12007. switch (type->getTypeCode())
  12008. {
  12009. case type_pattern:
  12010. case type_rule:
  12011. checkType = true;
  12012. break;
  12013. }
  12014. }
  12015. unsigned max = expr->numChildren();
  12016. if ((max == 0) && !checkType)
  12017. return LINK(expr);
  12018. bool same = true;
  12019. HqlExprArray children;
  12020. children.ensure(max);
  12021. for (unsigned idx=0;idx<max;idx++)
  12022. {
  12023. IHqlExpression * child = expr->queryChild(idx);
  12024. IHqlExpression * tchild = transform(child);
  12025. children.append(*tchild);
  12026. if (child != tchild)
  12027. same = false;
  12028. }
  12029. if (checkType)
  12030. {
  12031. OwnedITypeInfo newType = transformType(type);
  12032. if (type != newType)
  12033. return createWrapper(op, newType, children);
  12034. }
  12035. if (!same)
  12036. return expr->clone(children);
  12037. return LINK(expr);
  12038. }
  12039. IHqlExpression * HqlTreeNormalizer::createTransformedSelector(IHqlExpression * expr)
  12040. {
  12041. throwUnexpected();
  12042. }
  12043. IHqlExpression * normalizeExpression(HqlCppTranslator & translator, IHqlExpression * expr)
  12044. {
  12045. HqlTreeNormalizer normalizer(translator);
  12046. return normalizer.transformRoot(expr);
  12047. }
  12048. void normalizeHqlTree(HqlCppTranslator & translator, HqlExprArray & exprs)
  12049. {
  12050. bool seenForceLocal;
  12051. bool seenLocalUpload;
  12052. {
  12053. //First iterate through the expressions and call queryLocationIndependent() to avoid nested transforms (which are less efficient)
  12054. // ForEachItemIn(iInit, exprs)
  12055. // queryLocationIndependent(&exprs.item(iInit));
  12056. HqlTreeNormalizer normalizer(translator);
  12057. HqlExprArray transformed;
  12058. normalizer.analyseArray(exprs, 0);
  12059. normalizer.transformRoot(exprs, transformed);
  12060. // logTreeStats(exprs);
  12061. // logTreeStats(transformed);
  12062. // DBGLOG("Before normalize %u unique expressions, after normalize %u unique expressions", getNumUniqueExpressions(exprs), getNumUniqueExpressions(transformed));
  12063. replaceArray(exprs, transformed);
  12064. seenForceLocal = normalizer.querySeenForceLocal();
  12065. seenLocalUpload = normalizer.querySeenLocalUpload();
  12066. }
  12067. if (translator.queryOptions().constantFoldPostNormalize)
  12068. {
  12069. HqlExprArray transformed;
  12070. quickFoldExpressions(transformed, exprs, NULL, 0);
  12071. replaceArray(exprs, transformed);
  12072. }
  12073. translator.traceExpressions("before scope tag", exprs);
  12074. {
  12075. HqlScopeTagger normalizer(translator.queryErrorProcessor(), translator.queryLocalOnWarningMapper());
  12076. normalizer.transformRoot(exprs);
  12077. }
  12078. translator.checkNormalized(exprs);
  12079. if (translator.queryOptions().normalizeLocations)
  12080. normalizeAnnotations(translator, exprs);
  12081. translator.traceExpressions("after scope tag", exprs);
  12082. {
  12083. DFSLayoutTransformer transformer(translator.queryErrorProcessor(), translator.queryCallback(), translator.queryOptions());
  12084. HqlExprArray transformed;
  12085. transformer.transformRoot(exprs, transformed);
  12086. replaceArray(exprs, transformed);
  12087. }
  12088. {
  12089. KeyedProjectTransformer transformer;
  12090. HqlExprArray transformed;
  12091. transformer.transformRoot(exprs, transformed);
  12092. replaceArray(exprs, transformed);
  12093. }
  12094. {
  12095. HqlLinkedChildRowTransformer transformer(translator.queryOptions().implicitLinkedChildRows);
  12096. HqlExprArray transformed;
  12097. transformer.transformArray(exprs, transformed);
  12098. replaceArray(exprs, transformed);
  12099. }
  12100. if (seenLocalUpload)
  12101. {
  12102. LocalUploadTransformer transformer(translator.wu());
  12103. HqlExprArray transformed;
  12104. transformer.transformRoot(exprs, transformed);
  12105. replaceArray(exprs, transformed);
  12106. }
  12107. if (seenForceLocal)
  12108. {
  12109. //Add ,local to all sources, so that count(x) inside local() is differentiated from a global count(x)
  12110. ForceLocalTransformer localizer(translator.getTargetClusterType());
  12111. HqlExprArray transformed;
  12112. localizer.transformRoot(exprs, transformed);
  12113. replaceArray(exprs, transformed);
  12114. }
  12115. #ifdef USE_SELSEQ_UID
  12116. if (translator.queryOptions().detectAmbiguousSelector || translator.queryOptions().allowAmbiguousSelector)
  12117. {
  12118. LeftRightSelectorNormalizer transformer(translator.queryOptions().allowAmbiguousSelector);
  12119. transformer.analyseArray(exprs, 0);
  12120. if (!transformer.containsAmbiguity())
  12121. {
  12122. HqlExprArray transformed;
  12123. transformer.transformRoot(exprs, transformed);
  12124. replaceArray(exprs, transformed);
  12125. }
  12126. }
  12127. #endif
  12128. if (false)
  12129. {
  12130. NestedSelectorNormalizer transformer;
  12131. transformer.analyseArray(exprs, 0);
  12132. if (transformer.requiresTransforming())
  12133. {
  12134. HqlExprArray transformed;
  12135. transformer.transformRoot(exprs, transformed);
  12136. replaceArray(exprs, transformed);
  12137. }
  12138. }
  12139. #if 0
  12140. if (seenIndex)
  12141. {
  12142. FilteredIndexOptimizer transformer(true, false);
  12143. HqlExprArray transformed;
  12144. transformer.transformRoot(exprs, transformed);
  12145. replaceArray(exprs, transformed);
  12146. }
  12147. #endif
  12148. #ifdef _DEBUG
  12149. //spotPotentialDuplicateCode(exprs);
  12150. #endif
  12151. }
  12152. IHqlExpression * normalizeHqlTree(HqlCppTranslator & translator, IHqlExpression * expr)
  12153. {
  12154. HqlExprArray exprs;
  12155. expr->unwindList(exprs, no_comma);
  12156. normalizeHqlTree(translator, exprs);
  12157. return createComma(exprs);
  12158. }
  12159. void hoistNestedCompound(HqlCppTranslator & translator, HqlExprArray & exprs)
  12160. {
  12161. if (containsCompound(exprs))
  12162. {
  12163. NestedCompoundTransformer normalizer(translator);
  12164. normalizer.analyseArray(exprs, 0);
  12165. HqlExprArray transformed;
  12166. normalizer.transformRoot(exprs, transformed);
  12167. replaceArray(exprs, transformed);
  12168. }
  12169. }
  12170. void hoistNestedCompound(HqlCppTranslator & translator, WorkflowArray & workflow)
  12171. {
  12172. ForEachItemIn(i, workflow)
  12173. hoistNestedCompound(translator, workflow.item(i).queryExprs());
  12174. }
  12175. //---------------------------------------------------------------------------
  12176. static IHqlExpression * substituteClusterSize(unsigned numNodes, IHqlExpression * expr, ICodegenContextCallback * ctxCallback, IWorkUnit * wu);
  12177. static HqlTransformerInfo clusterSubstitueTransformerInfo("ClusterSubstitueTransformer");
  12178. class ClusterSubstitueTransformer : public NewHqlTransformer
  12179. {
  12180. public:
  12181. ClusterSubstitueTransformer(unsigned size, ICodegenContextCallback * _ctxCallback, IWorkUnit * _wu)
  12182. : NewHqlTransformer(clusterSubstitueTransformerInfo)
  12183. {
  12184. ctxCallback = _ctxCallback;
  12185. wu = _wu;
  12186. OwnedHqlExpr clusterSizeExpr = createValue(no_clustersize, makeIntType(4, false));
  12187. if (size)
  12188. clusterSizeValue.setown(getSizetConstant(size));
  12189. setTransformed(clusterSizeExpr, clusterSizeValue);
  12190. }
  12191. protected:
  12192. IHqlExpression * createTransformed(IHqlExpression * expr)
  12193. {
  12194. if (expr->isConstant())
  12195. return LINK(expr);
  12196. switch (expr->getOperator())
  12197. {
  12198. case no_clustersize:
  12199. //Cope if CLUSTERSIZE is assigned to a named symbol
  12200. if (clusterSizeValue)
  12201. return LINK(clusterSizeValue);
  12202. break;
  12203. case no_cluster:
  12204. return createSubstitutedChild(expr, expr->queryChild(1));
  12205. case no_colon:
  12206. {
  12207. ForEachChild(i, expr)
  12208. {
  12209. IHqlExpression * child = expr->queryChild(i);
  12210. if (child->getOperator() == no_persist)
  12211. {
  12212. IHqlExpression * cluster = queryRealChild(child, 1);
  12213. if (cluster && !isBlankString(cluster))
  12214. return createSubstitutedChild(expr, cluster);
  12215. }
  12216. else if (child->getOperator() == no_global)
  12217. {
  12218. IHqlExpression * cluster = queryRealChild(child, 0);
  12219. if (cluster && !isBlankString(cluster))
  12220. return createSubstitutedChild(expr, cluster);
  12221. }
  12222. }
  12223. break;
  12224. }
  12225. }
  12226. return NewHqlTransformer::createTransformed(expr);
  12227. }
  12228. IHqlExpression * createSubstitutedChild(IHqlExpression * expr, IHqlExpression * cluster)
  12229. {
  12230. StringBuffer clusterText;
  12231. getStringValue(clusterText, cluster);
  12232. if (clusterText.length())
  12233. ctxCallback->noteCluster(clusterText.str());
  12234. #if 0
  12235. Owned<IConstWUClusterInfo> clusterInfo = wu->getClusterInfo(clusterText.str());
  12236. if (clusterInfo)
  12237. {
  12238. unsigned numNodes = clusterInfo->getSize();
  12239. if (numNodes == 0) numNodes = 1;
  12240. HqlExprArray args;
  12241. unwindChildren(args, expr);
  12242. args.replace(*substituteClusterSize(numNodes, &args.item(0), ctxCallback, wu), 0);
  12243. return expr->clone(args);
  12244. }
  12245. #endif
  12246. return LINK(expr);
  12247. }
  12248. protected:
  12249. ICodegenContextCallback * ctxCallback;
  12250. IWorkUnit * wu;
  12251. OwnedHqlExpr clusterSizeValue;
  12252. };
  12253. IHqlExpression * substituteClusterSize(unsigned numNodes, IHqlExpression * expr, ICodegenContextCallback * ctxCallback, IWorkUnit * wu)
  12254. {
  12255. ClusterSubstitueTransformer transformer(numNodes, ctxCallback, wu);
  12256. return transformer.transformRoot(expr);
  12257. }
  12258. void HqlCppTranslator::substituteClusterSize(HqlExprArray & exprs)
  12259. {
  12260. unsigned numNodes = options.specifiedClusterSize;
  12261. ClusterSubstitueTransformer transformer(numNodes, ctxCallback, wu());
  12262. HqlExprArray transformed;
  12263. ForEachItemIn(i, exprs)
  12264. transformed.append(*transformer.transformRoot(&exprs.item(i)));
  12265. replaceArray(exprs, transformed);
  12266. }
  12267. IHqlExpression * HqlCppTranslator::separateLibraries(IHqlExpression * query, HqlExprArray & internalLibraries)
  12268. {
  12269. HqlExprArray exprs;
  12270. query->unwindList(exprs, no_comma);
  12271. traceExpressions("before transform graph for generation", exprs);
  12272. //Remove any meta entries from the tree.
  12273. ForEachItemInRev(i, exprs)
  12274. if (exprs.item(i).getOperator() == no_setmeta)
  12275. exprs.remove(i);
  12276. processEmbeddedLibraries(exprs, internalLibraries, isLibraryScope(query));
  12277. return createComma(exprs);
  12278. }
  12279. void HqlCppTranslator::normalizeGraphForGeneration(HqlExprArray & exprs, HqlQueryContext & query)
  12280. {
  12281. if (isLibraryScope(query.expr))
  12282. outputLibrary->mapLogicalToImplementation(exprs, query.expr);
  12283. else
  12284. query.expr->unwindList(exprs, no_comma);
  12285. //Ensure the incoming query will be freed up when no longer used
  12286. query.expr.clear();
  12287. traceExpressions("before transform graph for generation", exprs);
  12288. //Don't change the engine if libraries are involved, otherwise things will get very confused.
  12289. {
  12290. expandDelayedFunctionCalls(&queryErrorProcessor(), exprs);
  12291. }
  12292. {
  12293. traceExpressions("before normalize", exprs);
  12294. normalizeHqlTree(*this, exprs);
  12295. }
  12296. if (wu()->getDebugValueBool("dumpIR", false))
  12297. EclIR::dbglogIR(exprs);
  12298. checkNormalized(exprs);
  12299. #ifdef PICK_ENGINE_EARLY
  12300. if (options.pickBestEngine)
  12301. {
  12302. cycle_t startCycles = get_cycles_now();
  12303. pickBestEngine(exprs);
  12304. if (options.timeTransforms)
  12305. noteFinishedTiming("compile:transform:pick engine", startCycles);
  12306. }
  12307. #endif
  12308. allocateSequenceNumbers(exprs); // Added to all expressions/output statements etc.
  12309. traceExpressions("allocate Sequence", exprs);
  12310. checkNormalized(exprs);
  12311. }
  12312. void HqlCppTranslator::applyGlobalOptimizations(HqlExprArray & exprs)
  12313. {
  12314. traceExpressions("begin transformGraphForGeneration", exprs);
  12315. checkNormalized(exprs);
  12316. {
  12317. substituteClusterSize(exprs);
  12318. }
  12319. {
  12320. HqlExprArray folded;
  12321. unsigned foldOptions = DEFAULT_FOLD_OPTIONS;
  12322. if (options.foldConstantDatasets) foldOptions |= HFOconstantdatasets;
  12323. if (options.percolateConstants) foldOptions |= HFOpercolateconstants;
  12324. if (options.percolateFilters) foldOptions |= HFOpercolatefilters;
  12325. if (options.optimizeMax) foldOptions |= HFOx_op_not_x;
  12326. if (options.globalFoldOptions != (unsigned)-1)
  12327. foldOptions = options.globalFoldOptions;
  12328. foldHqlExpression(queryErrorProcessor(), folded, exprs, foldOptions);
  12329. replaceArray(exprs, folded);
  12330. }
  12331. traceExpressions("after global fold", exprs);
  12332. checkNormalized(exprs);
  12333. if (options.globalOptimize)
  12334. {
  12335. HqlExprArray folded;
  12336. optimizeHqlExpression(queryErrorProcessor(), folded, exprs, HOOfold);
  12337. replaceArray(exprs, folded);
  12338. }
  12339. traceExpressions("alloc", exprs);
  12340. checkNormalized(exprs);
  12341. modifyOutputLocations(exprs);
  12342. }
  12343. void HqlCppTranslator::transformWorkflowItem(WorkflowItem & curWorkflow)
  12344. {
  12345. #ifdef USE_SELSEQ_UID
  12346. if (options.normalizeSelectorSequence)
  12347. {
  12348. LeftRightTransformer normalizer;
  12349. normalizer.process(curWorkflow.queryExprs());
  12350. //traceExpressions("after implicit alias", workflow);
  12351. }
  12352. #endif
  12353. if (queryOptions().createImplicitAliases)
  12354. {
  12355. ImplicitAliasTransformer normalizer;
  12356. normalizer.process(curWorkflow.queryExprs());
  12357. //traceExpressions("after implicit alias", workflow);
  12358. }
  12359. {
  12360. hoistNestedCompound(*this, curWorkflow.queryExprs());
  12361. }
  12362. if (options.optimizeNestedConditional)
  12363. {
  12364. optimizeNestedConditional(curWorkflow.queryExprs());
  12365. traceExpressions("nested", curWorkflow);
  12366. checkNormalized(curWorkflow);
  12367. }
  12368. checkNormalized(curWorkflow);
  12369. //sort(x)[n] -> topn(x, n)[]n, count(x)>n -> count(choosen(x,n+1)) > n and possibly others
  12370. {
  12371. optimizeActivities(curWorkflow.queryWfid(), curWorkflow.queryExprs(), !targetThor(), options.optimizeNonEmpty);
  12372. }
  12373. checkNormalized(curWorkflow);
  12374. //----------------------------- Transformations below this mark may have created globals so be very careful with hoisting ---------------------
  12375. {
  12376. migrateExprToNaturalLevel(curWorkflow, wu(), *this); // Ensure expressions are evaluated at the best level - e.g., counts moved to most appropriate level.
  12377. //transformToAliases(exprs);
  12378. traceExpressions("migrate", curWorkflow);
  12379. checkNormalized(curWorkflow);
  12380. }
  12381. {
  12382. markThorBoundaries(curWorkflow); // work out which engine is going to perform which operation.
  12383. traceExpressions("boundary", curWorkflow);
  12384. checkNormalized(curWorkflow);
  12385. }
  12386. if (options.optimizeGlobalProjects)
  12387. {
  12388. insertImplicitProjects(*this, curWorkflow.queryExprs());
  12389. traceExpressions("implicit", curWorkflow);
  12390. checkNormalized(curWorkflow);
  12391. }
  12392. normalizeResultFormat(curWorkflow, options);
  12393. traceExpressions("results", curWorkflow);
  12394. checkNormalized(curWorkflow);
  12395. optimizePersists(curWorkflow.queryExprs());
  12396. traceExpressions("per", curWorkflow);
  12397. checkNormalized(curWorkflow);
  12398. // flattenDatasets(workflow);
  12399. // traceExpressions("flatten", workflow);
  12400. {
  12401. mergeThorGraphs(curWorkflow, options.resourceConditionalActions, options.resourceSequential); // reduces number of graphs sent to thor
  12402. }
  12403. traceExpressions("merged", curWorkflow);
  12404. checkNormalized(curWorkflow);
  12405. if (queryOptions().normalizeLocations)
  12406. normalizeAnnotations(*this, curWorkflow.queryExprs());
  12407. spotGlobalCSE(curWorkflow); // spot CSE within those graphs, and create some more
  12408. checkNormalized(curWorkflow);
  12409. //expandGlobalDatasets(workflow, wu(), *this);
  12410. {
  12411. mergeThorGraphs(curWorkflow, options.resourceConditionalActions, options.resourceSequential);
  12412. }
  12413. checkNormalized(curWorkflow);
  12414. removeTrivialGraphs(curWorkflow);
  12415. checkNormalized(curWorkflow);
  12416. }
  12417. bool HqlCppTranslator::transformGraphForGeneration(HqlQueryContext & query, WorkflowArray & workflow)
  12418. {
  12419. HqlExprArray exprs;
  12420. normalizeGraphForGeneration(exprs, query);
  12421. if (options.generateLogicalGraph || options.generateLogicalGraphOnly)
  12422. {
  12423. LogicalGraphCreator creator(wu());
  12424. creator.createLogicalGraph(exprs);
  12425. if (options.generateLogicalGraphOnly)
  12426. return false;
  12427. curActivityId = creator.queryMaxActivityId();
  12428. }
  12429. applyGlobalOptimizations(exprs);
  12430. if (exprs.ordinality() == 0)
  12431. return false; // No action needed
  12432. ::extractWorkflow(*this, exprs, workflow);
  12433. traceExpressions("workflow", workflow);
  12434. checkNormalized(workflow);
  12435. if (outputLibrary && workflow.ordinality() > 1)
  12436. {
  12437. unsigned cnt = 0;
  12438. ForEachItemIn(i, workflow)
  12439. {
  12440. if (!workflow.item(i).isFunction())
  12441. cnt++;
  12442. }
  12443. if (cnt > 1)
  12444. {
  12445. SCMStringBuffer libraryName;
  12446. getOutputLibraryName(libraryName, wu());
  12447. throwError2(HQLERR_LibraryCannotContainWorkflow, libraryName.str(), "");
  12448. }
  12449. }
  12450. ForEachItemIn(i, workflow)
  12451. {
  12452. WorkflowItem & curWorkflow = workflow.item(i);
  12453. transformWorkflowItem(curWorkflow);
  12454. }
  12455. #ifndef PICK_ENGINE_EARLY
  12456. if (options.pickBestEngine)
  12457. {
  12458. cycle_t startCycles = get_cycles_now();
  12459. pickBestEngine(workflow);
  12460. if (options.timeTransforms)
  12461. noteFinishedTiming("compile:transform:pick engine", startCycles);
  12462. }
  12463. #endif
  12464. updateClusterType();
  12465. ForEachItemIn(i2, workflow)
  12466. {
  12467. WorkflowItem & curWorkflow = workflow.item(i2);
  12468. traceExpressions("before convert to logical", curWorkflow);
  12469. convertLogicalToActivities(curWorkflow); // e.g., merge disk reads, transform group, all to sort etc.
  12470. #ifndef _DEBUG
  12471. if (options.regressionTest)
  12472. #endif
  12473. {
  12474. cycle_t startCycles = get_cycles_now();
  12475. checkDependencyConsistency(curWorkflow.queryExprs());
  12476. if (options.timeTransforms)
  12477. noteFinishedTiming("compile:transform:check dependency", startCycles);
  12478. }
  12479. traceExpressions("end transformGraphForGeneration", curWorkflow);
  12480. checkNormalized(curWorkflow);
  12481. }
  12482. return true;
  12483. }
  12484. //---------------------------------------------------------------------------
  12485. /*
  12486. Different transformers:
  12487. merge: required if a child get removed or merged with a parent of a non-table dataset.
  12488. adding is not a problem (unless tables are inserted) because all refs to ds.x will remain valid.
  12489. but if tables can be deleted/modified then then it will cause problems, since ds.x needs to be translated to ds'.x within the scope
  12490. It is also ok if only scalars are transformed.
  12491. Transformer base merge dependants [should be]
  12492. filterExtractor simple N
  12493. resource Scoped (Y) complex - could possibly derive from merging...? why scoped?
  12494. HqlThorBoundary New N
  12495. HqlResult New N isConditional,insideThor,insideCondition
  12496. creation of getresult only done on scalars, so I think it is ok.
  12497. could possibly remove the scoping if count() etc. were tagged as outer level or not
  12498. ThorHql Merging Y Again scoped because of no_count etc.
  12499. CompoundSource New add* Either added, or non-tables(limits) are cloned, so no merging issues.
  12500. CompoundActivity Merging Y When limit merged into a dataset. [ need a new way? ]
  12501. Workflow New [inTransform] I think no for the same reason as above, or it adds. layers.
  12502. NewScopeMigrate New ? I think it might be ok, because doesn't modify any tables, only scalars
  12503. ThorCount New N no issues.
  12504. Cse New ? Probably, but ok, if only done on scalars.
  12505. HqlTreeNormalizer Scoped * I don't think it does any, but need to be careful none are introduced.
  12506. */
  12507. /*
  12508. NOTES:
  12509. Consider adding a hqlmeta.hpp that defines all the characteristics of an IHqlExpression node - e.g.,
  12510. is it constant, number of child files, text, what filenames does it generate? what does it read,
  12511. what results does it read/write.
  12512. Dependancy code:
  12513. 1. In the resourcer
  12514. 2. TableDependencies In hqlttcpp to stop reordering when not valid.
  12515. ?Is GetResultHash called on globals that haven't been calculated???
  12516. */