hqlresource.cpp 182 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402
  1. /*##############################################################################
  2. HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. ############################################################################## */
  13. #include "platform.h"
  14. #include "jlib.hpp"
  15. #include "hqlexpr.hpp"
  16. #include "hqlattr.hpp"
  17. #include "hqlmeta.hpp"
  18. #include "hqlutil.hpp"
  19. #include "hqlcpputil.hpp"
  20. #include "hqlthql.hpp"
  21. #include "hqlcatom.hpp"
  22. #include "hqlfold.hpp"
  23. #include "hqlcerrors.hpp"
  24. #include "hqltrans.ipp"
  25. #include "hqlpmap.hpp"
  26. #include "hqltcppc.ipp"
  27. #include "hqlttcpp.ipp"
  28. #include "hqlresource.ipp"
  29. #include "../../thorlcr/thorutil/thbufdef.hpp"
  30. #define MINIMAL_CHANGES
  31. #define MAX_INLINE_COMMON_COUNT 5
  32. //#define TRACE_RESOURCING
  33. //#define VERIFY_RESOURCING
  34. //#define SPOT_UNCONDITIONAL_CONDITIONS
  35. #define DEFAULT_LARGEMEM_BUFFER_SIZE (0x58000000) // ~ 1.4GB
  36. #define DEFAULT_MAX_SOCKETS 2000 // configurable by setting max_sockets in .ini
  37. #define DEFAULT_TOTAL_MEMORY ((1024*1024*1800)-DEFAULT_LARGEMEM_BUFFER_SIZE)
  38. #define FIXED_CLUSTER_SIZE 400
  39. #define MEM_Const_Minimal (1*1024*1024)
  40. #define DEFAULT_MAX_ACTIVITIES 100
  41. //=== The following provides information about how each kind of activity is resourced ====
  42. static void setHashResources(IHqlExpression * expr, CResources & resources, const CResourceOptions & options)
  43. {
  44. unsigned memneeded = MEM_Const_Minimal+resources.clusterSize*4*DISTRIBUTE_SINGLE_BUFFER_SIZE+DISTRIBUTE_PULL_BUFFER_SIZE;
  45. resources.set(RESslavememory, memneeded).set(REShashdist, 1);
  46. }
  47. //MORE: Use a single function to map an hqlexpression to an activity kind.
  48. void getResources(IHqlExpression * expr, CResources & resources, const CResourceOptions & options)
  49. {
  50. //MORE: What effect should a child query have? Definitely the following, but what about other resourcing?
  51. if (options.isChildQuery || queryHint(expr, lightweightAtom))
  52. {
  53. resources.setLightweight();
  54. return;
  55. }
  56. bool isLocal = isLocalActivity(expr);
  57. bool isGrouped = isGroupedActivity(expr);
  58. switch (expr->getOperator())
  59. {
  60. case no_join:
  61. case no_selfjoin:
  62. case no_denormalize:
  63. case no_denormalizegroup:
  64. if (isKeyedJoin(expr) || expr->hasAttribute(_lightweight_Atom))
  65. resources.setLightweight();
  66. else if (expr->hasAttribute(lookupAtom))
  67. {
  68. if (expr->hasAttribute(fewAtom))
  69. {
  70. resources.setLightweight().set(RESslavememory, MEM_Const_Minimal+LOOKUPJOINL_SMART_BUFFER_SIZE);
  71. resources.setManyToMasterSockets(1);
  72. }
  73. else
  74. {
  75. resources.setHeavyweight().set(RESslavememory, MEM_Const_Minimal+LOOKUPJOINL_SMART_BUFFER_SIZE);
  76. resources.setManyToMasterSockets(1);
  77. }
  78. }
  79. else if (expr->hasAttribute(smartAtom))
  80. {
  81. resources.setHeavyweight();
  82. setHashResources(expr, resources, options);
  83. }
  84. else if (expr->hasAttribute(hashAtom))
  85. {
  86. resources.setHeavyweight();
  87. setHashResources(expr, resources, options);
  88. }
  89. else
  90. {
  91. resources.setHeavyweight().set(RESslavememory, MEM_Const_Minimal+SORT_BUFFER_TOTAL+JOINR_SMART_BUFFER_SIZE);
  92. if (!isLocal)
  93. {
  94. #ifndef SORT_USING_MP
  95. resources.setManyToManySockets(2);
  96. #endif
  97. }
  98. }
  99. break;
  100. case no_dedup:
  101. if (isGrouped || (!expr->hasAttribute(allAtom) && !expr->hasAttribute(hashAtom)))
  102. {
  103. resources.setLightweight();
  104. if (!isGrouped && !isLocal)
  105. {
  106. resources.set(RESslavememory, MEM_Const_Minimal+DEDUP_SMART_BUFFER_SIZE);
  107. resources.setManyToMasterSockets(1);
  108. }
  109. }
  110. else if (isLocal)
  111. {
  112. resources.setHeavyweight().set(RESslavememory, MEM_Const_Minimal+DEDUP_SMART_BUFFER_SIZE);
  113. //This can't be right....
  114. //resources.setManyToMasterSockets(1);
  115. }
  116. else
  117. {
  118. //hash dedup.
  119. resources.setHeavyweight();
  120. setHashResources(expr, resources, options);
  121. }
  122. break;
  123. case no_rollup:
  124. resources.setLightweight();
  125. if (!isGrouped && !isLocal)
  126. {
  127. resources.set(RESslavememory, MEM_Const_Minimal+DEDUP_SMART_BUFFER_SIZE);
  128. //MORE: Is this still correct?
  129. resources.setManyToMasterSockets(1);
  130. }
  131. break;
  132. case no_distribute:
  133. case no_keyeddistribute:
  134. resources.setLightweight();
  135. setHashResources(expr, resources, options);
  136. break;
  137. case no_subsort:
  138. if (expr->hasAttribute(manyAtom))
  139. resources.setHeavyweight();
  140. else
  141. resources.setLightweight();
  142. break;
  143. case no_sort:
  144. if (isGrouped)
  145. {
  146. if (expr->hasAttribute(manyAtom))
  147. resources.setHeavyweight();
  148. else
  149. resources.setLightweight();
  150. }
  151. else if (expr->hasAttribute(fewAtom) && isLocal)
  152. resources.setLightweight();
  153. else if (isLocal)
  154. resources.setHeavyweight();
  155. else
  156. {
  157. resources.setHeavyweight();
  158. #ifndef SORT_USING_MP
  159. resources.setManyToManySockets(2);
  160. #endif
  161. }
  162. break;
  163. case no_topn:
  164. resources.setLightweight();
  165. break;
  166. case no_pipe:
  167. //surely it should be something like this.
  168. resources.setLightweight().set(RESslavesocket, 1);
  169. break;
  170. case no_table:
  171. {
  172. IHqlExpression * mode = expr->queryChild(2);
  173. if (mode && mode->getOperator() == no_pipe)
  174. {
  175. resources.setLightweight().set(RESslavesocket, 1);
  176. }
  177. else
  178. {
  179. resources.setLightweight();
  180. if (expr->hasAttribute(_workflowPersist_Atom) && expr->hasAttribute(distributedAtom))
  181. setHashResources(expr, resources, options); // may require a hash distribute
  182. }
  183. break;
  184. }
  185. case no_output:
  186. {
  187. IHqlExpression * filename = expr->queryChild(1);
  188. if (expr->hasAttribute(_spill_Atom))
  189. {
  190. //resources.setLightweight(); // assume no resources(!)
  191. }
  192. else if (filename && filename->getOperator() == no_pipe)
  193. {
  194. resources.setLightweight().set(RESslavesocket, 1);
  195. }
  196. else if (filename && !filename->isAttribute())
  197. {
  198. resources.setLightweight();
  199. }
  200. else
  201. {
  202. resources.setLightweight().set(RESslavememory, WORKUNITWRITE_SMART_BUFFER_SIZE);
  203. }
  204. break;
  205. }
  206. case no_distribution:
  207. resources.setLightweight().set(RESmastersocket, 16).set(RESslavesocket, 1);
  208. break;
  209. case no_aggregate:
  210. case no_newaggregate:
  211. {
  212. IHqlExpression * grouping = queryRealChild(expr, 3);
  213. if (grouping)
  214. {
  215. //Is this really correct???
  216. resources.setLightweight();
  217. setHashResources(expr, resources, options);
  218. }
  219. else
  220. {
  221. resources.setLightweight();
  222. //if (!isGrouped)
  223. // resources.set(RESmastersocket, 16).set(RESslavesocket, 1);
  224. }
  225. }
  226. break;
  227. case no_hqlproject:
  228. resources.setLightweight();
  229. //Add a flag onto count project to indicate it is a different variety.
  230. if (expr->hasAttribute(_countProject_Atom) && !isLocal)
  231. resources.set(RESslavememory, COUNTPROJECT_SMART_BUFFER_SIZE);
  232. break;
  233. case no_enth:
  234. resources.setLightweight();
  235. if (!isLocal)
  236. resources.set(RESslavememory, CHOOSESETS_SMART_BUFFER_SIZE);
  237. break;
  238. case no_metaactivity:
  239. if (expr->hasAttribute(pullAtom))
  240. resources.setLightweight().set(RESslavememory, PULL_SMART_BUFFER_SIZE);
  241. break;
  242. case no_setresult:
  243. case no_extractresult:
  244. case no_outputscalar:
  245. resources.setLightweight();//.set(RESmastersocket, 1).set(RESslavesocket, 1);
  246. break;
  247. case no_choosesets:
  248. resources.setLightweight();
  249. if (!isLocal || expr->hasAttribute(enthAtom) || expr->hasAttribute(lastAtom))
  250. resources.set(RESslavememory, CHOOSESETS_SMART_BUFFER_SIZE);
  251. break;
  252. case no_iterate:
  253. resources.setLightweight();
  254. if (!isGrouped && !isLocal)
  255. resources.setManyToMasterSockets(1).set(RESslavememory, ITERATE_SMART_BUFFER_SIZE);
  256. break;
  257. case no_choosen:
  258. resources.setLightweight().set(RESslavememory, FIRSTN_SMART_BUFFER_SIZE);
  259. break;
  260. case no_spill:
  261. case no_spillgraphresult:
  262. //assumed to take no resources;
  263. break;
  264. case no_addfiles:
  265. case no_merge:
  266. {
  267. resources.setLightweight();
  268. unsigned bufSize = FUNNEL_PERINPUT_BUFF_SIZE*expr->numChildren();
  269. if (bufSize < FUNNEL_MIN_BUFF_SIZE) bufSize = FUNNEL_MIN_BUFF_SIZE;
  270. resources.set(RESslavememory, MEM_Const_Minimal+bufSize);
  271. break;
  272. }
  273. case no_compound:
  274. //MORE: Should really be the total resources for the lhs... Really needs more thought.
  275. break;
  276. case no_libraryselect:
  277. //Do not allocate any resources for this, we don't want it to cause a spill under any circumstances
  278. break;
  279. case no_split:
  280. //Should really be included in the cost....
  281. default:
  282. resources.setLightweight();
  283. break;
  284. }
  285. }
  286. CResources & CResources::setLightweight()
  287. {
  288. return set(RESslavememory, 0x10000).set(RESactivities, 1);
  289. }
  290. CResources & CResources::setHeavyweight()
  291. {
  292. return set(RESslavememory, 0x100000).set(RESheavy, 1).set(RESactivities, 1);
  293. }
  294. //-------------------------------------------------------------------------------------------
  295. const char * queryResourceName(ResourceType kind)
  296. {
  297. switch (kind)
  298. {
  299. case RESslavememory: return "Slave Memory";
  300. case RESslavesocket: return "Slave Sockets";
  301. case RESmastermemory: return "Master Memory";
  302. case RESmastersocket: return "Master Sockets";
  303. case REShashdist: return "Hash Distributes";
  304. case RESheavy: return "Heavyweight";
  305. case RESactivities: return "Activities";
  306. }
  307. return "Unknown";
  308. }
  309. inline ResourcerInfo * queryResourceInfo(IHqlExpression * expr) { return (ResourcerInfo *)expr->queryBody()->queryTransformExtra(); }
  310. inline bool isResourcedActivity(IHqlExpression * expr)
  311. {
  312. if (!expr)
  313. return false;
  314. ResourcerInfo * extra = queryResourceInfo(expr);
  315. return extra && extra->isResourcedActivity();
  316. }
  317. bool isWorthForcingHoist(IHqlExpression * expr)
  318. {
  319. loop
  320. {
  321. switch (expr->getOperator())
  322. {
  323. case no_selectnth:
  324. case no_filter:
  325. case no_newaggregate:
  326. if (isResourcedActivity(expr->queryChild(0)))
  327. return true;
  328. break;
  329. default:
  330. return false;
  331. }
  332. expr = expr->queryChild(0);
  333. }
  334. }
  335. void CResources::add(const CResources & other)
  336. {
  337. for (unsigned i = 0; i < RESmax; i++)
  338. resource[i] += other.resource[i];
  339. }
  340. bool CResources::addExceeds(const CResources & other, const CResources & limit) const
  341. {
  342. for (unsigned i = 0; i < RESmax; i++)
  343. {
  344. if (resource[i] + other.resource[i] > limit.resource[i])
  345. {
  346. //DBGLOG("Cannot merge because limit for %s exceeded (%d cf %d)", queryResourceName((ResourceType)i), resource[i] + other.resource[i], limit.resource[i]);
  347. return true;
  348. }
  349. }
  350. return false;
  351. }
  352. StringBuffer & CResources::getExceedsReason(StringBuffer & reasonText, const CResources & other, const CResources & limit) const
  353. {
  354. bool first = true;
  355. for (unsigned i = 0; i < RESmax; i++)
  356. {
  357. if (resource[i] + other.resource[i] > limit.resource[i])
  358. {
  359. if (!first) reasonText.append(", ");
  360. first = false;
  361. reasonText.appendf("%s (%d>%d)", queryResourceName((ResourceType)i), resource[i] + other.resource[i], limit.resource[i]);
  362. }
  363. }
  364. return reasonText;
  365. }
  366. bool CResources::exceeds(const CResources & limit) const
  367. {
  368. for (unsigned i = 0; i < RESmax; i++)
  369. {
  370. if (resource[i] > limit.resource[i])
  371. return true;
  372. }
  373. return false;
  374. }
  375. void CResources::maximize(const CResources & other)
  376. {
  377. for (unsigned i = 0; i < RESmax; i++)
  378. {
  379. if (resource[i] < other.resource[i])
  380. resource[i] = other.resource[i];
  381. }
  382. }
  383. CResources & CResources::setManyToMasterSockets(unsigned numSockets)
  384. {
  385. set(RESslavesocket, numSockets);
  386. set(RESmastersocket, numSockets * clusterSize);
  387. return *this;
  388. }
  389. CResources & CResources::setManyToManySockets(unsigned numSockets)
  390. {
  391. return set(RESslavesocket, numSockets * clusterSize);
  392. }
  393. void CResources::sub(const CResources & other)
  394. {
  395. for (unsigned i = 0; i < RESmax; i++)
  396. resource[i] -= other.resource[i];
  397. }
  398. //===========================================================================
  399. inline bool isAffectedByResourcing(IHqlExpression * expr)
  400. {
  401. switch (expr->getOperator())
  402. {
  403. case no_record:
  404. case no_constant:
  405. case no_attr:
  406. return false;
  407. }
  408. return true;
  409. }
  410. bool isSimpleAggregateResult(IHqlExpression * expr)
  411. {
  412. //MORE: no_extractresult is really what is meant
  413. if (expr->getOperator() != no_extractresult)
  414. return false;
  415. IHqlExpression * value = expr->queryChild(0);
  416. if (value->getOperator() != no_datasetfromrow)
  417. return false;
  418. //MORE: This currently doesn't hoist selects from nested records, but not sure there is a syntax to do that either.
  419. IHqlExpression * ds = value->queryChild(0);
  420. if (!isSelectFirstRow(ds))
  421. return false;
  422. ds = ds->queryChild(0);
  423. if (ds->getOperator() != no_newaggregate)
  424. return false;
  425. return true;
  426. }
  427. bool lightweightAndReducesDatasetSize(IHqlExpression * expr)
  428. {
  429. switch (expr->getOperator())
  430. {
  431. case no_hqlproject:
  432. case no_newusertable:
  433. return reducesRowSize(expr);
  434. case no_dedup:
  435. if (isGroupedActivity(expr) || (!expr->hasAttribute(allAtom) && !expr->hasAttribute(hashAtom)))
  436. return true;
  437. break;
  438. case no_aggregate:
  439. case no_newaggregate:
  440. {
  441. IHqlExpression * grouping = queryRealChild(expr, 3);
  442. if (grouping)
  443. return false;
  444. return true;
  445. }
  446. case no_rollupgroup:
  447. case no_rollup:
  448. case no_choosen:
  449. case no_choosesets:
  450. case no_enth:
  451. case no_sample:
  452. case no_filter:
  453. case no_limit:
  454. case no_filtergroup:
  455. return true;
  456. case no_group:
  457. //removing grouping will reduce size of the spill file.
  458. if (!isGrouped(expr))
  459. return true;
  460. break;
  461. }
  462. return false;
  463. }
  464. bool heavyweightAndReducesSizeOrSkew(IHqlExpression * expr)
  465. {
  466. switch (expr->getOperator())
  467. {
  468. case no_aggregate:
  469. case no_newaggregate:
  470. {
  471. //more; hash aggregate?
  472. break;
  473. }
  474. case no_distribute:
  475. return true;
  476. }
  477. return false;
  478. }
  479. //---------------------------------------------------------------------------
  480. IHqlExpression * CResourceOptions::createSpillName(bool isGraphResult)
  481. {
  482. if (isGraphResult)
  483. return getSizetConstant(nextResult++);
  484. StringBuffer s;
  485. s.append("~spill::");
  486. getUniqueId(s);
  487. return createConstant(s.str());
  488. }
  489. //---------------------------------------------------------------------------
  490. IHqlExpression * appendUniqueAttr(IHqlExpression * expr)
  491. {
  492. return replaceOwnedAttribute(expr, createUniqueId());
  493. }
  494. bool queryAddUniqueToActivity(IHqlExpression * expr)
  495. {
  496. if (!isSourceActivity(expr))
  497. return false;
  498. switch (expr->getOperator())
  499. {
  500. case no_workunit_dataset:
  501. case no_getgraphresult:
  502. case no_getgraphloopresult:
  503. case no_xmlproject:
  504. case no_datasetfromrow:
  505. case no_datasetfromdictionary:
  506. case no_rows:
  507. case no_allnodes:
  508. case no_thisnode:
  509. case no_select: // it can get lost, and that then causes inconsistent trees.
  510. return false;
  511. }
  512. return true;
  513. }
  514. //---------------------------------------------------------------------------
  515. ResourceGraphLink::ResourceGraphLink(ResourceGraphInfo * _sourceGraph, IHqlExpression * _sourceNode, ResourceGraphInfo * _sinkGraph, IHqlExpression * _sinkNode, LinkKind _linkKind)
  516. {
  517. assertex(_sourceGraph);
  518. sourceGraph.set(_sourceGraph);
  519. sourceNode.set(_sourceNode);
  520. sinkGraph.set(_sinkGraph);
  521. sinkNode.set(_sinkNode);
  522. assertex(!sinkGraph || sinkNode);
  523. linkKind = _linkKind;
  524. trace("create");
  525. }
  526. ResourceGraphLink::~ResourceGraphLink()
  527. {
  528. trace("delete");
  529. }
  530. void ResourceGraphLink::changeSourceGraph(ResourceGraphInfo * newGraph)
  531. {
  532. sourceGraph->sinks.zap(*this);
  533. sourceGraph.set(newGraph);
  534. newGraph->sinks.append(*this);
  535. trace("change source");
  536. }
  537. void ResourceGraphLink::changeSinkGraph(ResourceGraphInfo * newGraph)
  538. {
  539. sinkGraph->sources.zap(*this);
  540. sinkGraph.set(newGraph);
  541. newGraph->sources.append(*this);
  542. trace("change sink");
  543. }
  544. bool ResourceGraphLink::isRedundantLink()
  545. {
  546. ResourcerInfo * info = queryResourceInfo(sourceNode);
  547. return info->expandRatherThanSpill(false);
  548. }
  549. void ResourceGraphLink::trace(const char * name)
  550. {
  551. #ifdef TRACE_RESOURCING
  552. PrintLog("%s: %lx source(%lx,%lx) sink(%lx,%lx) %s", name, this, sourceGraph.get(), sourceNode->queryBody(), sinkGraph.get(), sinkNode ? sinkNode->queryBody() : NULL,
  553. linkKind == SequenceLink ? "sequence" : "");
  554. #endif
  555. }
  556. ResourceGraphDependencyLink::ResourceGraphDependencyLink(ResourceGraphInfo * _sourceGraph, IHqlExpression * _sourceNode, ResourceGraphInfo * _sinkGraph, IHqlExpression * _sinkNode)
  557. : ResourceGraphLink(_sourceGraph, _sourceNode, _sinkGraph, _sinkNode, UnconditionalLink)
  558. {
  559. }
  560. void ResourceGraphDependencyLink::changeSourceGraph(ResourceGraphInfo * newGraph)
  561. {
  562. sourceGraph.set(newGraph);
  563. trace("change source");
  564. }
  565. void ResourceGraphDependencyLink::changeSinkGraph(ResourceGraphInfo * newGraph)
  566. {
  567. sinkGraph.set(newGraph);
  568. newGraph->dependsOn.append(*this);
  569. trace("change sink");
  570. }
  571. //---------------------------------------------------------------------------
  572. ResourceGraphInfo::ResourceGraphInfo(CResourceOptions * _options) : resources(_options->clusterSize)
  573. {
  574. options = _options;
  575. depth = 0;
  576. depthSequence = -1;
  577. beenResourced = false;
  578. isUnconditional = false;
  579. mergedConditionSource = false;
  580. hasConditionSource = false;
  581. hasSequentialSource = false;
  582. hasRootActivity = false;
  583. isDead = false;
  584. startedGeneratingResourced = false;
  585. inheritedExpandedDependencies = false;
  586. cachedDependent.other = NULL;
  587. }
  588. ResourceGraphInfo::~ResourceGraphInfo()
  589. {
  590. }
  591. bool ResourceGraphInfo::addCondition(IHqlExpression * condition)
  592. {
  593. if (conditions.find(*condition) == NotFound)
  594. {
  595. conditions.append(*LINK(condition));
  596. #ifdef SPOT_UNCONDITIONAL_CONDITIONS
  597. IAtom * name = condition->queryName();
  598. IAtom * invName = NULL;
  599. if (name == trueAtom)
  600. invName = falseAtom;
  601. else if (name == falseAtom)
  602. invName = trueAtom;
  603. if (invName)
  604. {
  605. IHqlExpression * parent = condition->queryChild(1);
  606. OwnedHqlExpr invTag = createAttribute(invName, LINK(condition->queryChild(0)), LINK(parent));
  607. if (conditions.find(*invTag) != NotFound)
  608. {
  609. if (!parent)
  610. return true;
  611. return addCondition(parent);
  612. }
  613. }
  614. #endif
  615. }
  616. return false;
  617. }
  618. bool ResourceGraphInfo::isSharedInput(IHqlExpression * expr)
  619. {
  620. IHqlExpression * body = expr->queryBody();
  621. if (unbalancedExternalSources.contains(*body))
  622. return false;
  623. if (queryResourceInfo(expr)->expandRatherThanSplit())
  624. return false;
  625. unsigned numUses = 0;
  626. ForEachItemIn(i, balancedExternalSources)
  627. {
  628. if (body == &balancedExternalSources.item(i))
  629. numUses++;
  630. }
  631. //NumUses could be zero if an input should be expanded, and that input is shared by another graph which also expands
  632. //the input. E.g. project(meta(diskread).
  633. return numUses > 1;
  634. }
  635. void ResourceGraphInfo::addSharedInput(IHqlExpression * expr, IHqlExpression * mapped)
  636. {
  637. sharedInputs.append(*LINK(expr));
  638. sharedInputs.append(*LINK(mapped));
  639. }
  640. IHqlExpression * ResourceGraphInfo::queryMappedSharedInput(IHqlExpression * expr)
  641. {
  642. unsigned max = sharedInputs.ordinality();
  643. for (unsigned i=0; i < max; i+= 2)
  644. {
  645. if (expr == &sharedInputs.item(i))
  646. return &sharedInputs.item(i+1);
  647. }
  648. return NULL;
  649. }
  650. bool ResourceGraphInfo::allocateResources(const CResources & value, const CResources & limit)
  651. {
  652. if (resources.addExceeds(value, limit))
  653. return false;
  654. resources.add(value);
  655. return true;
  656. }
  657. bool ResourceGraphInfo::canMergeActionAsSibling(bool sequential) const
  658. {
  659. //sequential requires root actions in separate graphs
  660. if (sequential && hasRootActivity)
  661. return false;
  662. //Dependent actions need to be in a separate graph if this flag is set.
  663. if (options->actionLinkInNewGraph && !hasRootActivity)
  664. return false;
  665. return true;
  666. }
  667. bool ResourceGraphInfo::containsActiveSinks() const
  668. {
  669. ForEachItemIn(idx, sinks)
  670. {
  671. ResourcerInfo * info = queryResourceInfo(sinks.item(idx).sourceNode);
  672. if (!info->expandRatherThanSpill(false))
  673. return true;
  674. }
  675. return false;
  676. }
  677. bool ResourceGraphInfo::containsActionSink() const
  678. {
  679. ForEachItemIn(idx, sinks)
  680. {
  681. IHqlExpression * sink = sinks.item(idx).sourceNode;
  682. if (sink->isAction())
  683. return true;
  684. }
  685. return false;
  686. }
  687. void ResourceGraphInfo::display()
  688. {
  689. StringBuffer s;
  690. s.appendf("graph %p src(", this);
  691. ForEachItemIn(idxs, sources)
  692. s.appendf("%p ", sources.item(idxs).sourceGraph.get());
  693. s.append(") dep(");
  694. ForEachItemIn(idxd, dependsOn)
  695. s.appendf("%p ", dependsOn.item(idxd).sourceGraph.get());
  696. s.append(")");
  697. if (isUnconditional)
  698. s.append(" <unconditional>");
  699. DBGLOG("%s", s.str());
  700. }
  701. void ResourceGraphInfo::getMergeFailReason(StringBuffer & reasonText, ResourceGraphInfo * otherGraph, const CResources & limit)
  702. {
  703. resources.getExceedsReason(reasonText, otherGraph->resources, limit);
  704. }
  705. unsigned ResourceGraphInfo::getDepth()
  706. {
  707. //If no graphs have been merged since this was last called it is still valid.
  708. if (depthSequence == options->state.updateSequence)
  709. return depth;
  710. depthSequence = options->state.updateSequence;
  711. depth = 0;
  712. ForEachItemIn(idx, dependsOn)
  713. {
  714. ResourceGraphInfo * source = dependsOn.item(idx).sourceGraph;
  715. if (source->getDepth() >= depth)
  716. depth = source->getDepth() + 1;
  717. }
  718. ForEachItemIn(idx2, sources)
  719. {
  720. ResourceGraphInfo * source = sources.item(idx2).sourceGraph;
  721. if (source->getDepth() >= depth)
  722. depth = source->getDepth() + 1;
  723. }
  724. return depth;
  725. }
  726. bool ResourceGraphInfo::hasSameConditions(ResourceGraphInfo & other)
  727. {
  728. if (conditions.ordinality() != other.conditions.ordinality())
  729. return false;
  730. ForEachItemIn(i, conditions)
  731. if (other.conditions.find(conditions.item(i)) == NotFound)
  732. return false;
  733. return true;
  734. }
  735. bool ResourceGraphInfo::evalDependentOn(ResourceGraphInfo & other, bool ignoreSources)
  736. {
  737. ForEachItemIn(idx, dependsOn)
  738. {
  739. ResourceGraphInfo * cur = dependsOn.item(idx).sourceGraph;
  740. if (cur == &other)
  741. return true;
  742. if (cur->isDependentOn(other, false))
  743. return true;
  744. }
  745. ForEachItemIn(idx2, sources)
  746. {
  747. ResourceGraphInfo * cur = sources.item(idx2).sourceGraph;
  748. if ((cur == &other) && !ignoreSources)
  749. return true;
  750. if (cur->isDependentOn(other, false))
  751. return true;
  752. }
  753. return false;
  754. }
  755. bool ResourceGraphInfo::isDependentOn(ResourceGraphInfo & other, bool ignoreSources)
  756. {
  757. //Cache the last query so that traversal doesn't convert a graph into a tree walk
  758. if ((cachedDependent.other == &other) && (cachedDependent.ignoreSources == ignoreSources) &&
  759. (cachedDependent.updateSequence == options->state.updateSequence))
  760. return cachedDependent.value;
  761. if (getDepth() <= other.getDepth())
  762. return false;
  763. cachedDependent.other = &other;
  764. cachedDependent.ignoreSources = ignoreSources;
  765. cachedDependent.updateSequence = options->state.updateSequence;
  766. cachedDependent.value = evalDependentOn(other, ignoreSources);
  767. return cachedDependent.value;
  768. }
  769. bool ResourceGraphInfo::isVeryCheap()
  770. {
  771. if (sinks.ordinality() != 1)
  772. return false;
  773. IHqlExpression * sourceNode = sinks.item(0).sourceNode;
  774. if (isSimpleAggregateResult(sourceNode))
  775. return true;
  776. // Not sure about the following....
  777. // if (sourceNode->getOperator() == no_setresult)
  778. // return true;
  779. //Could be other examples...
  780. return false;
  781. }
  782. bool ResourceGraphInfo::mergeInSource(ResourceGraphInfo & other, const CResources & limit)
  783. {
  784. bool mergeConditions = false;
  785. if (!isUnconditional)
  786. {
  787. //if it is conditional and it is very cheap then it is still more efficient to merge
  788. //rather than read from a spill file.
  789. if (other.isUnconditional || !hasSameConditions(other))
  790. {
  791. if ((hasConditionSource || !isVeryCheap()) && (other.sinks.ordinality() != 1))
  792. return false;
  793. mergeConditions = true;
  794. }
  795. }
  796. if (isDependentOn(other, true))
  797. return false;
  798. if (!options->canSplit())
  799. {
  800. //Don't merge two graphs that will cause a splitter to be created
  801. //Either already used internally, or an output will be merged twice
  802. HqlExprArray mergeNodes;
  803. ForEachItemIn(idx, sources)
  804. {
  805. ResourceGraphLink & cur = sources.item(idx);
  806. if (cur.sourceGraph == &other)
  807. {
  808. IHqlExpression * sourceNode = cur.sourceNode->queryBody();
  809. ResourcerInfo * info = queryResourceInfo(sourceNode);
  810. if ((info->numInternalUses() != 0) || (mergeNodes.find(*sourceNode) != NotFound) || info->preventMerge())
  811. return false;
  812. mergeNodes.append(*LINK(sourceNode));
  813. }
  814. }
  815. }
  816. if (options->checkResources() && !allocateResources(other.resources, limit))
  817. return false;
  818. if (hasSequentialSource && other.hasSequentialSource)
  819. return false;
  820. mergeGraph(other, mergeConditions);
  821. return true;
  822. }
  823. void ResourceGraphInfo::mergeGraph(ResourceGraphInfo & other, bool mergeConditions)
  824. {
  825. #ifdef TRACE_RESOURCING
  826. DBGLOG("Merging%s source into%s sink", other.isUnconditional ? "" : " conditional", isUnconditional ? "" : " conditional");
  827. other.display();
  828. display();
  829. PrintLog("Merge %p into %p", &other, this);
  830. #endif
  831. if (other.hasConditionSource)
  832. hasConditionSource = true;
  833. if (other.hasSequentialSource)
  834. {
  835. assertex(!hasSequentialSource);
  836. hasSequentialSource = true;
  837. }
  838. //Recalculate the dependents, because sources of the source merged in may no longer be indirect
  839. //although they may be via another path.
  840. options->noteGraphsChanged();
  841. //If was very cheap and merged into an unconditional graph, make sure this is now tagged as
  842. //unconditional...
  843. if (other.isUnconditional)
  844. isUnconditional = true;
  845. //We need to stop spills being an arm of a conditional branch - otherwise they won't get executed.
  846. //so see if we have merged any conditional branches in
  847. if (other.mergedConditionSource)
  848. mergedConditionSource = true;
  849. if (mergeConditions)
  850. {
  851. //Replace conditions with those of parent. Only called when a very simple graph is
  852. //merged in, so replace conditions instead of merging
  853. conditions.kill();
  854. ForEachItemIn(i, other.conditions)
  855. conditions.append(OLINK(other.conditions.item(i)));
  856. }
  857. //sources and sinks are updated elsewhere...
  858. }
  859. bool ResourceGraphInfo::mergeInSibling(ResourceGraphInfo & other, const CResources & limit)
  860. {
  861. if ((!isUnconditional || !other.isUnconditional) && !hasSameConditions(other))
  862. return false;
  863. if (hasSequentialSource && other.hasSequentialSource)
  864. return false;
  865. if (isDependentOn(other, false) || other.isDependentOn(*this, false))
  866. return false;
  867. if (options->checkResources() && !allocateResources(other.resources, limit))
  868. return false;
  869. mergeGraph(other, false);
  870. return true;
  871. }
  872. void ResourceGraphInfo::removeResources(const CResources & value)
  873. {
  874. resources.sub(value);
  875. }
  876. //---------------------------------------------------------------------------
  877. unsigned ChildDependentArray::findOriginal(IHqlExpression * expr)
  878. {
  879. ForEachItem(i)
  880. {
  881. if (item(i).original == expr)
  882. return i;
  883. }
  884. return NotFound;
  885. }
  886. //---------------------------------------------------------------------------
  887. static void appendCloneAttribute(HqlExprArray & args, IHqlExpression * expr, IAtom * name)
  888. {
  889. IHqlExpression * prop = expr->queryAttribute(name);
  890. if (prop)
  891. args.append(*LINK(prop));
  892. }
  893. ResourcerInfo::ResourcerInfo(IHqlExpression * _original, CResourceOptions * _options)
  894. {
  895. original.set(_original);
  896. numUses = 0;
  897. numExternalUses = 0;
  898. gatheredDependencies = false;
  899. containsActivity = false;
  900. isActivity = false;
  901. isRootActivity = false;
  902. transformed = NULL;
  903. conditionSourceCount = 0;
  904. pathToExpr = PathUnknown;
  905. outputToUseForSpill = NULL;
  906. isAlreadyInScope = false;
  907. isSpillPoint = false;
  908. options = _options;
  909. balanced = true;
  910. currentSource = 0;
  911. linkedFromChild = false;
  912. forceHoist = false;
  913. neverSplit = false;
  914. isConditionalFilter = false;
  915. projectResult = true; // projected must also be non empty to actually project this dataset
  916. visited = false;
  917. }
  918. void ResourcerInfo::setConditionSource(IHqlExpression * condition, bool isFirst)
  919. {
  920. if (isFirst)
  921. {
  922. conditionSourceCount++;
  923. graph->hasConditionSource = true;
  924. }
  925. }
  926. bool ResourcerInfo::addCondition(IHqlExpression * condition)
  927. {
  928. conditions.append(*LINK(condition));
  929. return graph->addCondition(condition);
  930. }
  931. void ResourcerInfo::addProjected(IHqlExpression * next)
  932. {
  933. if (projectResult && !projected.contains(*next))
  934. projected.append(*LINK(next));
  935. }
  936. void ResourcerInfo::clearProjected()
  937. {
  938. projectResult = false;
  939. projected.kill();
  940. }
  941. void ResourcerInfo::addSpillFlags(HqlExprArray & args, bool isRead)
  942. {
  943. if (isGrouped(original))
  944. args.append(*createAttribute(groupedAtom));
  945. if (outputToUseForSpill)
  946. {
  947. assertex(isRead);
  948. appendCloneAttribute(args, outputToUseForSpill, __compressed__Atom);
  949. appendCloneAttribute(args, outputToUseForSpill, jobTempAtom);
  950. appendCloneAttribute(args, outputToUseForSpill, _spill_Atom);
  951. }
  952. else
  953. {
  954. args.append(*createAttribute(__compressed__Atom));
  955. args.append(*createAttribute(_spill_Atom));
  956. args.append(*createAttribute(_noReplicate_Atom));
  957. if (options->targetRoxie())
  958. args.append(*createAttribute(_noAccess_Atom));
  959. }
  960. if (isRead)
  961. {
  962. args.append(*createAttribute(_noVirtual_Atom)); // Don't interpret virtual fields as virtual...
  963. if (!original->isDataset() || hasSingleRow(original))
  964. args.append(*createAttribute(rowAtom)); // add a flag so the selectnth[1] can be removed later...
  965. }
  966. else
  967. {
  968. unsigned numUses = numExternalUses;
  969. if (false) // options->cloneConditionals)
  970. {
  971. //Remove all the links from conditional graphs...
  972. ForEachItemIn(i1, graph->sinks)
  973. {
  974. ResourceGraphLink & cur = graph->sinks.item(i1);
  975. if ((cur.sourceNode == original) && !cur.sinkGraph->isUnconditional)
  976. numUses--;
  977. }
  978. numUses += calcNumConditionalUses();
  979. }
  980. args.append(*createAttribute(_tempCount_Atom, getSizetConstant(numUses)));
  981. }
  982. }
  983. bool ResourcerInfo::alwaysExpand()
  984. {
  985. return (original->getOperator() == no_mapto);
  986. }
  987. static IHqlExpression * stripTrueFalse(IHqlExpression * expr)
  988. {
  989. if (expr->queryName() == instanceAtom)
  990. {
  991. IHqlExpression * parent = expr->queryChild(2);
  992. if (parent)
  993. parent = stripTrueFalse(parent);
  994. return createAttribute(instanceAtom, LINK(expr->queryChild(0)), LINK(expr->queryChild(1)), parent);
  995. }
  996. else
  997. {
  998. IHqlExpression * parent = expr->queryChild(1);
  999. if (parent && parent->isAttribute())
  1000. parent = stripTrueFalse(parent);
  1001. return createAttribute(tempAtom, LINK(expr->queryChild(0)), parent);
  1002. }
  1003. }
  1004. unsigned ResourcerInfo::calcNumConditionalUses()
  1005. {
  1006. //for thor and hthor, where the conditional graph is cloned, the maximum number of times it can be read
  1007. //is 1 for a shared conditional graph, or once for each combination of conditions it is used as the
  1008. //input for. However if it is used more than once in a single branch of a condition it needs to be counted
  1009. //several times.
  1010. //It is always better to overestimate than underestimate. (But even better to get it right.)
  1011. HqlExprArray uniqueConditions;
  1012. UnsignedArray uniqueCounts;
  1013. ForEachItemIn(idx, conditions)
  1014. {
  1015. IHqlExpression & cur = conditions.item(idx);
  1016. OwnedHqlExpr unique = stripTrueFalse(&cur);
  1017. unsigned numOccurences = 0;
  1018. ForEachItemIn(j, conditions)
  1019. if (&conditions.item(j) == &cur)
  1020. numOccurences++;
  1021. unsigned match = uniqueConditions.find(*unique);
  1022. if (match == NotFound)
  1023. {
  1024. uniqueConditions.append(*unique.getClear());
  1025. uniqueCounts.append(numOccurences);
  1026. }
  1027. else
  1028. {
  1029. if (uniqueCounts.item(match) < numOccurences)
  1030. uniqueCounts.replace(numOccurences, match);
  1031. }
  1032. }
  1033. unsigned condUses = 0;
  1034. ForEachItemIn(k, uniqueCounts)
  1035. condUses += uniqueCounts.item(k);
  1036. return condUses;
  1037. }
  1038. // Each aggregate has form. Setresult(select(selectnth(aggregate...,1),field),name,seq)
  1039. IHqlExpression * ResourcerInfo::createAggregation(IHqlExpression * expr)
  1040. {
  1041. LinkedHqlExpr transformed = expr;
  1042. ForEachItemIn(idx, aggregates)
  1043. {
  1044. IHqlExpression & cur = aggregates.item(idx);
  1045. IHqlExpression * row2ds = cur.queryChild(0);
  1046. IHqlExpression * selectnth = row2ds->queryChild(0);
  1047. IHqlExpression * aggregate = selectnth->queryChild(0);
  1048. IHqlExpression * aggregateRecord = aggregate->queryChild(1);
  1049. assertex(aggregate->getOperator() == no_newaggregate);
  1050. HqlExprArray aggargs, setargs;
  1051. //Through aggregate has form (dataset, record, transform, list of set results);
  1052. aggargs.append(*LINK(transformed));
  1053. aggargs.append(*LINK(aggregateRecord));
  1054. IHqlExpression * mapped = replaceSelector(aggregate->queryChild(2), original, expr->queryNormalizedSelector());
  1055. aggargs.append(*mapped);
  1056. OwnedHqlExpr active = createDataset(no_anon, LINK(aggregateRecord), NULL);
  1057. OwnedHqlExpr mappedSelect = replaceSelector(cur.queryChild(1), row2ds, active);
  1058. setargs.append(*LINK(active));
  1059. setargs.append(*LINK(mappedSelect));
  1060. unwindChildren(setargs, &cur, 1);
  1061. aggargs.append(*createValue(no_extractresult, makeVoidType(), setargs));
  1062. transformed.setown(createDataset(no_throughaggregate, aggargs));
  1063. }
  1064. return transformed.getClear();
  1065. }
  1066. bool ResourcerInfo::useGraphResult()
  1067. {
  1068. if (options->useResultsForChildSpills && linkedFromChild)
  1069. return true;
  1070. if (options->alwaysUseGraphResults)
  1071. return true;
  1072. if (!options->useGraphResults)
  1073. return false;
  1074. if (linkedFromChild)
  1075. return true;
  1076. //Roxie converts spills into splitters, so best to retain them
  1077. if (options->targetClusterType == RoxieCluster)
  1078. return false;
  1079. return true;
  1080. }
  1081. bool ResourcerInfo::useGlobalResult()
  1082. {
  1083. return (linkedFromChild && !useGraphResult());
  1084. }
  1085. IHqlExpression * ResourcerInfo::createSpillName()
  1086. {
  1087. if (outputToUseForSpill)
  1088. return LINK(outputToUseForSpill->queryChild(1));
  1089. if (!spillName)
  1090. {
  1091. if (useGraphResult())
  1092. spillName.setown(options->createSpillName(true));
  1093. else if (useGlobalResult())
  1094. {
  1095. StringBuffer valueText;
  1096. getUniqueId(valueText.append("spill"));
  1097. spillName.setown(createConstant(valueText.str()));
  1098. }
  1099. else
  1100. spillName.setown(options->createSpillName(false));
  1101. }
  1102. return LINK(spillName);
  1103. }
  1104. IHqlExpression * ResourcerInfo::createSpilledRead(IHqlExpression * spillReason)
  1105. {
  1106. OwnedHqlExpr dataset;
  1107. HqlExprArray args;
  1108. IHqlExpression * record = original->queryRecord();
  1109. bool loseDistribution = true;
  1110. if (useGraphResult())
  1111. {
  1112. if (spilledDataset)
  1113. args.append(*LINK(spilledDataset));
  1114. else
  1115. args.append(*LINK(record));
  1116. args.append(*LINK(options->graphIdExpr));
  1117. args.append(*createSpillName());
  1118. if (isGrouped(original))
  1119. args.append(*createAttribute(groupedAtom));
  1120. if (!original->isDataset())
  1121. args.append(*createAttribute(rowAtom));
  1122. args.append(*createAttribute(_spill_Atom));
  1123. IHqlExpression * recordCountAttr = queryRecordCountInfo(original);
  1124. if (recordCountAttr)
  1125. args.append(*LINK(recordCountAttr));
  1126. if (options->targetThor() && original->isDataset() && !options->isChildQuery)
  1127. args.append(*createAttribute(_distributed_Atom));
  1128. node_operator readOp = spilledDataset ? no_readspill : no_getgraphresult;
  1129. if (original->isDatarow())
  1130. dataset.setown(createRow(readOp, args));
  1131. else if (original->isDictionary())
  1132. dataset.setown(createDictionary(readOp, args));
  1133. else
  1134. dataset.setown(createDataset(readOp, args));
  1135. loseDistribution = false;
  1136. }
  1137. else if (useGlobalResult())
  1138. {
  1139. args.append(*LINK(record));
  1140. args.append(*createAttribute(nameAtom, createSpillName()));
  1141. args.append(*createAttribute(sequenceAtom, getLocalSequenceNumber()));
  1142. addSpillFlags(args, true);
  1143. IHqlExpression * recordCountAttr = queryRecordCountInfo(original);
  1144. if (recordCountAttr)
  1145. args.append(*LINK(recordCountAttr));
  1146. if (original->isDictionary())
  1147. dataset.setown(createDictionary(no_workunit_dataset, args));
  1148. else
  1149. dataset.setown(createDataset(no_workunit_dataset, args));
  1150. }
  1151. else
  1152. {
  1153. if (spilledDataset)
  1154. {
  1155. args.append(*LINK(spilledDataset));
  1156. args.append(*createSpillName());
  1157. }
  1158. else
  1159. {
  1160. args.append(*createSpillName());
  1161. args.append(*LINK(record));
  1162. }
  1163. args.append(*createValue(no_thor));
  1164. addSpillFlags(args, true);
  1165. args.append(*createUniqueId());
  1166. if (options->isChildQuery && options->targetRoxie())
  1167. {
  1168. args.append(*createAttribute(_colocal_Atom));
  1169. args.append(*createLocalAttribute());
  1170. }
  1171. if (spillReason)
  1172. args.append(*LINK(spillReason));
  1173. if (spilledDataset)
  1174. dataset.setown(createDataset(no_readspill, args));
  1175. else
  1176. {
  1177. IHqlExpression * recordCountAttr = queryRecordCountInfo(original);
  1178. if (recordCountAttr)
  1179. args.append(*LINK(recordCountAttr));
  1180. dataset.setown(createDataset(no_table, args));
  1181. }
  1182. loseDistribution = false;
  1183. }
  1184. dataset.setown(preserveTableInfo(dataset, original, loseDistribution, NULL));
  1185. return wrapRowOwn(dataset.getClear());
  1186. }
  1187. IHqlExpression * ResourcerInfo::createSpilledWrite(IHqlExpression * transformed)
  1188. {
  1189. assertex(!outputToUseForSpill);
  1190. HqlExprArray args;
  1191. if (useGraphResult())
  1192. {
  1193. assertex(options->graphIdExpr);
  1194. if (options->createSpillAsDataset && !linkedFromChild)
  1195. {
  1196. IHqlExpression * value = LINK(transformed);
  1197. if (value->isDatarow())
  1198. value = createDatasetFromRow(value);
  1199. spilledDataset.setown(createDataset(no_commonspill, value));
  1200. args.append(*LINK(spilledDataset));
  1201. }
  1202. else
  1203. args.append(*LINK(transformed));
  1204. args.append(*LINK(options->graphIdExpr));
  1205. args.append(*createSpillName());
  1206. args.append(*createAttribute(_spill_Atom));
  1207. if (linkedFromChild)
  1208. args.append(*createAttribute(_accessedFromChild_Atom));
  1209. if (spilledDataset)
  1210. return createValue(no_writespill, makeVoidType(), args);
  1211. return createValue(no_setgraphresult, makeVoidType(), args);
  1212. }
  1213. else if (useGlobalResult())
  1214. {
  1215. args.append(*LINK(transformed));
  1216. args.append(*createAttribute(sequenceAtom, getLocalSequenceNumber()));
  1217. args.append(*createAttribute(namedAtom, createSpillName()));
  1218. addSpillFlags(args, false);
  1219. return createValue(no_output, makeVoidType(), args);
  1220. }
  1221. else
  1222. {
  1223. if (options->createSpillAsDataset)
  1224. {
  1225. IHqlExpression * value = LINK(transformed);
  1226. if (value->isDatarow())
  1227. value = createDatasetFromRow(value);
  1228. spilledDataset.setown(createDataset(no_commonspill, value));
  1229. args.append(*LINK(spilledDataset));
  1230. }
  1231. else
  1232. args.append(*LINK(transformed));
  1233. args.append(*createSpillName());
  1234. addSpillFlags(args, false);
  1235. if (options->createSpillAsDataset)
  1236. return createValue(no_writespill, makeVoidType(), args);
  1237. return createValue(no_output, makeVoidType(), args);
  1238. }
  1239. }
  1240. bool ResourcerInfo::okToSpillThrough()
  1241. {
  1242. bool isGraphResult = useGraphResult();
  1243. if (isGraphResult)
  1244. return options->allowThroughResult;
  1245. if (useGlobalResult())
  1246. return false;
  1247. return (options->allowThroughSpill && !options->createSpillAsDataset);
  1248. }
  1249. void ResourcerInfo::noteUsedFromChild(bool _forceHoist)
  1250. {
  1251. linkedFromChild = true;
  1252. outputToUseForSpill = NULL;
  1253. if (_forceHoist)
  1254. forceHoist = true;
  1255. }
  1256. unsigned ResourcerInfo::numInternalUses()
  1257. {
  1258. return numUses - numExternalUses - aggregates.ordinality();
  1259. }
  1260. bool ResourcerInfo::spillSharesSplitter()
  1261. {
  1262. if (outputToUseForSpill || useGraphResult() || useGlobalResult())
  1263. return false;
  1264. if (okToSpillThrough())
  1265. return false;
  1266. if (!isSplit() || !balanced)
  1267. return false;
  1268. return true;
  1269. }
  1270. void ResourcerInfo::setRootActivity()
  1271. {
  1272. isRootActivity = true;
  1273. }
  1274. IHqlExpression * ResourcerInfo::createSpiller(IHqlExpression * transformed, bool reuseSplitter)
  1275. {
  1276. if (outputToUseForSpill)
  1277. return LINK(transformed);
  1278. if (!okToSpillThrough())
  1279. {
  1280. OwnedHqlExpr split;
  1281. if (reuseSplitter)
  1282. {
  1283. assertex(transformed->getOperator() == no_split);
  1284. split.set(transformed);
  1285. }
  1286. else
  1287. {
  1288. if (transformed->isDataset())
  1289. split.setown(createDataset(no_split, LINK(transformed), createComma(createAttribute(balancedAtom), createUniqueId())));
  1290. else
  1291. split.setown(createRow(no_split, LINK(transformed), createComma(createAttribute(balancedAtom), createUniqueId())));
  1292. split.setown(cloneInheritedAnnotations(original, split));
  1293. }
  1294. splitterOutput.setown(createSpilledWrite(split));
  1295. return split.getClear();
  1296. }
  1297. HqlExprArray args;
  1298. node_operator op;
  1299. args.append(*LINK(transformed));
  1300. if (useGraphResult())
  1301. {
  1302. op = no_spillgraphresult;
  1303. args.append(*LINK(options->graphIdExpr));
  1304. args.append(*createSpillName());
  1305. args.append(*createAttribute(_spill_Atom));
  1306. }
  1307. else
  1308. {
  1309. op = no_spill;
  1310. args.append(*createSpillName());
  1311. addSpillFlags(args, false);
  1312. }
  1313. OwnedHqlExpr spill;
  1314. if (original->isDatarow())
  1315. spill.setown(createRow(op, args));
  1316. else
  1317. spill.setown(createDataset(op, args));
  1318. return cloneInheritedAnnotations(original, spill);
  1319. }
  1320. IHqlExpression * ResourcerInfo::createSplitter(IHqlExpression * transformed)
  1321. {
  1322. if (transformed->getOperator() == no_libraryscopeinstance)
  1323. return LINK(transformed);
  1324. IHqlExpression * attr = createUniqueId();
  1325. if (balanced)
  1326. attr = createComma(attr, createAttribute(balancedAtom));
  1327. OwnedHqlExpr split;
  1328. if (transformed->isDataset())
  1329. split.setown(createDataset(no_split, LINK(transformed), attr));
  1330. else
  1331. split.setown(createRow(no_split, LINK(transformed), attr));
  1332. return cloneInheritedAnnotations(original, split);
  1333. }
  1334. IHqlExpression * ResourcerInfo::createTransformedExpr(IHqlExpression * expr)
  1335. {
  1336. LinkedHqlExpr transformed = expr;
  1337. if (aggregates.ordinality())
  1338. transformed.setown(createAggregation(transformed));
  1339. if (spillSharesSplitter())
  1340. {
  1341. transformed.setown(createSplitter(transformed));
  1342. if (isExternalSpill())
  1343. transformed.setown(createSpiller(transformed, true));
  1344. }
  1345. else
  1346. {
  1347. if (isExternalSpill())
  1348. transformed.setown(createSpiller(transformed, false));
  1349. if (isSplit())
  1350. transformed.setown(createSplitter(transformed));
  1351. else if (isSpilledWrite())
  1352. transformed.setown(createSpilledWrite(transformed));
  1353. }
  1354. return transformed.getClear();
  1355. }
  1356. bool ResourcerInfo::expandRatherThanSpill(bool noteOtherSpills)
  1357. {
  1358. if (options->shareDontExpand)
  1359. return expandRatherThanSplit();
  1360. IHqlExpression * expr = original;
  1361. if (!options->minimiseSpills || linkedFromChild)
  1362. noteOtherSpills = false;
  1363. if (noteOtherSpills)
  1364. {
  1365. ResourcerInfo * info = queryResourceInfo(expr);
  1366. if (info && info->isSpilledWrite())
  1367. return (info->transformed == NULL);
  1368. }
  1369. bool isFiltered = false;
  1370. bool isProcessed = false;
  1371. loop
  1372. {
  1373. ResourcerInfo * info = queryResourceInfo(expr);
  1374. if (info && info->neverSplit)
  1375. return true;
  1376. if (info && info->forceHoist)
  1377. return false;
  1378. node_operator op = expr->getOperator();
  1379. switch (op)
  1380. {
  1381. case no_table:
  1382. {
  1383. //This is only executed for hthor/thor. Roxie has used expandRatherThanSplit().
  1384. //We need to balance the saving from reading reduced data in the other branches with the cost of
  1385. //writing the spill file to disk.
  1386. if (isFiltered && (numExternalUses >= options->filteredSpillThreshold))
  1387. return false;
  1388. IHqlExpression * mode = expr->queryChild(2);
  1389. switch (mode->getOperator())
  1390. {
  1391. case no_thor: case no_flat:
  1392. //MORE: The following is possibly better - but roxie should be able to read from non spill data files in child queries fine
  1393. //if ((options->targetClusterType == RoxieCluster) && linkedFromChild)) return false;
  1394. return true;
  1395. default:
  1396. return false;
  1397. }
  1398. }
  1399. case no_stepped:
  1400. return true;
  1401. case no_inlinetable:
  1402. {
  1403. IHqlExpression * transforms = expr->queryChild(0);
  1404. //The inline table code means this should generate smaller code, and more efficient
  1405. if (!isFiltered && !isProcessed && transforms->isConstant())
  1406. return true;
  1407. if (transforms->numChildren() > MAX_INLINE_COMMON_COUNT)
  1408. return false;
  1409. return true;
  1410. }
  1411. case no_getresult:
  1412. case no_temptable:
  1413. case no_rows:
  1414. case no_xmlproject:
  1415. case no_workunit_dataset:
  1416. return !isProcessed && !isFiltered;
  1417. case no_getgraphresult:
  1418. return !expr->hasAttribute(_streaming_Atom); // we must not duplicate streaming inputs!
  1419. case no_keyindex:
  1420. case no_newkeyindex:
  1421. if (!isFiltered)
  1422. return true;
  1423. return options->cloneFilteredIndex;
  1424. case no_datasetfromrow:
  1425. if (getNumActivityArguments(expr) == 0)
  1426. return true;
  1427. return false;
  1428. case no_fail:
  1429. case no_null:
  1430. return !expr->isAction();
  1431. case no_assertsorted:
  1432. case no_sorted:
  1433. case no_grouped:
  1434. case no_distributed:
  1435. case no_preservemeta:
  1436. case no_nofold:
  1437. case no_nohoist:
  1438. case no_section:
  1439. case no_sectioninput:
  1440. case no_dataset_alias:
  1441. expr = expr->queryChild(0);
  1442. break;
  1443. case no_newusertable:
  1444. case no_limit:
  1445. case no_keyedlimit:
  1446. expr = expr->queryChild(0);
  1447. isProcessed = true;
  1448. break;
  1449. case no_hqlproject:
  1450. if (expr->hasAttribute(_countProject_Atom) || expr->hasAttribute(prefetchAtom))
  1451. return false;
  1452. expr = expr->queryChild(0);
  1453. isProcessed = true;
  1454. break;
  1455. //MORE: Not so sure about all the following, include them so behaviour doesn't change
  1456. case no_compound_diskread:
  1457. case no_compound_disknormalize:
  1458. case no_compound_diskaggregate:
  1459. case no_compound_diskcount:
  1460. case no_compound_diskgroupaggregate:
  1461. case no_compound_indexread:
  1462. case no_compound_indexnormalize:
  1463. case no_compound_indexaggregate:
  1464. case no_compound_indexcount:
  1465. case no_compound_indexgroupaggregate:
  1466. case no_compound_childread:
  1467. case no_compound_childnormalize:
  1468. case no_compound_childaggregate:
  1469. case no_compound_childcount:
  1470. case no_compound_childgroupaggregate:
  1471. case no_compound_selectnew:
  1472. case no_compound_inline:
  1473. case no_datasetfromdictionary:
  1474. expr = expr->queryChild(0);
  1475. break;
  1476. case no_filter:
  1477. isFiltered = true;
  1478. expr = expr->queryChild(0);
  1479. break;
  1480. case no_select:
  1481. {
  1482. if (options->targetClusterType == RoxieCluster)
  1483. return false;
  1484. if (!isNewSelector(expr))
  1485. return true;
  1486. expr = expr->queryChild(0);
  1487. break;
  1488. }
  1489. default:
  1490. return false;
  1491. }
  1492. //The following reduces the number of spills by taking into account other spills.
  1493. if (noteOtherSpills)
  1494. {
  1495. ResourcerInfo * info = queryResourceInfo(expr);
  1496. if (info)
  1497. {
  1498. if (info->isSpilledWrite())
  1499. return (info->transformed == NULL);
  1500. if (info->numExternalUses)
  1501. {
  1502. if (isFiltered && (numExternalUses >= options->filteredSpillThreshold))
  1503. return false;
  1504. return true;
  1505. }
  1506. }
  1507. }
  1508. }
  1509. }
  1510. bool ResourcerInfo::expandRatherThanSplit()
  1511. {
  1512. //MORE: This doesn't really work - should do indexMatching first.
  1513. //should only expand if one side that uses this is also filtered
  1514. IHqlExpression * expr = original;
  1515. loop
  1516. {
  1517. ResourcerInfo * info = queryResourceInfo(expr);
  1518. if (info && info->neverSplit)
  1519. return true;
  1520. switch (expr->getOperator())
  1521. {
  1522. case no_keyindex:
  1523. case no_newkeyindex:
  1524. case no_rowset:
  1525. case no_getgraphloopresultset:
  1526. return true;
  1527. case no_null:
  1528. return !expr->isAction();
  1529. case no_inlinetable:
  1530. if (options->expandSingleConstRow && hasSingleRow(expr))
  1531. {
  1532. IHqlExpression * values = expr->queryChild(0);
  1533. if (values->queryChild(0)->isConstant())
  1534. return true;
  1535. }
  1536. return false;
  1537. case no_stepped:
  1538. case no_rowsetrange:
  1539. case no_rowsetindex:
  1540. return true;
  1541. case no_sorted:
  1542. case no_grouped:
  1543. case no_distributed:
  1544. case no_preservemeta:
  1545. case no_compound_diskread:
  1546. case no_compound_disknormalize:
  1547. case no_compound_diskaggregate:
  1548. case no_compound_diskcount:
  1549. case no_compound_diskgroupaggregate:
  1550. case no_compound_indexread:
  1551. case no_compound_indexnormalize:
  1552. case no_compound_indexaggregate:
  1553. case no_compound_indexcount:
  1554. case no_compound_indexgroupaggregate:
  1555. case no_compound_childread:
  1556. case no_compound_childnormalize:
  1557. case no_compound_childaggregate:
  1558. case no_compound_childcount:
  1559. case no_compound_childgroupaggregate:
  1560. case no_compound_selectnew:
  1561. case no_compound_inline:
  1562. case no_section:
  1563. case no_sectioninput:
  1564. case no_dataset_alias:
  1565. break;
  1566. case no_select:
  1567. if (options->targetClusterType == RoxieCluster)
  1568. return false;
  1569. if (!isNewSelector(expr))
  1570. {
  1571. if (!hasLinkCountedModifier(expr))
  1572. return false;
  1573. return true;
  1574. }
  1575. break;
  1576. case no_rows:
  1577. //If executing in a child query then you'll have less thread contention if the iterator is duplicated
  1578. //So should probably uncomment the following.
  1579. //return true;
  1580. return false;
  1581. default:
  1582. return false;
  1583. }
  1584. expr = expr->queryChild(0);
  1585. }
  1586. }
  1587. bool neverCommonUp(IHqlExpression * expr)
  1588. {
  1589. loop
  1590. {
  1591. node_operator op = expr->getOperator();
  1592. switch (op)
  1593. {
  1594. case no_keyindex:
  1595. case no_newkeyindex:
  1596. return true;
  1597. case no_filter:
  1598. expr = expr->queryChild(0);
  1599. break;
  1600. default:
  1601. return false;
  1602. }
  1603. }
  1604. }
  1605. bool ResourcerInfo::neverCommonUp()
  1606. {
  1607. return ::neverCommonUp(original);
  1608. }
  1609. bool ResourcerInfo::isExternalSpill()
  1610. {
  1611. if (expandRatherThanSpill(true) || (numInternalUses() == 0))
  1612. return false;
  1613. return (numExternalUses != 0);
  1614. }
  1615. bool ResourcerInfo::isSplit()
  1616. {
  1617. return numSplitPaths() > 1;
  1618. }
  1619. unsigned ResourcerInfo::numSplitPaths()
  1620. {
  1621. unsigned internal = numInternalUses();
  1622. if ((internal == 0) || !options->allowSplitBetweenSubGraphs)
  1623. return internal;
  1624. //MORE
  1625. return internal;
  1626. }
  1627. bool ResourcerInfo::isSpilledWrite()
  1628. {
  1629. if (numInternalUses() == 0)
  1630. return true;
  1631. return false;
  1632. }
  1633. IHqlExpression * ResourcerInfo::wrapRowOwn(IHqlExpression * expr)
  1634. {
  1635. if (!original->isDataset() && !original->isDictionary() && !expr->isDatarow())
  1636. expr = createRow(no_selectnth, expr, getSizetConstant(1));
  1637. return expr;
  1638. }
  1639. //---------------------------------------------------------------------------
  1640. EclResourcer::EclResourcer(IErrorReceiver * _errors, IConstWorkUnit * _wu, ClusterType _targetClusterType, unsigned _clusterSize, const HqlCppOptions & _translatorOptions)
  1641. {
  1642. wu.set(_wu);
  1643. errors = _errors;
  1644. lockTransformMutex();
  1645. targetClusterType = _targetClusterType;
  1646. clusterSize = _clusterSize ? _clusterSize : FIXED_CLUSTER_SIZE;
  1647. insideNeverSplit = false;
  1648. insideSteppedNeverSplit = false;
  1649. sequential = false;
  1650. options.minimizeSpillSize = _translatorOptions.minimizeSpillSize;
  1651. unsigned totalMemory = _translatorOptions.resourceMaxMemory ? _translatorOptions.resourceMaxMemory : DEFAULT_TOTAL_MEMORY;
  1652. unsigned maxSockets = _translatorOptions.resourceMaxSockets ? _translatorOptions.resourceMaxSockets : DEFAULT_MAX_SOCKETS;
  1653. unsigned maxActivities = _translatorOptions.resourceMaxActivities ? _translatorOptions.resourceMaxActivities : DEFAULT_MAX_ACTIVITIES;
  1654. unsigned maxHeavy = _translatorOptions.resourceMaxHeavy;
  1655. unsigned maxDistribute = _translatorOptions.resourceMaxDistribute;
  1656. resourceLimit = new CResources(0);
  1657. resourceLimit->set(RESactivities, maxActivities);
  1658. switch (targetClusterType)
  1659. {
  1660. case ThorLCRCluster:
  1661. resourceLimit->set(RESheavy, maxHeavy).set(REShashdist, maxDistribute);
  1662. resourceLimit->set(RESmastersocket, maxSockets).set(RESslavememory,totalMemory);
  1663. resourceLimit->set(RESslavesocket, maxSockets).set(RESmastermemory, totalMemory);
  1664. break;
  1665. default:
  1666. resourceLimit->set(RESheavy, 0xffffffff).set(REShashdist, 0xffffffff);
  1667. resourceLimit->set(RESmastersocket, 0xffffffff).set(RESslavememory, 0xffffffff);
  1668. resourceLimit->set(RESslavesocket, 0xffffffff).set(RESmastermemory, 0xffffffff);
  1669. clusterSize = 1;
  1670. break;
  1671. }
  1672. if (_translatorOptions.unlimitedResources)
  1673. {
  1674. resourceLimit->set(RESheavy, 0xffffffff).set(REShashdist, 0xffffffff);
  1675. resourceLimit->set(RESmastersocket, 0xffffffff).set(RESslavememory,0xffffffff);
  1676. resourceLimit->set(RESslavesocket, 0xffffffff).set(RESmastermemory,0xffffffff);
  1677. }
  1678. options.isChildQuery = false;
  1679. options.targetClusterType = targetClusterType;
  1680. options.filteredSpillThreshold = _translatorOptions.filteredReadSpillThreshold;
  1681. options.allowThroughSpill = (targetClusterType != RoxieCluster) && (targetClusterType != ThorLCRCluster) && _translatorOptions.allowThroughSpill;
  1682. options.allowThroughResult = (targetClusterType != RoxieCluster) && (targetClusterType != ThorLCRCluster);
  1683. options.cloneFilteredIndex = (targetClusterType != RoxieCluster);
  1684. options.spillSharedConditionals = (targetClusterType == RoxieCluster);
  1685. options.shareDontExpand = (targetClusterType == RoxieCluster);
  1686. options.graphIdExpr = NULL;
  1687. //MORE The following doesn't always work - it gets sometimes confused about spill files - see latestheaderbuild for an example.
  1688. //Try again once cloneConditionals is false for thor
  1689. options.minimiseSpills = _translatorOptions.minimiseSpills;
  1690. spillMultiCondition = _translatorOptions.spillMultiCondition;
  1691. spotThroughAggregate = _translatorOptions.spotThroughAggregate && (targetClusterType != RoxieCluster) && (targetClusterType != ThorLCRCluster);
  1692. options.noConditionalLinks = (targetClusterType != HThorCluster);
  1693. options.hoistResourced = _translatorOptions.hoistResourced;
  1694. options.alwaysUseGraphResults = _translatorOptions.alwaysUseGraphResults;
  1695. options.useGraphResults = false; // modified by later call
  1696. options.groupedChildIterators = _translatorOptions.groupedChildIterators;
  1697. options.allowSplitBetweenSubGraphs = false;//(targetClusterType == RoxieCluster);
  1698. options.clusterSize = clusterSize;
  1699. options.preventKeyedSplit = _translatorOptions.preventKeyedSplit;
  1700. options.preventSteppedSplit = _translatorOptions.preventSteppedSplit;
  1701. options.minimizeSkewBeforeSpill = _translatorOptions.minimizeSkewBeforeSpill;
  1702. options.expandSingleConstRow = true;
  1703. options.createSpillAsDataset = _translatorOptions.optimizeSpillProject && (targetClusterType != HThorCluster);
  1704. options.combineSiblings = _translatorOptions.combineSiblingGraphs && (targetClusterType != HThorCluster) && (targetClusterType != RoxieCluster);
  1705. options.optimizeSharedInputs = _translatorOptions.optimizeSharedGraphInputs && options.combineSiblings;
  1706. options.actionLinkInNewGraph = _translatorOptions.actionLinkInNewGraph || (targetClusterType == HThorCluster);
  1707. options.convertCompoundToExecuteWhen = false;
  1708. options.useResultsForChildSpills = _translatorOptions.useResultsForChildSpills;
  1709. }
  1710. EclResourcer::~EclResourcer()
  1711. {
  1712. delete resourceLimit;
  1713. unlockTransformMutex();
  1714. }
  1715. void EclResourcer::setChildQuery(bool value)
  1716. {
  1717. options.isChildQuery = value;
  1718. }
  1719. void EclResourcer::setNewChildQuery(IHqlExpression * graphIdExpr, unsigned numResults)
  1720. {
  1721. options.graphIdExpr = graphIdExpr;
  1722. options.nextResult = numResults;
  1723. }
  1724. void EclResourcer::changeGraph(IHqlExpression * expr, ResourceGraphInfo * newGraph)
  1725. {
  1726. ResourcerInfo * info = queryResourceInfo(expr);
  1727. info->graph.set(newGraph);
  1728. ForEachItemInRev(idx, links)
  1729. {
  1730. ResourceGraphLink & cur = links.item(idx);
  1731. if (cur.sourceNode == expr)
  1732. cur.changeSourceGraph(newGraph);
  1733. else if (cur.sinkNode == expr)
  1734. cur.changeSinkGraph(newGraph);
  1735. assertex(cur.sinkGraph != cur.sourceGraph);
  1736. }
  1737. }
  1738. ResourceGraphInfo * EclResourcer::createGraph()
  1739. {
  1740. ResourceGraphInfo * graph = new ResourceGraphInfo(&options);
  1741. graphs.append(*LINK(graph));
  1742. //PrintLog("Create graph %p", graph);
  1743. return graph;
  1744. }
  1745. void EclResourcer::connectGraphs(ResourceGraphInfo * sourceGraph, IHqlExpression * sourceNode, ResourceGraphInfo * sinkGraph, IHqlExpression * sinkNode, LinkKind kind)
  1746. {
  1747. ResourceGraphLink * link = new ResourceGraphLink(sourceGraph, sourceNode, sinkGraph, sinkNode, kind);
  1748. links.append(*link);
  1749. if (sourceGraph)
  1750. sourceGraph->sinks.append(*link);
  1751. if (sinkGraph)
  1752. sinkGraph->sources.append(*link);
  1753. }
  1754. ResourcerInfo * EclResourcer::queryCreateResourceInfo(IHqlExpression * expr)
  1755. {
  1756. IHqlExpression * body = expr->queryBody();
  1757. ResourcerInfo * info = (ResourcerInfo *)body->queryTransformExtra();
  1758. if (!info)
  1759. {
  1760. info = new ResourcerInfo(expr, &options);
  1761. body->setTransformExtraOwned(info);
  1762. }
  1763. return info;
  1764. }
  1765. void EclResourcer::replaceGraphReferences(IHqlExpression * expr, ResourceGraphInfo * oldGraph, ResourceGraphInfo * newGraph)
  1766. {
  1767. ResourcerInfo * info = queryResourceInfo(expr);
  1768. if (!info || !info->containsActivity)
  1769. return;
  1770. if (info->isActivity && info->graph != oldGraph)
  1771. return;
  1772. info->graph.set(newGraph);
  1773. unsigned first = getFirstActivityArgument(expr);
  1774. unsigned last = first + getNumActivityArguments(expr);
  1775. for (unsigned idx=first; idx < last; idx++)
  1776. replaceGraphReferences(expr->queryChild(idx), oldGraph, newGraph);
  1777. }
  1778. void EclResourcer::removeLink(ResourceGraphLink & link, bool keepExternalUses)
  1779. {
  1780. ResourcerInfo * info = (ResourcerInfo *)queryResourceInfo(link.sourceNode);
  1781. assertex(info && info->numExternalUses > 0);
  1782. if (!keepExternalUses)
  1783. info->numExternalUses--;
  1784. if (link.sinkGraph)
  1785. link.sinkGraph->sources.zap(link);
  1786. link.sourceGraph->sinks.zap(link);
  1787. links.zap(link);
  1788. }
  1789. void EclResourcer::replaceGraphReferences(ResourceGraphInfo * oldGraph, ResourceGraphInfo * newGraph)
  1790. {
  1791. ForEachItemIn(idx1, oldGraph->sinks)
  1792. {
  1793. ResourceGraphLink & sink = oldGraph->sinks.item(idx1);
  1794. replaceGraphReferences(sink.sourceNode, oldGraph, newGraph);
  1795. }
  1796. ForEachItemInRev(idx2, links)
  1797. {
  1798. ResourceGraphLink & cur = links.item(idx2);
  1799. if (cur.sourceGraph == oldGraph)
  1800. {
  1801. if (cur.sinkGraph == newGraph)
  1802. removeLink(cur, false);
  1803. else
  1804. cur.changeSourceGraph(newGraph);
  1805. }
  1806. else if (cur.sinkGraph == oldGraph)
  1807. {
  1808. if (cur.sourceGraph == newGraph)
  1809. removeLink(cur, false);
  1810. else
  1811. cur.changeSinkGraph(newGraph);
  1812. }
  1813. }
  1814. }
  1815. //------------------------------------------------------------------------------------------
  1816. // PASS1: Gather information about splitter locations..
  1817. void EclResourcer::tagActiveCursors(HqlExprCopyArray & activeRows)
  1818. {
  1819. ForEachItemIn(i, activeRows)
  1820. {
  1821. IHqlExpression & cur = activeRows.item(i);
  1822. activeSelectors.append(cur);
  1823. queryCreateResourceInfo(&cur)->isAlreadyInScope = true;
  1824. }
  1825. }
  1826. inline bool projectSelectorDatasetToField(IHqlExpression * row)
  1827. {
  1828. return ((row->getOperator() == no_selectnth) && getFieldCount(row->queryRecord()) > 1);
  1829. }
  1830. static HqlTransformerInfo eclHoistLocatorInfo("EclHoistLocator");
  1831. class EclHoistLocator : public NewHqlTransformer
  1832. {
  1833. public:
  1834. EclHoistLocator(ChildDependentArray & _matched) : NewHqlTransformer(eclHoistLocatorInfo), matched(_matched)
  1835. {
  1836. alwaysSingle = true;
  1837. }
  1838. void analyseChild(IHqlExpression * expr, bool _alwaysSingle)
  1839. {
  1840. alwaysSingle = _alwaysSingle;
  1841. analyse(expr, 0);
  1842. }
  1843. void noteDataset(IHqlExpression * expr, IHqlExpression * hoisted, bool alwaysHoist)
  1844. {
  1845. unsigned match = matched.findOriginal(expr);
  1846. if (match == NotFound)
  1847. {
  1848. if (!hoisted)
  1849. hoisted = expr;
  1850. CChildDependent & depend = * new CChildDependent(expr, hoisted, alwaysHoist, alwaysSingle, false);
  1851. matched.append(depend);
  1852. }
  1853. else
  1854. {
  1855. CChildDependent & prev = matched.item(match);
  1856. if (alwaysHoist && !prev.alwaysHoist)
  1857. prev.alwaysHoist = true;
  1858. if (alwaysSingle && !prev.isSingleNode)
  1859. prev.isSingleNode = true;
  1860. }
  1861. }
  1862. void noteScalar(IHqlExpression * expr, IHqlExpression * value)
  1863. {
  1864. unsigned match = matched.findOriginal(expr);
  1865. if (match == NotFound)
  1866. {
  1867. OwnedHqlExpr hoisted;
  1868. IHqlExpression * projected = NULL;
  1869. if (value->getOperator() == no_select)
  1870. {
  1871. bool isNew;
  1872. IHqlExpression * row = querySelectorDataset(value, isNew);
  1873. if(isNew || row->isDatarow())
  1874. {
  1875. if (projectSelectorDatasetToField(row))
  1876. projected = value;
  1877. hoisted.set(row);
  1878. }
  1879. else
  1880. {
  1881. //very unusual - possibly a thisnode(myfield) hoisted from an allnodes
  1882. //use a createrow of a single element in the default behavour below
  1883. }
  1884. }
  1885. else if (value->getOperator() == no_createset)
  1886. {
  1887. IHqlExpression * ds = value->queryChild(0);
  1888. IHqlExpression * selected = value->queryChild(1);
  1889. OwnedHqlExpr field;
  1890. //Project down to a single field.so implicit fields can still be optimized
  1891. if (selected->getOperator() == no_select)
  1892. field.set(selected->queryChild(1));
  1893. else
  1894. field.setown(createField(valueId, selected->getType(), NULL));
  1895. OwnedHqlExpr record = createRecord(field);
  1896. OwnedHqlExpr self = getSelf(record);
  1897. OwnedHqlExpr assign = createAssign(createSelectExpr(LINK(self), LINK(field)), LINK(selected));
  1898. OwnedHqlExpr transform = createValue(no_newtransform, makeTransformType(record->getType()), LINK(assign));
  1899. hoisted.setown(createDataset(no_newusertable, LINK(ds), createComma(LINK(record), LINK(transform))));
  1900. }
  1901. if (!hoisted)
  1902. {
  1903. OwnedHqlExpr field = createField(valueId, value->getType(), NULL);
  1904. OwnedHqlExpr record = createRecord(field);
  1905. OwnedHqlExpr self = getSelf(record);
  1906. OwnedHqlExpr assign = createAssign(createSelectExpr(LINK(self), LINK(field)), LINK(value));
  1907. OwnedHqlExpr transform = createValue(no_transform, makeTransformType(record->getType()), LINK(assign));
  1908. hoisted.setown(createRow(no_createrow, LINK(transform)));
  1909. }
  1910. CChildDependent & depend = * new CChildDependent(expr, hoisted, true, true, true);
  1911. depend.projected = projected;
  1912. matched.append(depend);
  1913. }
  1914. }
  1915. protected:
  1916. ChildDependentArray & matched;
  1917. bool alwaysSingle;
  1918. };
  1919. class EclChildSplitPointLocator : public EclHoistLocator
  1920. {
  1921. public:
  1922. EclChildSplitPointLocator(IHqlExpression * _original, HqlExprCopyArray & _selectors, ChildDependentArray & _matches, bool _groupedChildIterators)
  1923. : EclHoistLocator(_matches), selectors(_selectors), groupedChildIterators(_groupedChildIterators)
  1924. {
  1925. original = _original;
  1926. okToSelect = false;
  1927. gathered = false;
  1928. conditionalDepth = 0;
  1929. executedOnce = false;
  1930. switch (original->getOperator())
  1931. {
  1932. case no_call:
  1933. case no_externalcall:
  1934. case no_libraryscopeinstance:
  1935. okToSelect = true;
  1936. break;
  1937. }
  1938. }
  1939. void findSplitPoints(IHqlExpression * expr, unsigned from, unsigned to, bool _alwaysSingle, bool _executedOnce)
  1940. {
  1941. alwaysSingle = _alwaysSingle;
  1942. for (unsigned i=from; i < to; i++)
  1943. {
  1944. IHqlExpression * cur = expr->queryChild(i);
  1945. executedOnce = _executedOnce || cur->isAttribute(); // assume attributes are only executed once.
  1946. findSplitPoints(cur);
  1947. }
  1948. alwaysSingle = false;
  1949. }
  1950. protected:
  1951. void findSplitPoints(IHqlExpression * expr)
  1952. {
  1953. //containsNonActiveDataset() would be nice - but that isn't percolated outside assigns etc.
  1954. if (containsAnyDataset(expr) || containsMustHoist(expr) || !expr->isIndependentOfScope())
  1955. {
  1956. if (!gathered)
  1957. {
  1958. gatherAmbiguousSelectors(original);
  1959. gathered = true;
  1960. }
  1961. analyse(expr, 0);
  1962. }
  1963. }
  1964. bool queryHoistDataset(IHqlExpression * ds)
  1965. {
  1966. bool alwaysHoist = true;
  1967. if (executedOnce)
  1968. {
  1969. if (conditionalDepth != 0)
  1970. alwaysHoist = false;
  1971. }
  1972. return alwaysHoist;
  1973. }
  1974. bool queryNoteDataset(IHqlExpression * ds)
  1975. {
  1976. bool alwaysHoist = queryHoistDataset(ds);
  1977. //MORE: It should be possible to remove this condition, but it causes problems with resourcing hsss.xhql amongst others -> disable for the moment
  1978. if (alwaysHoist)
  1979. noteDataset(ds, ds, alwaysHoist);
  1980. return alwaysHoist;
  1981. }
  1982. virtual void analyseExpr(IHqlExpression * expr)
  1983. {
  1984. if (alreadyVisited(expr))
  1985. return;
  1986. node_operator op = expr->getOperator();
  1987. switch (op)
  1988. {
  1989. case no_select:
  1990. {
  1991. bool isNew;
  1992. IHqlExpression * ds = querySelectorDataset(expr, isNew);
  1993. if (isNew)
  1994. {
  1995. if (isEvaluateable(ds))
  1996. {
  1997. //MORE: Following isn't a very nice test - stops implicit denormalize getting messed up
  1998. if (expr->isDataset())
  1999. break;
  2000. //Debtable.....
  2001. //Don't hoist counts on indexes or dataset - it may mean they are evaluated more frequently than need be.
  2002. //If dependencies and root graphs are handled correctly this could be deleted.
  2003. if (isCompoundAggregate(ds))
  2004. break;
  2005. if (!expr->isDatarow() && !expr->isDataset() && !expr->isDictionary())
  2006. {
  2007. if (queryHoistDataset(ds))
  2008. {
  2009. noteScalar(expr, expr);
  2010. return;
  2011. }
  2012. }
  2013. else
  2014. {
  2015. if (queryNoteDataset(ds))
  2016. return;
  2017. }
  2018. }
  2019. }
  2020. break;
  2021. }
  2022. case no_createset:
  2023. {
  2024. IHqlExpression * ds = expr->queryChild(0);
  2025. if (isEvaluateable(ds))
  2026. {
  2027. if (queryHoistDataset(ds))
  2028. {
  2029. noteScalar(expr, expr);
  2030. //?? queryNoteDataset(ds);
  2031. return;
  2032. }
  2033. }
  2034. break;
  2035. }
  2036. case no_assign:
  2037. {
  2038. IHqlExpression * rhs = expr->queryChild(1);
  2039. //if rhs is a new, evaluatable, dataset then we want to add it
  2040. if ((rhs->isDataset() || rhs->isDictionary()) && isEvaluateable(rhs))
  2041. {
  2042. if (queryNoteDataset(rhs))
  2043. return;
  2044. }
  2045. break;
  2046. }
  2047. case no_sizeof:
  2048. case no_allnodes:
  2049. case no_nohoist:
  2050. case no_forcegraph:
  2051. return;
  2052. case no_globalscope:
  2053. case no_evalonce:
  2054. if (expr->hasAttribute(optAtom) && !expr->isIndependentOfScope())
  2055. break;
  2056. if (expr->isDataset() || expr->isDatarow() || expr->isDictionary())
  2057. noteDataset(expr, expr->queryChild(0), true);
  2058. else
  2059. noteScalar(expr, expr->queryChild(0));
  2060. return;
  2061. case no_thisnode:
  2062. throwUnexpected();
  2063. case no_getgraphresult:
  2064. if (expr->hasAttribute(_streaming_Atom) || expr->hasAttribute(_distributed_Atom))
  2065. {
  2066. noteDataset(expr, expr, true);
  2067. return;
  2068. }
  2069. break;
  2070. case no_getgraphloopresult:
  2071. noteDataset(expr, expr, true);
  2072. return;
  2073. case no_createdictionary:
  2074. if (isEvaluateable(expr) && !isConstantDictionary(expr))
  2075. noteDataset(expr, expr, true);
  2076. return;
  2077. case no_selectnth:
  2078. if (expr->queryChild(1)->isConstant())
  2079. {
  2080. IHqlExpression * ds = expr->queryChild(0);
  2081. switch (ds->getOperator())
  2082. {
  2083. case no_getgraphresult:
  2084. if (!expr->hasAttribute(_streaming_Atom) && !expr->hasAttribute(_distributed_Atom))
  2085. break;
  2086. //fallthrough
  2087. case no_getgraphloopresult:
  2088. noteDataset(expr, expr, true);
  2089. return;
  2090. }
  2091. }
  2092. break;
  2093. }
  2094. bool wasOkToSelect = okToSelect;
  2095. if (expr->isDataset())
  2096. {
  2097. switch (expr->getOperator())
  2098. {
  2099. case no_compound_diskread:
  2100. case no_compound_disknormalize:
  2101. case no_compound_diskaggregate:
  2102. case no_compound_diskcount:
  2103. case no_compound_diskgroupaggregate:
  2104. case no_compound_indexread:
  2105. case no_compound_indexnormalize:
  2106. case no_compound_indexaggregate:
  2107. case no_compound_indexcount:
  2108. case no_compound_indexgroupaggregate:
  2109. case no_compound_childread:
  2110. case no_compound_childnormalize:
  2111. case no_compound_childaggregate:
  2112. case no_compound_childcount:
  2113. case no_compound_childgroupaggregate:
  2114. case no_compound_selectnew:
  2115. case no_compound_inline:
  2116. case no_newkeyindex:
  2117. case no_keyindex:
  2118. case no_table:
  2119. okToSelect = false;
  2120. break;
  2121. }
  2122. if (okToSelect && isEvaluateable(expr))
  2123. {
  2124. if (queryNoteDataset(expr))
  2125. return;
  2126. }
  2127. }
  2128. else
  2129. okToSelect = true;
  2130. switch (op)
  2131. {
  2132. case no_if:
  2133. case no_choose:
  2134. case no_chooseds:
  2135. {
  2136. IHqlExpression * cond = expr->queryChild(0);
  2137. analyseExpr(cond);
  2138. if (expr->isDataset() || expr->isDatarow() || expr->isDictionary() || expr->isAction())
  2139. conditionalDepth++;
  2140. doAnalyseChildren(expr, 1);
  2141. if (expr->isDataset() || expr->isDatarow() || expr->isDictionary() || expr->isAction())
  2142. conditionalDepth--;
  2143. break;
  2144. }
  2145. case no_mapto:
  2146. {
  2147. analyseExpr(expr->queryChild(0));
  2148. if (expr->isDataset() || expr->isDatarow() || expr->isDictionary())
  2149. conditionalDepth++;
  2150. analyseExpr(expr->queryChild(1));
  2151. if (expr->isDataset() || expr->isDatarow() || expr->isDictionary())
  2152. conditionalDepth--;
  2153. break;
  2154. }
  2155. case no_attr_expr:
  2156. //Ignore internal tracking attributes e.g., _selectors_Atom
  2157. if (!isInternalAttributeName(expr->queryName()))
  2158. {
  2159. //Default action for no_attr_expr is to not walk children, but we need to here.
  2160. bool wasExecutedOnce = executedOnce;
  2161. executedOnce = true;
  2162. analyseChildren(expr);
  2163. executedOnce = wasExecutedOnce;
  2164. }
  2165. break;
  2166. default:
  2167. NewHqlTransformer::analyseExpr(expr);
  2168. break;
  2169. }
  2170. okToSelect = wasOkToSelect;
  2171. }
  2172. bool isCompoundAggregate(IHqlExpression * ds)
  2173. {
  2174. return false;
  2175. //Generates worse code unless we take into account whether or not the newDisk operation flags are enabled.
  2176. if (!isTrivialSelectN(ds))
  2177. return false;
  2178. IHqlExpression * agg = ds->queryChild(0);
  2179. if (isSimpleCountAggregate(agg, true))
  2180. return true;
  2181. return false;
  2182. }
  2183. void gatherAmbiguousSelectors(IHqlExpression * expr)
  2184. {
  2185. //Horrible code to try and cope with ambiguous left selectors.
  2186. //o Tree is ambiguous so same child expression can occur in different contexts - so can't depend on the context it is found in to work out if can hoist
  2187. //o If any selector that is hidden within child expressions matches one in scope then can't hoist it.
  2188. //If the current expression creates a selector, then can't hoist anything that depends on it [only add to hidden if in selectors to reduce searching]
  2189. //o Want to hoist as much as possible.
  2190. if (selectors.empty())
  2191. return;
  2192. unsigned first = getFirstActivityArgument(expr);
  2193. unsigned last = first + getNumActivityArguments(expr);
  2194. unsigned max = expr->numChildren();
  2195. unsigned i;
  2196. HqlExprCopyArray hiddenSelectors;
  2197. for (i = 0; i < first; i++)
  2198. expr->queryChild(i)->gatherTablesUsed(&hiddenSelectors, NULL);
  2199. for (i = last; i < max; i++)
  2200. expr->queryChild(i)->gatherTablesUsed(&hiddenSelectors, NULL);
  2201. ForEachItemIn(iSel, selectors)
  2202. {
  2203. IHqlExpression & cur = selectors.item(iSel);
  2204. if (hiddenSelectors.contains(cur))
  2205. ambiguousSelectors.append(cur);
  2206. }
  2207. switch (getChildDatasetType(expr))
  2208. {
  2209. case childdataset_datasetleft:
  2210. case childdataset_left:
  2211. {
  2212. IHqlExpression * ds = expr->queryChild(0);
  2213. IHqlExpression * selSeq = querySelSeq(expr);
  2214. OwnedHqlExpr left = createSelector(no_left, ds, selSeq);
  2215. if (selectors.contains(*left))
  2216. ambiguousSelectors.append(*left);
  2217. break;
  2218. }
  2219. case childdataset_same_left_right:
  2220. case childdataset_top_left_right:
  2221. case childdataset_nway_left_right:
  2222. {
  2223. IHqlExpression * ds = expr->queryChild(0);
  2224. IHqlExpression * selSeq = querySelSeq(expr);
  2225. OwnedHqlExpr left = createSelector(no_left, ds, selSeq);
  2226. OwnedHqlExpr right = createSelector(no_right, ds, selSeq);
  2227. if (selectors.contains(*left))
  2228. ambiguousSelectors.append(*left);
  2229. if (selectors.contains(*right))
  2230. ambiguousSelectors.append(*right);
  2231. break;
  2232. }
  2233. case childdataset_leftright:
  2234. {
  2235. IHqlExpression * leftDs = expr->queryChild(0);
  2236. IHqlExpression * rightDs = expr->queryChild(1);
  2237. IHqlExpression * selSeq = querySelSeq(expr);
  2238. OwnedHqlExpr left = createSelector(no_left, leftDs, selSeq);
  2239. OwnedHqlExpr right = createSelector(no_right, rightDs, selSeq);
  2240. if (selectors.contains(*left))
  2241. ambiguousSelectors.append(*left);
  2242. if (selectors.contains(*right))
  2243. ambiguousSelectors.append(*right);
  2244. break;
  2245. }
  2246. }
  2247. }
  2248. bool isEvaluateable(IHqlExpression * ds, bool ignoreInline = false)
  2249. {
  2250. //Don't hoist an alias - it could create unnecessary duplicate spills - hoist its input
  2251. if (ds->getOperator() == no_dataset_alias)
  2252. return false;
  2253. //Not allowed to hoist
  2254. if (isContextDependent(ds, (conditionalDepth == 0), true))
  2255. return false;
  2256. //MORE: Needs more work for child queries - need a GroupedChildIterator activity
  2257. if (isGrouped(ds) && selectors.ordinality() && !groupedChildIterators)
  2258. return false;
  2259. //Check datasets are available
  2260. HqlExprCopyArray scopeUsed;
  2261. ds->gatherTablesUsed(NULL, &scopeUsed);
  2262. ForEachItemIn(i, scopeUsed)
  2263. {
  2264. IHqlExpression & cur = scopeUsed.item(i);
  2265. if (!selectors.contains(cur))
  2266. return false;
  2267. if (ambiguousSelectors.contains(cur))
  2268. return false;
  2269. }
  2270. if (!isEfficientToHoistDataset(ds, ignoreInline))
  2271. return false;
  2272. return true;
  2273. }
  2274. bool isEfficientToHoistDataset(IHqlExpression * ds, bool ignoreInline) const
  2275. {
  2276. //MORE: This whole function could do with some significant improvements. Whether it is inefficient to hoist
  2277. //depends on at least the following...
  2278. //a) cost of serializing v cost of re-evaluating (which can depend on the engine).
  2279. //b) How many times it will be evaluated in the child context
  2280. if (ds->getOperator() == no_createdictionary)
  2281. return true;
  2282. if (isInlineTrivialDataset(ds))
  2283. return false;
  2284. #ifdef MINIMAL_CHANGES
  2285. if (!ignoreInline)
  2286. {
  2287. //Generally this appears to be better to hoist since it involves calling a transform.
  2288. if (ds->getOperator() == no_dataset_from_transform)
  2289. return true;
  2290. if (canProcessInline(NULL, ds))
  2291. return false;
  2292. }
  2293. #endif
  2294. return true;
  2295. }
  2296. protected:
  2297. IHqlExpression * original;
  2298. HqlExprCopyArray & selectors;
  2299. HqlExprCopyArray ambiguousSelectors;
  2300. unsigned conditionalDepth;
  2301. bool okToSelect;
  2302. bool gathered;
  2303. bool groupedChildIterators;
  2304. bool executedOnce;
  2305. };
  2306. void EclResourcer::gatherChildSplitPoints(IHqlExpression * expr, ResourcerInfo * info, unsigned first, unsigned last)
  2307. {
  2308. //NB: Don't call member functions to ensure correct nesting of transform mutexes.
  2309. EclChildSplitPointLocator locator(expr, activeSelectors, info->childDependents, options.groupedChildIterators);
  2310. unsigned max = expr->numChildren();
  2311. //If child queries are supported then don't hoist the expressions if they might only be evaluated once
  2312. //because they may be conditional
  2313. switch (expr->getOperator())
  2314. {
  2315. case no_setresult:
  2316. case no_selectnth:
  2317. //set results, only done once=>don't hoist conditionals
  2318. locator.findSplitPoints(expr, last, max, true, true);
  2319. return;
  2320. case no_loop:
  2321. if ((options.targetClusterType == ThorLCRCluster) && !options.isChildQuery)
  2322. {
  2323. //This is ugly! The body is executed in parallel, so don't force that as a work unit result
  2324. //It means some child query expressions within loops don't get forced into work unit writes
  2325. //but that just means that the generated code will be not as good as it could be.
  2326. const unsigned bodyArg = 4;
  2327. locator.findSplitPoints(expr, 1, bodyArg, true, false);
  2328. locator.findSplitPoints(expr, bodyArg, bodyArg+1, false, false);
  2329. locator.findSplitPoints(expr, bodyArg+1, max, true, false);
  2330. return;
  2331. }
  2332. break;
  2333. }
  2334. locator.findSplitPoints(expr, 0, first, true, true); // IF() conditions only evaluated once... => don't force
  2335. locator.findSplitPoints(expr, last, max, true, false);
  2336. }
  2337. class EclThisNodeLocator : public EclHoistLocator
  2338. {
  2339. public:
  2340. EclThisNodeLocator(ChildDependentArray & _matches)
  2341. : EclHoistLocator(_matches)
  2342. {
  2343. allNodesDepth = 0;
  2344. }
  2345. protected:
  2346. virtual void analyseExpr(IHqlExpression * expr)
  2347. {
  2348. //NB: This doesn't really work for no_thisnode occurring in multiple contexts. We should probably hoist it from everywhere if it is hoistable from anywhere,
  2349. // although theoretically that gives us problems with ambiguous selectors.
  2350. if (alreadyVisited(expr) || !containsThisNode(expr))
  2351. return;
  2352. node_operator op = expr->getOperator();
  2353. switch (op)
  2354. {
  2355. case no_allnodes:
  2356. allNodesDepth++;
  2357. NewHqlTransformer::analyseExpr(expr);
  2358. allNodesDepth--;
  2359. return;
  2360. case no_thisnode:
  2361. if (allNodesDepth == 0)
  2362. {
  2363. if (expr->isDataset() || expr->isDatarow() || expr->isDictionary())
  2364. noteDataset(expr, expr->queryChild(0), true);
  2365. else
  2366. noteScalar(expr, expr->queryChild(0));
  2367. return;
  2368. }
  2369. allNodesDepth--;
  2370. NewHqlTransformer::analyseExpr(expr);
  2371. allNodesDepth++;
  2372. return;
  2373. }
  2374. NewHqlTransformer::analyseExpr(expr);
  2375. }
  2376. protected:
  2377. unsigned allNodesDepth;
  2378. };
  2379. static bool isPotentialCompoundSteppedIndexRead(IHqlExpression * expr)
  2380. {
  2381. loop
  2382. {
  2383. switch (expr->getOperator())
  2384. {
  2385. case no_compound_diskread:
  2386. case no_compound_disknormalize:
  2387. case no_compound_diskaggregate:
  2388. case no_compound_diskcount:
  2389. case no_compound_diskgroupaggregate:
  2390. case no_compound_childread:
  2391. case no_compound_childnormalize:
  2392. case no_compound_childaggregate:
  2393. case no_compound_childcount:
  2394. case no_compound_childgroupaggregate:
  2395. case no_compound_selectnew:
  2396. case no_compound_inline:
  2397. return false;
  2398. case no_compound_indexread:
  2399. case no_newkeyindex:
  2400. return true;
  2401. case no_getgraphloopresult:
  2402. return true; // Could be an index read in another graph iteration, so don't combine
  2403. case no_keyedlimit:
  2404. case no_preload:
  2405. case no_filter:
  2406. case no_hqlproject:
  2407. case no_newusertable:
  2408. case no_limit:
  2409. case no_sorted:
  2410. case no_preservemeta:
  2411. case no_distributed:
  2412. case no_grouped:
  2413. case no_stepped:
  2414. case no_section:
  2415. case no_sectioninput:
  2416. case no_dataset_alias:
  2417. break;
  2418. case no_choosen:
  2419. {
  2420. IHqlExpression * arg2 = expr->queryChild(2);
  2421. if (arg2 && !arg2->isPure())
  2422. return false;
  2423. break;
  2424. }
  2425. default:
  2426. return false;
  2427. }
  2428. expr = expr->queryChild(0);
  2429. }
  2430. }
  2431. bool EclResourcer::findSplitPoints(IHqlExpression * expr, bool isProjected)
  2432. {
  2433. ResourcerInfo * info = queryResourceInfo(expr);
  2434. if (info && info->visited)
  2435. {
  2436. if (!isProjected)
  2437. info->clearProjected();
  2438. if (info->isAlreadyInScope || info->isActivity || !info->containsActivity)
  2439. return info->containsActivity;
  2440. }
  2441. else
  2442. {
  2443. info = queryCreateResourceInfo(expr);
  2444. info->visited = true;
  2445. if (!isProjected)
  2446. info->clearProjected();
  2447. bool isActivity = true;
  2448. switch (expr->getOperator())
  2449. {
  2450. case no_select:
  2451. //either a select from a setresult or use of a child-dataset
  2452. if (isNewSelector(expr))
  2453. {
  2454. info->containsActivity = findSplitPoints(expr->queryChild(0), false);
  2455. assertex(queryResourceInfo(expr->queryChild(0))->isActivity);
  2456. }
  2457. if (expr->isDataset() || expr->isDatarow())
  2458. {
  2459. info->isActivity = true;
  2460. info->containsActivity = true;
  2461. }
  2462. return info->containsActivity;
  2463. case no_mapto:
  2464. throwUnexpected();
  2465. case no_activerow:
  2466. info->isActivity = true;
  2467. info->containsActivity = false;
  2468. return false;
  2469. case no_rowset: // don't resource this as an activity
  2470. isActivity = false;
  2471. break;
  2472. case no_attr:
  2473. case no_attr_expr:
  2474. case no_attr_link:
  2475. case no_getgraphloopresultset:
  2476. info->isActivity = false;
  2477. info->containsActivity = false;
  2478. return false;
  2479. case no_datasetlist:
  2480. isActivity = false;
  2481. break;
  2482. case no_rowsetrange:
  2483. {
  2484. //Don't resource this as an activity if it is a function of the input graph rows,
  2485. //however we do want to if it is coming from a dataset list.
  2486. IHqlExpression * ds = expr->queryChild(0);
  2487. //MORE: Should walk further down the tree to allow for nested rowsetranges etc.
  2488. if (ds->getOperator() == no_rowset || ds->getOperator() == no_getgraphloopresultset)
  2489. {
  2490. info->isActivity = false;
  2491. info->containsActivity = false;
  2492. return false;
  2493. }
  2494. isActivity = false;
  2495. break;
  2496. }
  2497. }
  2498. ITypeInfo * type = expr->queryType();
  2499. if (!type || type->isScalar())
  2500. return false;
  2501. info->isActivity = isActivity;
  2502. info->containsActivity = true;
  2503. }
  2504. unsigned first = getFirstActivityArgument(expr);
  2505. unsigned last = first + getNumActivityArguments(expr);
  2506. if (options.hoistResourced)
  2507. {
  2508. switch (expr->getOperator())
  2509. {
  2510. case no_allnodes:
  2511. {
  2512. //MORE: This needs to recursively walk and lift any contained no_selfnode, but don't go past another nested no_allnodes;
  2513. EclThisNodeLocator locator(info->childDependents);
  2514. locator.analyseChild(expr->queryChild(0), true);
  2515. break;
  2516. }
  2517. case no_childquery:
  2518. throwUnexpected();
  2519. default:
  2520. {
  2521. for (unsigned idx=first; idx < last; idx++)
  2522. {
  2523. IHqlExpression * cur = expr->queryChild(idx);
  2524. findSplitPoints(cur, false);
  2525. }
  2526. gatherChildSplitPoints(expr, info, first, last);
  2527. break;
  2528. }
  2529. }
  2530. ForEachItemIn(i2, info->childDependents)
  2531. {
  2532. CChildDependent & cur = info->childDependents.item(i2);
  2533. ResourcerInfo * hoistedInfo = queryCreateResourceInfo(cur.hoisted);
  2534. if (cur.projected)
  2535. hoistedInfo->addProjected(cur.projected);
  2536. else
  2537. hoistedInfo->clearProjected();
  2538. if (cur.alwaysHoist)
  2539. findSplitPoints(cur.hoisted, (cur.projected != NULL));
  2540. allChildDependents.append(OLINK(cur));
  2541. }
  2542. }
  2543. else
  2544. {
  2545. for (unsigned idx=first; idx < last; idx++)
  2546. findSplitPoints(expr->queryChild(idx), false);
  2547. }
  2548. return info->containsActivity;
  2549. }
  2550. void EclResourcer::findSplitPoints(HqlExprArray & exprs)
  2551. {
  2552. //Finding items that should be hoisted from child queries, and evaluated at this level is tricky:
  2553. //* After hoisting a child expression, that may in turn contain other child expressions.
  2554. //* Some child expressions are only worth hoisting if they are a simple function of something
  2555. // that will be evaluated here.
  2556. //* If we're creating a single project for each x[1] then that needs to be done after all the
  2557. // expressions to hoist have been found.
  2558. //* The usage counts have to correct - including
  2559. //* The select child dependents for a ds[n] need to be inherited by the project(ds)[n]
  2560. //=>
  2561. //First walk the expression tree gathering all the expressions that will be used.
  2562. //Project the expressions that need projecting.
  2563. //Finally walk the tree again, this time calculating the correct usage counts.
  2564. //
  2565. //On reflection the hoisting and projecting could be implemented as a completely separate transformation.
  2566. //However, it shares a reasonable amount of the logic with the rest of the resourcing, so keep together
  2567. //for the moment.
  2568. ForEachItemIn(idx, exprs)
  2569. findSplitPoints(&exprs.item(idx), false);
  2570. extendSplitPoints();
  2571. projectChildDependents();
  2572. deriveUsageCounts(exprs);
  2573. }
  2574. void EclResourcer::extendSplitPoints()
  2575. {
  2576. //NB: findSplitPoints might call this array to be extended
  2577. for (unsigned i1=0; i1 < allChildDependents.ordinality(); i1++)
  2578. {
  2579. CChildDependent & cur = allChildDependents.item(i1);
  2580. if (!cur.alwaysHoist && isWorthForcingHoist(cur.hoisted))
  2581. findSplitPoints(cur.hoisted, (cur.projected != NULL));
  2582. }
  2583. }
  2584. IHqlExpression * EclResourcer::projectChildDependent(IHqlExpression * expr)
  2585. {
  2586. ResourcerInfo * info = queryResourceInfo(expr);
  2587. if (!info || !info->projectResult || info->projected.empty())
  2588. return expr;
  2589. if (info->projectedExpr)
  2590. return info->projectedExpr;
  2591. assertex(expr->getOperator() == no_selectnth);
  2592. IHqlExpression * row = expr;
  2593. assertex(projectSelectorDatasetToField(row));
  2594. unsigned totalFields = getFieldCount(row->queryRecord());
  2595. if (totalFields == info->projected.ordinality())
  2596. {
  2597. info->clearProjected();
  2598. return expr;
  2599. }
  2600. //Create a projection containing each of the fields that are used from the child queries.
  2601. IHqlExpression * ds = row->queryChild(0);
  2602. HqlExprArray fields;
  2603. HqlExprArray values;
  2604. ForEachItemIn(i, info->projected)
  2605. {
  2606. IHqlExpression * value = &info->projected.item(i);
  2607. IHqlExpression * field = value->queryChild(1);
  2608. LinkedHqlExpr projectedField = field;
  2609. //Check for a very unusual situation where the same field is projected from two different sub records
  2610. while (fields.contains(*projectedField))
  2611. projectedField.setown(cloneFieldMangleName(field)); // Generates a new mangled name each time
  2612. OwnedHqlExpr activeDs = createRow(no_activetable, LINK(ds->queryNormalizedSelector()));
  2613. fields.append(*LINK(projectedField));
  2614. values.append(*replaceSelector(value, row, activeDs));
  2615. }
  2616. OwnedHqlExpr projectedRecord = createRecord(fields);
  2617. OwnedHqlExpr self = getSelf(projectedRecord);
  2618. HqlExprArray assigns;
  2619. ForEachItemIn(i2, fields)
  2620. {
  2621. IHqlExpression * field = &fields.item(i2);
  2622. IHqlExpression * value = &values.item(i2);
  2623. assigns.append(*createAssign(createSelectExpr(LINK(self), LINK(field)), LINK(value)));
  2624. }
  2625. OwnedHqlExpr transform = createValue(no_newtransform, makeTransformType(projectedRecord->getType()), assigns);
  2626. OwnedHqlExpr projectedDs = createDataset(no_newusertable, LINK(ds), createComma(LINK(projectedRecord), LINK(transform)));
  2627. info->projectedExpr.setown(replaceChild(row, 0, projectedDs));
  2628. findSplitPoints(info->projectedExpr, false);
  2629. //Ensure child dependents that are no longer used are not processed - otherwise the usage counts will be wrong.
  2630. ForEachItemIn(ic, info->childDependents)
  2631. {
  2632. CChildDependent & cur = info->childDependents.item(ic);
  2633. //Clearing these ensures that isResourcedActivity returns false;
  2634. cur.hoisted.clear();
  2635. cur.projectedHoisted.clear();
  2636. }
  2637. return info->projectedExpr;
  2638. }
  2639. void EclResourcer::projectChildDependents()
  2640. {
  2641. ForEachItemIn(i2, allChildDependents)
  2642. {
  2643. CChildDependent & cur = allChildDependents.item(i2);
  2644. if (isResourcedActivity(cur.hoisted))
  2645. cur.projectedHoisted.set(projectChildDependent(cur.hoisted));
  2646. }
  2647. }
  2648. void EclResourcer::deriveUsageCounts(IHqlExpression * expr)
  2649. {
  2650. ResourcerInfo * info = queryResourceInfo(expr);
  2651. if (!info || !(info->containsActivity || info->isActivity))
  2652. return;
  2653. bool savedInsideNeverSplit = insideNeverSplit;
  2654. bool savedInsideSteppedNeverSplit = insideSteppedNeverSplit;
  2655. if (insideSteppedNeverSplit && info)
  2656. {
  2657. if (!isPotentialCompoundSteppedIndexRead(expr) && (expr->getOperator() != no_datasetlist) && expr->getOperator() != no_inlinetable)
  2658. insideSteppedNeverSplit = false;
  2659. }
  2660. if (info->numUses)
  2661. {
  2662. if (insideNeverSplit || insideSteppedNeverSplit)
  2663. info->neverSplit = true;
  2664. if (info->isAlreadyInScope || info->isActivity || !info->containsActivity)
  2665. {
  2666. info->numUses++;
  2667. return;
  2668. }
  2669. }
  2670. else
  2671. {
  2672. info->numUses++;
  2673. if (insideNeverSplit || insideSteppedNeverSplit)
  2674. info->neverSplit = true;
  2675. switch (expr->getOperator())
  2676. {
  2677. case no_select:
  2678. //either a select from a setresult or use of a child-dataset
  2679. if (isNewSelector(expr))
  2680. deriveUsageCounts(expr->queryChild(0));
  2681. return;
  2682. case no_activerow:
  2683. case no_attr:
  2684. case no_attr_expr:
  2685. case no_attr_link:
  2686. case no_rowset: // don't resource this as an activity
  2687. case no_getgraphloopresultset:
  2688. return;
  2689. case no_keyedlimit:
  2690. if (options.preventKeyedSplit)
  2691. insideNeverSplit = true;
  2692. break;
  2693. case no_filter:
  2694. if (options.preventKeyedSplit && filterIsKeyed(expr))
  2695. insideNeverSplit = true;
  2696. else
  2697. {
  2698. LinkedHqlExpr invariant;
  2699. OwnedHqlExpr cond = extractFilterConditions(invariant, expr, expr->queryNormalizedSelector(), false);
  2700. if (invariant)
  2701. info->isConditionalFilter = true;
  2702. }
  2703. break;
  2704. case no_hqlproject:
  2705. case no_newusertable:
  2706. case no_aggregate:
  2707. case no_newaggregate:
  2708. if (options.preventKeyedSplit && expr->hasAttribute(keyedAtom))
  2709. insideNeverSplit = true;
  2710. break;
  2711. case no_stepped:
  2712. case no_mergejoin:
  2713. case no_nwayjoin:
  2714. if (options.preventSteppedSplit)
  2715. insideSteppedNeverSplit = true;
  2716. break;
  2717. case no_compound_diskread:
  2718. case no_compound_disknormalize:
  2719. case no_compound_diskaggregate:
  2720. case no_compound_diskcount:
  2721. case no_compound_diskgroupaggregate:
  2722. case no_compound_indexread:
  2723. case no_compound_indexnormalize:
  2724. case no_compound_indexaggregate:
  2725. case no_compound_indexcount:
  2726. case no_compound_indexgroupaggregate:
  2727. case no_compound_childread:
  2728. case no_compound_childnormalize:
  2729. case no_compound_childaggregate:
  2730. case no_compound_childcount:
  2731. case no_compound_childgroupaggregate:
  2732. case no_compound_selectnew:
  2733. case no_compound_inline:
  2734. insideNeverSplit = true;
  2735. break;
  2736. }
  2737. ITypeInfo * type = expr->queryType();
  2738. if (!type || type->isScalar())
  2739. {
  2740. insideNeverSplit = savedInsideNeverSplit;
  2741. insideSteppedNeverSplit = savedInsideSteppedNeverSplit;
  2742. return;
  2743. }
  2744. }
  2745. unsigned first = getFirstActivityArgument(expr);
  2746. unsigned last = first + getNumActivityArguments(expr);
  2747. for (unsigned idx=first; idx < last; idx++)
  2748. deriveUsageCounts(expr->queryChild(idx));
  2749. insideNeverSplit = savedInsideNeverSplit;
  2750. insideSteppedNeverSplit = savedInsideSteppedNeverSplit;
  2751. }
  2752. void EclResourcer::deriveUsageCounts(const HqlExprArray & exprs)
  2753. {
  2754. ForEachItemIn(idx2, exprs)
  2755. deriveUsageCounts(&exprs.item(idx2));
  2756. ForEachItemIn(i2, allChildDependents)
  2757. {
  2758. CChildDependent & cur = allChildDependents.item(i2);
  2759. if (isResourcedActivity(cur.projectedHoisted))
  2760. deriveUsageCounts(cur.projectedHoisted);
  2761. }
  2762. }
  2763. //------------------------------------------------------------------------------------------
  2764. // PASS2: Actually create the subgraphs based on splitters.
  2765. void EclResourcer::createInitialGraph(IHqlExpression * expr, IHqlExpression * owner, ResourceGraphInfo * ownerGraph, LinkKind linkKind, bool forceNewGraph)
  2766. {
  2767. ResourcerInfo * info = queryResourceInfo(expr);
  2768. if (!info || !info->containsActivity)
  2769. return;
  2770. LinkKind childLinkKind = UnconditionalLink;
  2771. Linked<ResourceGraphInfo> thisGraph = ownerGraph;
  2772. bool forceNewChildGraph = false;
  2773. if (info->isActivity)
  2774. {
  2775. //Need to ensure no_libraryselects are not separated from the no_libraryscopeinstance
  2776. //so ensure they are placed in the same graph.
  2777. node_operator op = expr->getOperator();
  2778. if (op == no_libraryselect)
  2779. {
  2780. ResourcerInfo * moduleInfo = queryResourceInfo(expr->queryChild(1));
  2781. if (!info->graph && moduleInfo->graph)
  2782. info->graph.set(moduleInfo->graph);
  2783. }
  2784. if (info->graph)
  2785. {
  2786. connectGraphs(info->graph, expr, ownerGraph, owner, linkKind);
  2787. info->numExternalUses++;
  2788. return;
  2789. }
  2790. unsigned numUses = info->numUses;
  2791. switch (op)
  2792. {
  2793. case no_libraryscopeinstance:
  2794. numUses = 1;
  2795. break;
  2796. case no_libraryselect:
  2797. forceNewGraph = true;
  2798. break;
  2799. }
  2800. if (!ownerGraph || numUses > 1 || (linkKind != UnconditionalLink) || forceNewGraph)
  2801. {
  2802. thisGraph.setown(createGraph());
  2803. connectGraphs(thisGraph, expr, ownerGraph, owner, linkKind);
  2804. info->numExternalUses++;
  2805. if (!ownerGraph && sequential)
  2806. thisGraph->hasSequentialSource = true;
  2807. }
  2808. info->graph.set(thisGraph);
  2809. if (info->isRootActivity)
  2810. thisGraph->hasRootActivity = true;
  2811. switch (expr->getOperator())
  2812. {
  2813. case no_compound:
  2814. //NB: First argument is forced into a separate graph
  2815. if (options.convertCompoundToExecuteWhen)
  2816. {
  2817. createInitialGraph(expr->queryChild(0), expr, thisGraph, UnconditionalLink, true);
  2818. }
  2819. else
  2820. {
  2821. HqlExprArray actions;
  2822. expr->queryChild(0)->unwindList(actions, no_actionlist);
  2823. ForEachItemIn(i, actions)
  2824. createInitialGraph(&actions.item(i), expr, NULL, UnconditionalLink, true);
  2825. }
  2826. createInitialGraph(expr->queryChild(1), expr, thisGraph, UnconditionalLink, false);
  2827. return;
  2828. case no_executewhen:
  2829. {
  2830. bool newGraph = expr->isAction() && (options.targetClusterType == HThorCluster);
  2831. createInitialGraph(expr->queryChild(0), expr, thisGraph, UnconditionalLink, newGraph);
  2832. createInitialGraph(expr->queryChild(1), expr, thisGraph, UnconditionalLink, true);
  2833. return;
  2834. }
  2835. case no_keyindex:
  2836. case no_newkeyindex:
  2837. return;
  2838. case no_parallel:
  2839. case no_actionlist:
  2840. {
  2841. ForEachChild(i, expr)
  2842. createInitialGraph(expr->queryChild(i), expr, thisGraph, UnconditionalLink, options.actionLinkInNewGraph);
  2843. return;
  2844. }
  2845. case no_if:
  2846. case no_choose:
  2847. case no_chooseds:
  2848. //conditional nodes, the child branches are marked as conditional
  2849. childLinkKind = UnconditionalLink;
  2850. thisGraph->mergedConditionSource = true;
  2851. if (!options.noConditionalLinks || (expr->isAction() && options.actionLinkInNewGraph))
  2852. forceNewChildGraph = true;
  2853. break;
  2854. case no_filter:
  2855. if (info->isConditionalFilter)
  2856. {
  2857. thisGraph->mergedConditionSource = true;
  2858. if (!options.noConditionalLinks)
  2859. forceNewChildGraph = true;
  2860. }
  2861. break;
  2862. // case no_nonempty:
  2863. case no_sequential:
  2864. case no_orderedactionlist:
  2865. {
  2866. unsigned first = getFirstActivityArgument(expr);
  2867. unsigned last = first + getNumActivityArguments(expr);
  2868. createInitialGraph(expr->queryChild(first), expr, thisGraph, SequenceLink, options.actionLinkInNewGraph);
  2869. for (unsigned idx = first+1; idx < last; idx++)
  2870. createInitialGraph(expr->queryChild(idx), expr, thisGraph, SequenceLink, options.actionLinkInNewGraph);
  2871. return;
  2872. }
  2873. case no_case:
  2874. case no_map:
  2875. {
  2876. throwUnexpected();
  2877. }
  2878. case no_output:
  2879. {
  2880. //Tag the inputs to an output statement, so that if a spill was going to occur we read
  2881. //from the output file instead of spilling.
  2882. //Needs the grouping to be saved in the same way. Could cope with compressed matching, but not
  2883. //much point - since fairly unlikely.
  2884. IHqlExpression * filename = expr->queryChild(1);
  2885. if (filename && (filename->getOperator() == no_constant) && !expr->hasAttribute(xmlAtom) && !expr->hasAttribute(csvAtom))
  2886. {
  2887. IHqlExpression * dataset = expr->queryChild(0);
  2888. if (expr->hasAttribute(groupedAtom) == isGrouped(dataset))
  2889. {
  2890. StringBuffer filenameText;
  2891. filename->queryValue()->getStringValue(filenameText);
  2892. ResourcerInfo * childInfo = queryResourceInfo(dataset);
  2893. if (!childInfo->linkedFromChild && !isUpdatedConditionally(expr))
  2894. childInfo->outputToUseForSpill = expr;
  2895. }
  2896. }
  2897. if (isUpdatedConditionally(expr))
  2898. thisGraph->mergedConditionSource = true;
  2899. break;
  2900. }
  2901. case no_buildindex:
  2902. if (isUpdatedConditionally(expr))
  2903. thisGraph->mergedConditionSource = true;
  2904. break;
  2905. }
  2906. }
  2907. unsigned first = getFirstActivityArgument(expr);
  2908. unsigned last = first + getNumActivityArguments(expr);
  2909. for (unsigned idx = first; idx < last; idx++)
  2910. createInitialGraph(expr->queryChild(idx), expr, thisGraph, childLinkKind, forceNewChildGraph);
  2911. ForEachItemIn(i2, info->childDependents)
  2912. {
  2913. CChildDependent & cur = info->childDependents.item(i2);
  2914. if (isResourcedActivity(cur.projectedHoisted))
  2915. createInitialGraph(cur.projectedHoisted, expr, thisGraph, SequenceLink, true);
  2916. }
  2917. }
  2918. void EclResourcer::createInitialGraphs(HqlExprArray & exprs)
  2919. {
  2920. ForEachItemIn(idx, exprs)
  2921. {
  2922. IHqlExpression * rootExpr = &exprs.item(idx);
  2923. ResourcerInfo * info = queryResourceInfo(rootExpr);
  2924. assertex(info->isActivity);
  2925. info->setRootActivity();
  2926. createInitialGraph(rootExpr, NULL, NULL, UnconditionalLink, false);
  2927. }
  2928. }
  2929. void EclResourcer::createInitialRemoteGraph(IHqlExpression * expr, IHqlExpression * owner, ResourceGraphInfo * ownerGraph, bool forceNewGraph)
  2930. {
  2931. ResourcerInfo * info = queryResourceInfo(expr);
  2932. if (!info || !info->containsActivity)
  2933. return;
  2934. Linked<ResourceGraphInfo> thisGraph = ownerGraph;
  2935. if (info->isActivity)
  2936. {
  2937. if (info->graph)
  2938. {
  2939. connectGraphs(info->graph, expr, ownerGraph, owner, UnconditionalLink);
  2940. info->numExternalUses++;
  2941. return;
  2942. }
  2943. if (!ownerGraph || forceNewGraph)
  2944. {
  2945. thisGraph.setown(createGraph());
  2946. connectGraphs(thisGraph, expr, ownerGraph, owner, UnconditionalLink);
  2947. info->numExternalUses++;
  2948. }
  2949. info->graph.set(thisGraph);
  2950. switch (expr->getOperator())
  2951. {
  2952. case no_compound:
  2953. if (options.convertCompoundToExecuteWhen)
  2954. createInitialRemoteGraph(expr->queryChild(0), expr, thisGraph, true);
  2955. else
  2956. createInitialRemoteGraph(expr->queryChild(0), expr, NULL, true);
  2957. createInitialRemoteGraph(expr->queryChild(1), expr, thisGraph, false);
  2958. return;
  2959. case no_executewhen:
  2960. createInitialRemoteGraph(expr->queryChild(0), expr, thisGraph, false);
  2961. createInitialRemoteGraph(expr->queryChild(1), expr, thisGraph, true);
  2962. return;
  2963. }
  2964. }
  2965. unsigned first = getFirstActivityArgument(expr);
  2966. unsigned last = first + getNumActivityArguments(expr);
  2967. for (unsigned idx = first; idx < last; idx++)
  2968. createInitialRemoteGraph(expr->queryChild(idx), expr, thisGraph, false);
  2969. }
  2970. void EclResourcer::createInitialRemoteGraphs(HqlExprArray & exprs)
  2971. {
  2972. ForEachItemIn(idx, exprs)
  2973. createInitialRemoteGraph(&exprs.item(idx), NULL, NULL, false);
  2974. }
  2975. //------------------------------------------------------------------------------------------
  2976. // PASS3: Tag graphs/links that are conditional or unconditional
  2977. void EclResourcer::markChildDependentsAsUnconditional(ResourcerInfo * info, IHqlExpression * condition)
  2978. {
  2979. if (options.hoistResourced)
  2980. {
  2981. ForEachItemIn(i2, info->childDependents)
  2982. {
  2983. CChildDependent & cur = info->childDependents.item(i2);
  2984. if (isResourcedActivity(cur.projectedHoisted))
  2985. markAsUnconditional(cur.projectedHoisted, NULL, condition);
  2986. }
  2987. }
  2988. }
  2989. void EclResourcer::markAsUnconditional(IHqlExpression * expr, ResourceGraphInfo * ownerGraph, IHqlExpression * condition)
  2990. {
  2991. ResourcerInfo * info = queryResourceInfo(expr);
  2992. if (!info || !info->containsActivity)
  2993. return;
  2994. if (!info->isActivity)
  2995. {
  2996. unsigned first = getFirstActivityArgument(expr);
  2997. unsigned last = first + getNumActivityArguments(expr);
  2998. for (unsigned idx=first; idx < last; idx++)
  2999. markAsUnconditional(expr->queryChild(idx), ownerGraph, condition);
  3000. return;
  3001. }
  3002. if (condition)
  3003. if (info->addCondition(condition))
  3004. condition = NULL;
  3005. if (info->pathToExpr == ResourcerInfo::PathUnconditional)
  3006. return;
  3007. if ((info->pathToExpr == ResourcerInfo::PathConditional) && condition)
  3008. {
  3009. if (targetClusterType == RoxieCluster)
  3010. {
  3011. if (spillMultiCondition)
  3012. {
  3013. if (info->graph != ownerGraph)
  3014. info->graph->isUnconditional = true;
  3015. }
  3016. return;
  3017. }
  3018. else
  3019. {
  3020. if (info->graph != ownerGraph)
  3021. return;
  3022. }
  3023. }
  3024. bool wasConditional = (info->pathToExpr == ResourcerInfo::PathConditional);
  3025. if (!condition)
  3026. {
  3027. info->pathToExpr = ResourcerInfo::PathUnconditional;
  3028. if (info->graph != ownerGraph)
  3029. info->graph->isUnconditional = true;
  3030. }
  3031. else
  3032. info->pathToExpr = ResourcerInfo::PathConditional;
  3033. node_operator op = expr->getOperator();
  3034. switch (op)
  3035. {
  3036. case no_if:
  3037. case no_choose:
  3038. case no_chooseds:
  3039. if (options.noConditionalLinks)
  3040. break;
  3041. if (condition)
  3042. markCondition(expr, condition, wasConditional);
  3043. else
  3044. {
  3045. //This list is processed in a second phase.
  3046. if (rootConditions.find(*expr) == NotFound)
  3047. rootConditions.append(*LINK(expr));
  3048. }
  3049. markChildDependentsAsUnconditional(info, condition);
  3050. return;
  3051. case no_filter:
  3052. if (!info->isConditionalFilter || options.noConditionalLinks)
  3053. break;
  3054. if (condition)
  3055. markCondition(expr, condition, wasConditional);
  3056. else
  3057. {
  3058. //This list is processed in a second phase.
  3059. if (rootConditions.find(*expr) == NotFound)
  3060. rootConditions.append(*LINK(expr));
  3061. }
  3062. markChildDependentsAsUnconditional(info, condition);
  3063. return;
  3064. case no_compound:
  3065. if (!options.convertCompoundToExecuteWhen)
  3066. {
  3067. HqlExprArray actions;
  3068. expr->queryChild(0)->unwindList(actions, no_actionlist);
  3069. ForEachItemIn(i, actions)
  3070. markAsUnconditional(&actions.item(i), NULL, NULL);
  3071. markAsUnconditional(expr->queryChild(1), ownerGraph, condition);
  3072. return;
  3073. }
  3074. break;
  3075. case no_sequential:
  3076. case no_orderedactionlist:
  3077. // case no_nonempty:
  3078. if (!options.isChildQuery)
  3079. {
  3080. unsigned first = getFirstActivityArgument(expr);
  3081. unsigned last = first + getNumActivityArguments(expr);
  3082. IHqlExpression * child0 = expr->queryChild(0);
  3083. markAsUnconditional(child0, info->graph, condition);
  3084. queryResourceInfo(child0)->graph->hasConditionSource = true; // force it to generate even if contains something very simple e.g., null action
  3085. for (unsigned idx = first+1; idx < last; idx++)
  3086. {
  3087. OwnedHqlExpr tag = createAttribute(instanceAtom, LINK(expr), getSizetConstant(idx), LINK(condition));
  3088. IHqlExpression * child = expr->queryChild(idx);
  3089. markAsUnconditional(child, queryResourceInfo(child)->graph, tag);
  3090. queryResourceInfo(child)->setConditionSource(tag, !wasConditional);
  3091. }
  3092. markChildDependentsAsUnconditional(info, condition);
  3093. return;
  3094. }
  3095. break;
  3096. case no_case:
  3097. case no_map:
  3098. UNIMPLEMENTED;
  3099. }
  3100. unsigned first = getFirstActivityArgument(expr);
  3101. unsigned last = first + getNumActivityArguments(expr);
  3102. for (unsigned idx=first; idx < last; idx++)
  3103. markAsUnconditional(expr->queryChild(idx), info->graph, condition);
  3104. markChildDependentsAsUnconditional(info, condition);
  3105. }
  3106. void EclResourcer::markConditionBranch(unsigned childIndex, IHqlExpression * expr, IHqlExpression * condition, bool wasConditional)
  3107. {
  3108. IHqlExpression * child = queryRealChild(expr, childIndex);
  3109. if (child)
  3110. {
  3111. OwnedHqlExpr tag;
  3112. if (expr->getOperator() == no_if)
  3113. tag.setown(createAttribute(((childIndex==1) ? trueAtom : falseAtom), LINK(expr), LINK(condition)));
  3114. else
  3115. tag.setown(createAttribute(trueAtom, LINK(expr), LINK(condition), getSizetConstant(childIndex)));
  3116. markAsUnconditional(child, queryResourceInfo(child)->graph, tag);
  3117. queryResourceInfo(child)->setConditionSource(tag, !wasConditional);
  3118. }
  3119. }
  3120. void EclResourcer::markCondition(IHqlExpression * expr, IHqlExpression * condition, bool wasConditional)
  3121. {
  3122. if (expr->getOperator() == no_filter)
  3123. {
  3124. markConditionBranch(0, expr, condition, wasConditional);
  3125. }
  3126. else
  3127. {
  3128. ForEachChildFrom(i, expr, 1)
  3129. markConditionBranch(i, expr, condition, wasConditional);
  3130. }
  3131. }
  3132. void EclResourcer::markConditions(HqlExprArray & exprs)
  3133. {
  3134. ForEachItemIn(idx, exprs)
  3135. markAsUnconditional(&exprs.item(idx), NULL, NULL);
  3136. ForEachItemIn(idx2, rootConditions)
  3137. markCondition(&rootConditions.item(idx2), NULL, false);
  3138. }
  3139. //------------------------------------------------------------------------------------------
  3140. // PASS4: Split subgraphs based on resource requirements for activities
  3141. //This will need to be improved if we allow backtracking to get the best combination of activities to fit in the subgraph
  3142. void EclResourcer::createResourceSplit(IHqlExpression * expr, IHqlExpression * owner, ResourceGraphInfo * ownerNewGraph, ResourceGraphInfo * originalGraph)
  3143. {
  3144. ResourcerInfo * info = queryResourceInfo(expr);
  3145. info->graph.setown(createGraph());
  3146. info->graph->isUnconditional = originalGraph->isUnconditional;
  3147. changeGraph(expr, info->graph);
  3148. connectGraphs(info->graph, expr, ownerNewGraph, owner, UnconditionalLink);
  3149. info->numExternalUses++;
  3150. }
  3151. void EclResourcer::getResources(IHqlExpression * expr, CResources & exprResources)
  3152. {
  3153. ::getResources(expr, exprResources, options);
  3154. }
  3155. bool EclResourcer::calculateResourceSpillPoints(IHqlExpression * expr, ResourceGraphInfo * graph, CResources & resourcesSoFar, bool hasGoodSpillPoint, bool canSpill)
  3156. {
  3157. ResourcerInfo * info = queryResourceInfo(expr);
  3158. if (!info || !info->containsActivity)
  3159. return true;
  3160. if (!info->isActivity)
  3161. {
  3162. unsigned first = getFirstActivityArgument(expr);
  3163. unsigned last = first + getNumActivityArguments(expr);
  3164. if (last - first == 1)
  3165. return calculateResourceSpillPoints(expr->queryChild(first), graph, resourcesSoFar, hasGoodSpillPoint, canSpill);
  3166. for (unsigned idx = first; idx < last; idx++)
  3167. calculateResourceSpillPoints(expr->queryChild(idx), graph, resourcesSoFar, false, canSpill);
  3168. return true;
  3169. }
  3170. if (info->graph != graph)
  3171. return true;
  3172. CResources exprResources(clusterSize);
  3173. getResources(expr, exprResources);
  3174. info->isSpillPoint = false;
  3175. Owned<CResources> curResources = LINK(&resourcesSoFar);
  3176. if (resourcesSoFar.addExceeds(exprResources, *resourceLimit))
  3177. {
  3178. if (hasGoodSpillPoint)
  3179. return false;
  3180. info->isSpillPoint = true;
  3181. spilled = true;
  3182. curResources.setown(new CResources(clusterSize));
  3183. if (exprResources.exceeds(*resourceLimit))
  3184. throwError2(HQLERR_CannotResourceActivity, getOpString(expr->getOperator()), clusterSize);
  3185. }
  3186. if (options.minimizeSkewBeforeSpill)
  3187. {
  3188. if (canSpill && heavyweightAndReducesSizeOrSkew(expr))
  3189. {
  3190. //if the input activity is going to cause us to run out of resources, then it is going to be better to split here than anywhere else
  3191. //this code may conceivably cause extra spills far away because the hash distributes are moved.
  3192. IHqlExpression * childExpr = expr->queryChild(0);
  3193. ResourcerInfo * childInfo = queryResourceInfo(childExpr);
  3194. if (childInfo->graph == graph)
  3195. {
  3196. CResources childResources(clusterSize);
  3197. getResources(childExpr, childResources);
  3198. childResources.add(exprResources);
  3199. if (curResources->addExceeds(childResources, *resourceLimit))
  3200. {
  3201. info->isSpillPoint = true;
  3202. spilled = true;
  3203. calculateResourceSpillPoints(childExpr, graph, exprResources, false, true);
  3204. return true;
  3205. }
  3206. }
  3207. //otherwise continue as normal.
  3208. }
  3209. }
  3210. curResources->add(exprResources);
  3211. unsigned first = getFirstActivityArgument(expr);
  3212. unsigned last = first + getNumActivityArguments(expr);
  3213. if (hasGoodSpillPoint)
  3214. {
  3215. if (exprResources.resource[RESheavy] || exprResources.resource[REShashdist] || last-first != 1)
  3216. hasGoodSpillPoint = false;
  3217. }
  3218. else if (!info->isSpillPoint && canSpill)
  3219. {
  3220. if (lightweightAndReducesDatasetSize(expr) || queryHint(expr, spillAtom))
  3221. {
  3222. CResources savedResources(*curResources);
  3223. if (!calculateResourceSpillPoints(expr->queryChild(0), graph, *curResources, true, true))
  3224. {
  3225. curResources->set(savedResources);
  3226. info->isSpillPoint = true;
  3227. spilled = true;
  3228. calculateResourceSpillPoints(expr->queryChild(0), graph, exprResources, false, true);
  3229. }
  3230. return true;
  3231. }
  3232. }
  3233. node_operator op = expr->getOperator();
  3234. if ((op == no_if) || (op == no_choose) || (op == no_chooseds))
  3235. {
  3236. //For conditions, spill on intersection of resources used, not union.
  3237. CResources savedResources(*curResources);
  3238. if (!calculateResourceSpillPoints(expr->queryChild(1), graph, *curResources, hasGoodSpillPoint, true))
  3239. return false;
  3240. ForEachChildFrom(i, expr, 2)
  3241. {
  3242. if (expr->queryChild(i)->isAttribute())
  3243. continue;
  3244. if (!calculateResourceSpillPoints(expr->queryChild(i), graph, savedResources, hasGoodSpillPoint, true))
  3245. return false;
  3246. curResources->maximize(savedResources);
  3247. }
  3248. }
  3249. else
  3250. {
  3251. for (unsigned idx = first; idx < last; idx++)
  3252. if (!calculateResourceSpillPoints(expr->queryChild(idx), graph, *curResources, hasGoodSpillPoint, true))
  3253. return false;
  3254. }
  3255. return true;
  3256. }
  3257. void EclResourcer::insertResourceSpillPoints(IHqlExpression * expr, IHqlExpression * owner, ResourceGraphInfo * ownerOriginalGraph, ResourceGraphInfo * ownerNewGraph)
  3258. {
  3259. ResourcerInfo * info = queryResourceInfo(expr);
  3260. if (!info || !info->containsActivity)
  3261. return;
  3262. if (!info->isActivity)
  3263. {
  3264. unsigned first = getFirstActivityArgument(expr);
  3265. unsigned last = first + getNumActivityArguments(expr);
  3266. for (unsigned idx = first; idx < last; idx++)
  3267. insertResourceSpillPoints(expr->queryChild(idx), expr, ownerOriginalGraph, ownerNewGraph);
  3268. return;
  3269. }
  3270. ResourceGraphInfo * originalGraph = info->graph; //NB: Graph will never cease to exist, so don't need to link.
  3271. if (originalGraph != ownerOriginalGraph)
  3272. return;
  3273. if (info->isSpillPoint)
  3274. createResourceSplit(expr, owner, ownerNewGraph, originalGraph);
  3275. else if (info->graph != ownerNewGraph)
  3276. changeGraph(expr, ownerNewGraph);
  3277. CResources exprResources(clusterSize);
  3278. getResources(expr, exprResources);
  3279. bool ok = info->graph->allocateResources(exprResources, *resourceLimit);
  3280. assertex(ok);
  3281. node_operator op = expr->getOperator();
  3282. if ((op == no_if) || (op == no_choose) || (op == no_chooseds))
  3283. {
  3284. CResources savedResources(info->graph->resources);
  3285. insertResourceSpillPoints(expr->queryChild(1), expr, originalGraph, info->graph);
  3286. ForEachChildFrom(i, expr, 2)
  3287. {
  3288. if (expr->queryChild(i)->isAttribute())
  3289. continue;
  3290. CResources branchResources(info->graph->resources);
  3291. info->graph->resources.set(savedResources);
  3292. insertResourceSpillPoints(expr->queryChild(i), expr, originalGraph, info->graph);
  3293. info->graph->resources.maximize(branchResources);
  3294. }
  3295. }
  3296. else
  3297. {
  3298. unsigned first = getFirstActivityArgument(expr);
  3299. unsigned last = first + getNumActivityArguments(expr);
  3300. for (unsigned idx = first; idx < last; idx++)
  3301. insertResourceSpillPoints(expr->queryChild(idx), expr, originalGraph, info->graph);
  3302. }
  3303. }
  3304. void EclResourcer::resourceSubGraph(ResourceGraphInfo * graph)
  3305. {
  3306. if (graph->beenResourced)
  3307. return;
  3308. graph->beenResourced = true;
  3309. ForEachItemIn(idx, graph->sources)
  3310. resourceSubGraph(graph->sources.item(idx).sourceGraph);
  3311. IHqlExpression * sourceNode = graph->sinks.item(0).sourceNode->queryBody();
  3312. #ifdef _DEBUG
  3313. //Sanity check, ensure there is only a single sink for this graph.
  3314. //However because libraryselects are tightly bound to their library instance there may be multiple library selects.
  3315. //They won't affect the resourcing though, since they'll plug into the same library instance, and the selects use no resources.
  3316. ForEachItemIn(idx2, graph->sinks)
  3317. {
  3318. IHqlExpression * thisSink = graph->sinks.item(idx2).sourceNode->queryBody();
  3319. if (thisSink->getOperator() != no_libraryselect)
  3320. assertex(thisSink == sourceNode);
  3321. }
  3322. #endif
  3323. spilled = false;
  3324. CResources resources(clusterSize);
  3325. calculateResourceSpillPoints(sourceNode, graph, resources, false, false);
  3326. if (spilled)
  3327. insertResourceSpillPoints(sourceNode, NULL, graph, graph);
  3328. else
  3329. graph->resources.set(resources);
  3330. }
  3331. void EclResourcer::resourceSubGraphs(HqlExprArray & exprs)
  3332. {
  3333. ForEachItemIn(idx, graphs)
  3334. resourceSubGraph(&graphs.item(idx));
  3335. }
  3336. //------------------------------------------------------------------------------------------
  3337. // PASS5: Link subrgaphs with dependency information so they don't get merged by accident.
  3338. void EclResourcer::addDependencySource(IHqlExpression * search, ResourceGraphInfo * curGraph, IHqlExpression * expr)
  3339. {
  3340. //MORE: Should we check this doesn't already exist?
  3341. dependencySource.search.append(*LINK(search));
  3342. dependencySource.graphs.append(*LINK(curGraph));
  3343. dependencySource.exprs.append(*LINK(expr));
  3344. }
  3345. void EclResourcer::addDependencyUse(IHqlExpression * search, ResourceGraphInfo * curGraph, IHqlExpression * expr)
  3346. {
  3347. unsigned index = dependencySource.search.find(*search);
  3348. if (index != NotFound)
  3349. {
  3350. if (&dependencySource.graphs.item(index) == curGraph)
  3351. {
  3352. //Don't give a warning if get/set is within the same activity (e.g., within a local())
  3353. if (&dependencySource.exprs.item(index) != expr)
  3354. //errors->reportWarning(HQLWRN_RecursiveDependendencies, HQLWRN_RecursiveDependendencies_Text, *codeGeneratorAtom, 0, 0, 0);
  3355. errors->reportError(HQLWRN_RecursiveDependendencies, HQLWRN_RecursiveDependendencies_Text, codeGeneratorId->str(), 0, 0, 0);
  3356. }
  3357. else
  3358. {
  3359. ResourceGraphLink * link = new ResourceGraphDependencyLink(&dependencySource.graphs.item(index), &dependencySource.exprs.item(index), curGraph, expr);
  3360. curGraph->dependsOn.append(*link);
  3361. links.append(*link);
  3362. }
  3363. }
  3364. }
  3365. void EclResourcer::addRefExprDependency(IHqlExpression * expr, ResourceGraphInfo * curGraph, IHqlExpression * activityExpr)
  3366. {
  3367. IHqlExpression * filename = queryTableFilename(expr);
  3368. if (filename)
  3369. {
  3370. OwnedHqlExpr value = createAttribute(fileAtom, getNormalizedFilename(filename));
  3371. addDependencySource(value, curGraph, activityExpr);
  3372. }
  3373. }
  3374. bool EclResourcer::addExprDependency(IHqlExpression * expr, ResourceGraphInfo * curGraph, IHqlExpression * activityExpr)
  3375. {
  3376. switch (expr->getOperator())
  3377. {
  3378. case no_buildindex:
  3379. case no_output:
  3380. {
  3381. IHqlExpression * filename = queryRealChild(expr, 1);
  3382. if (filename)
  3383. {
  3384. switch (filename->getOperator())
  3385. {
  3386. case no_pipe:
  3387. // allWritten = true;
  3388. break;
  3389. default:
  3390. OwnedHqlExpr value = createAttribute(fileAtom, getNormalizedFilename(filename));
  3391. addDependencySource(value, curGraph, activityExpr);
  3392. break;
  3393. }
  3394. }
  3395. else
  3396. {
  3397. IHqlExpression * seq = querySequence(expr);
  3398. assertex(seq && seq->queryValue());
  3399. IHqlExpression * name = queryResultName(expr);
  3400. OwnedHqlExpr value = createAttribute(resultAtom, LINK(seq), LINK(name));
  3401. addDependencySource(value, curGraph, activityExpr);
  3402. }
  3403. return true;
  3404. }
  3405. case no_keydiff:
  3406. {
  3407. addRefExprDependency(expr->queryChild(0), curGraph, activityExpr);
  3408. addRefExprDependency(expr->queryChild(1), curGraph, activityExpr);
  3409. OwnedHqlExpr value = createAttribute(fileAtom, getNormalizedFilename(expr->queryChild(2)));
  3410. addDependencySource(value, curGraph, activityExpr);
  3411. return true;
  3412. }
  3413. case no_keypatch:
  3414. {
  3415. addRefExprDependency(expr->queryChild(0), curGraph, activityExpr);
  3416. OwnedHqlExpr patchName = createAttribute(fileAtom, getNormalizedFilename(expr->queryChild(1)));
  3417. addDependencyUse(patchName, curGraph, activityExpr);
  3418. OwnedHqlExpr value = createAttribute(fileAtom, getNormalizedFilename(expr->queryChild(2)));
  3419. addDependencySource(value, curGraph, activityExpr);
  3420. return true;
  3421. }
  3422. case no_table:
  3423. {
  3424. IHqlExpression * filename = expr->queryChild(0);
  3425. OwnedHqlExpr value = createAttribute(fileAtom, getNormalizedFilename(filename));
  3426. addDependencyUse(value, curGraph, activityExpr);
  3427. return !filename->isConstant();
  3428. }
  3429. case no_select:
  3430. return isNewSelector(expr);
  3431. case no_workunit_dataset:
  3432. {
  3433. IHqlExpression * sequence = queryAttributeChild(expr, sequenceAtom, 0);
  3434. IHqlExpression * name = queryAttributeChild(expr, nameAtom, 0);
  3435. OwnedHqlExpr value = createAttribute(resultAtom, LINK(sequence), LINK(name));
  3436. addDependencyUse(value, curGraph, activityExpr);
  3437. return false;
  3438. }
  3439. case no_getresult:
  3440. {
  3441. IHqlExpression * sequence = queryAttributeChild(expr, sequenceAtom, 0);
  3442. IHqlExpression * name = queryAttributeChild(expr, namedAtom, 0);
  3443. OwnedHqlExpr value = createAttribute(resultAtom, LINK(sequence), LINK(name));
  3444. addDependencyUse(value, curGraph, activityExpr);
  3445. return false;
  3446. }
  3447. case no_getgraphresult:
  3448. {
  3449. OwnedHqlExpr value = createAttribute(resultAtom, LINK(expr->queryChild(1)), LINK(expr->queryChild(2)));
  3450. addDependencyUse(value, curGraph, activityExpr);
  3451. return false;
  3452. }
  3453. case no_setgraphresult:
  3454. {
  3455. OwnedHqlExpr value = createAttribute(resultAtom, LINK(expr->queryChild(1)), LINK(expr->queryChild(2)));
  3456. addDependencySource(value, curGraph, activityExpr);
  3457. return true;
  3458. }
  3459. case no_ensureresult:
  3460. case no_setresult:
  3461. case no_extractresult:
  3462. {
  3463. IHqlExpression * sequence = queryAttributeChild(expr, sequenceAtom, 0);
  3464. IHqlExpression * name = queryAttributeChild(expr, namedAtom, 0);
  3465. OwnedHqlExpr value = createAttribute(resultAtom, LINK(sequence), LINK(name));
  3466. addDependencySource(value, curGraph, activityExpr);
  3467. return true;
  3468. }
  3469. case no_attr:
  3470. case no_attr_link:
  3471. case no_record:
  3472. case no_field:
  3473. return false; //no need to look any further
  3474. default:
  3475. return true;
  3476. }
  3477. }
  3478. void EclResourcer::doAddChildDependencies(IHqlExpression * expr, ResourceGraphInfo * graph, IHqlExpression * activityExpr)
  3479. {
  3480. if (expr->queryTransformExtra())
  3481. return;
  3482. expr->setTransformExtraUnlinked(expr);
  3483. if (addExprDependency(expr, graph, activityExpr))
  3484. {
  3485. ForEachChild(idx, expr)
  3486. doAddChildDependencies(expr->queryChild(idx), graph, activityExpr);
  3487. }
  3488. }
  3489. void EclResourcer::addChildDependencies(IHqlExpression * expr, ResourceGraphInfo * graph, IHqlExpression * activityExpr)
  3490. {
  3491. if (graph)
  3492. {
  3493. lockTransformMutex();
  3494. doAddChildDependencies(expr, graph, activityExpr);
  3495. unlockTransformMutex();
  3496. }
  3497. }
  3498. void EclResourcer::addDependencies(IHqlExpression * expr, ResourceGraphInfo * graph, IHqlExpression * activityExpr)
  3499. {
  3500. ResourcerInfo * info = queryResourceInfo(expr);
  3501. if (info && info->containsActivity)
  3502. {
  3503. if (info->isActivity)
  3504. {
  3505. if (info->gatheredDependencies)
  3506. return;
  3507. info->gatheredDependencies = true;
  3508. graph = info->graph;
  3509. activityExpr = expr;
  3510. }
  3511. if (addExprDependency(expr, graph, activityExpr))
  3512. {
  3513. unsigned first = getFirstActivityArgument(expr);
  3514. unsigned last = first + getNumActivityArguments(expr);
  3515. ForEachChild(idx, expr)
  3516. {
  3517. if ((idx >= first) && (idx < last))
  3518. addDependencies(expr->queryChild(idx), graph, activityExpr);
  3519. else
  3520. addChildDependencies(expr->queryChild(idx), graph, activityExpr);
  3521. }
  3522. }
  3523. }
  3524. else
  3525. addChildDependencies(expr, graph, activityExpr);
  3526. ForEachItemIn(i, info->childDependents)
  3527. {
  3528. CChildDependent & cur = info->childDependents.item(i);
  3529. if (isResourcedActivity(cur.projectedHoisted))
  3530. {
  3531. addDependencies(cur.projectedHoisted, NULL, NULL);
  3532. ResourcerInfo * sourceInfo = queryResourceInfo(cur.projectedHoisted);
  3533. if (cur.isSingleNode)
  3534. sourceInfo->noteUsedFromChild(cur.forceHoist);
  3535. ResourceGraphLink * link = new ResourceGraphDependencyLink(sourceInfo->graph, cur.projectedHoisted, graph, expr);
  3536. graph->dependsOn.append(*link);
  3537. links.append(*link);
  3538. }
  3539. }
  3540. }
  3541. void EclResourcer::addDependencies(HqlExprArray & exprs)
  3542. {
  3543. ForEachItemIn(idx, exprs)
  3544. addDependencies(&exprs.item(idx), NULL, NULL);
  3545. }
  3546. void EclResourcer::spotUnbalancedSplitters(IHqlExpression * expr, unsigned whichSource, IHqlExpression * path, ResourceGraphInfo * graph)
  3547. {
  3548. ResourcerInfo * info = queryResourceInfo(expr);
  3549. if (!info)
  3550. return;
  3551. if (graph && info->graph && info->graph != graph)
  3552. {
  3553. if ((info->currentSource == whichSource) && (info->pathToSplitter != path))
  3554. graph->unbalancedExternalSources.append(*LINK(expr->queryBody()));
  3555. info->currentSource = whichSource;
  3556. info->pathToSplitter.set(path);
  3557. return;
  3558. }
  3559. else
  3560. {
  3561. if (info->currentSource == whichSource)
  3562. {
  3563. if (info->pathToSplitter != path)
  3564. info->balanced = false;
  3565. return;
  3566. }
  3567. info->currentSource = whichSource;
  3568. info->pathToSplitter.set(path);
  3569. }
  3570. if (info->containsActivity)
  3571. {
  3572. unsigned first = getFirstActivityArgument(expr);
  3573. unsigned num = getNumActivityArguments(expr);
  3574. bool modify = false;
  3575. if (num > 1)
  3576. {
  3577. switch (expr->getOperator())
  3578. {
  3579. case no_addfiles:
  3580. if (expr->hasAttribute(_ordered_Atom) || expr->hasAttribute(_orderedPull_Atom) || isGrouped(expr))
  3581. modify = true;
  3582. break;
  3583. default:
  3584. modify = true;
  3585. break;
  3586. }
  3587. }
  3588. unsigned last = first + num;
  3589. for (unsigned idx = first; idx < last; idx++)
  3590. {
  3591. OwnedHqlExpr childPath = modify ? createAttribute(pathAtom, getSizetConstant(idx), LINK(path)) : LINK(path);
  3592. spotUnbalancedSplitters(expr->queryChild(idx), whichSource, childPath, graph);
  3593. }
  3594. }
  3595. //Now check dependencies between graphs (for roxie)
  3596. if (!graph)
  3597. {
  3598. if (info->graph)
  3599. {
  3600. GraphLinkArray & graphLinks = info->graph->dependsOn;
  3601. ForEachItemIn(i, graphLinks)
  3602. {
  3603. ResourceGraphLink & link = graphLinks.item(i);
  3604. if (link.sinkNode == expr)
  3605. {
  3606. OwnedHqlExpr childPath = createAttribute(dependencyAtom, LINK(link.sourceNode), LINK(path));
  3607. spotUnbalancedSplitters(link.sourceNode, whichSource, childPath, graph);
  3608. }
  3609. }
  3610. }
  3611. else
  3612. {
  3613. ForEachItemIn(i, links)
  3614. {
  3615. ResourceGraphLink & link = links.item(i);
  3616. if (link.sinkNode == expr)
  3617. {
  3618. OwnedHqlExpr childPath = createAttribute(dependencyAtom, LINK(link.sourceNode), LINK(path));
  3619. spotUnbalancedSplitters(link.sourceNode, whichSource, childPath, graph);
  3620. }
  3621. }
  3622. }
  3623. }
  3624. }
  3625. void EclResourcer::spotUnbalancedSplitters(HqlExprArray & exprs)
  3626. {
  3627. unsigned curSource = 1;
  3628. switch (targetClusterType)
  3629. {
  3630. case HThorCluster:
  3631. break;
  3632. case ThorLCRCluster:
  3633. {
  3634. //Thor only handles one graph at a time, so only walk expressions within a single graph.
  3635. ForEachItemIn(i1, graphs)
  3636. {
  3637. ResourceGraphInfo & curGraph = graphs.item(i1);
  3638. ForEachItemIn(i2, curGraph.sinks)
  3639. {
  3640. ResourceGraphLink & cur = curGraph.sinks.item(i2);
  3641. spotUnbalancedSplitters(cur.sourceNode, curSource++, 0, &curGraph);
  3642. }
  3643. }
  3644. }
  3645. break;
  3646. case RoxieCluster:
  3647. {
  3648. //Roxie pulls all at once, so need to analyse globally.
  3649. ForEachItemIn(idx, exprs)
  3650. spotUnbalancedSplitters(&exprs.item(idx), curSource++, 0, NULL);
  3651. break;
  3652. }
  3653. }
  3654. }
  3655. void EclResourcer::spotSharedInputs(IHqlExpression * expr, ResourceGraphInfo * graph)
  3656. {
  3657. ResourcerInfo * info = queryResourceInfo(expr);
  3658. if (!info)
  3659. return;
  3660. if (info->graph && info->graph != graph)
  3661. {
  3662. IHqlExpression * body = expr->queryBody();
  3663. if (!graph->unbalancedExternalSources.contains(*body))
  3664. graph->balancedExternalSources.append(*LINK(body));
  3665. return;
  3666. }
  3667. if (info->isSplit())
  3668. {
  3669. //overload currentSource to track if we have visited this splitter before. It cannot have value value NotFound up to now
  3670. if (info->currentSource == NotFound)
  3671. return;
  3672. info->currentSource = NotFound;
  3673. }
  3674. if (info->containsActivity)
  3675. {
  3676. unsigned first = getFirstActivityArgument(expr);
  3677. unsigned num = getNumActivityArguments(expr);
  3678. unsigned last = first + num;
  3679. for (unsigned idx = first; idx < last; idx++)
  3680. {
  3681. spotSharedInputs(expr->queryChild(idx), graph);
  3682. }
  3683. }
  3684. }
  3685. void EclResourcer::spotSharedInputs()
  3686. {
  3687. //Thor only handles one graph at a time, so only walk expressions within a single graph.
  3688. ForEachItemIn(i1, graphs)
  3689. {
  3690. ResourceGraphInfo & curGraph = graphs.item(i1);
  3691. HqlExprCopyArray visited;
  3692. ForEachItemIn(i2, curGraph.sinks)
  3693. {
  3694. ResourceGraphLink & cur = curGraph.sinks.item(i2);
  3695. IHqlExpression * curExpr = cur.sourceNode->queryBody();
  3696. if (!visited.contains(*curExpr))
  3697. {
  3698. ResourcerInfo * info = queryResourceInfo(curExpr);
  3699. if (!info->isExternalSpill() && !info->expandRatherThanSpill(true))
  3700. {
  3701. spotSharedInputs(curExpr, &curGraph);
  3702. visited.append(*curExpr);
  3703. }
  3704. }
  3705. }
  3706. }
  3707. }
  3708. //------------------------------------------------------------------------------------------
  3709. // PASS6: Merge sub graphs that can share resources and don't have dependencies
  3710. // MORE: Once sources are merged, should try merging between trees.
  3711. static bool conditionsMatch(const HqlExprArray & left, const HqlExprArray & right)
  3712. {
  3713. if (left.ordinality() != right.ordinality())
  3714. return false;
  3715. ForEachItemIn(i, left)
  3716. {
  3717. if (!left.contains(right.item(i)) || !right.contains(left.item(i)))
  3718. return false;
  3719. }
  3720. return true;
  3721. }
  3722. bool EclResourcer::queryMergeGraphLink(ResourceGraphLink & link)
  3723. {
  3724. if (link.linkKind == UnconditionalLink)
  3725. {
  3726. //Don't combine any dependencies
  3727. const GraphLinkArray & sinks = link.sourceGraph->sinks;
  3728. ForEachItemIn(i1, sinks)
  3729. {
  3730. ResourceGraphLink & cur = sinks.item(i1);
  3731. if (cur.sinkGraph && cur.sourceNode->isAction())
  3732. return false;
  3733. }
  3734. //Roxie pulls all subgraphs at same time, so no problem with conditional links since handled at run time.
  3735. if (options.noConditionalLinks)
  3736. return true;
  3737. //No conditionals in the sink graph=>will be executed just as frequently
  3738. if (!link.sinkGraph->mergedConditionSource)
  3739. return true;
  3740. //Is this the only place this source graph is used? If so, always fine to merge
  3741. if (sinks.ordinality() == 1)
  3742. return true;
  3743. //1) if context the source graph is being merged into is unconditional, then it is ok [ could have conditional and unconditional paths to same graph]
  3744. //2) if context is conditional, then we don't really want to do it unless the conditions on all sinks are identical, and the only links occur between these two graphs.
  3745. // (situation occurs with spill fed into two branches of a join).
  3746. bool isConditionalInSinkGraph = false;
  3747. bool accessedFromManyGraphs = false;
  3748. ForEachItemIn(i, sinks)
  3749. {
  3750. ResourceGraphLink & cur = sinks.item(i);
  3751. if (cur.sinkNode)
  3752. {
  3753. if (cur.sinkGraph != link.sinkGraph)
  3754. accessedFromManyGraphs = true;
  3755. else
  3756. {
  3757. if (!isConditionalInSinkGraph)
  3758. {
  3759. ResourcerInfo * sinkInfo = queryResourceInfo(cur.sinkNode);
  3760. //If this is conditional, don't merge if there is a link to another graph
  3761. if ((!cur.isDependency() && sinkInfo->isConditionExpr()) ||
  3762. //if (sinkInfo->isConditionExpr() ||
  3763. (!sinkInfo->isUnconditional() && sinkInfo->conditions.ordinality()))
  3764. isConditionalInSinkGraph = true;
  3765. }
  3766. }
  3767. }
  3768. }
  3769. if (isConditionalInSinkGraph && accessedFromManyGraphs)
  3770. return false;
  3771. return true;
  3772. }
  3773. return false;
  3774. }
  3775. unsigned EclResourcer::getMaxDepth() const
  3776. {
  3777. unsigned maxDepth = 0;
  3778. for (unsigned idx = 0; idx < graphs.ordinality(); idx++)
  3779. {
  3780. unsigned depth = graphs.item(idx).getDepth();
  3781. if (depth > maxDepth)
  3782. maxDepth = depth;
  3783. }
  3784. return maxDepth;
  3785. }
  3786. void EclResourcer::mergeSubGraphs(unsigned pass)
  3787. {
  3788. unsigned maxDepth = getMaxDepth();
  3789. for (unsigned curDepth = maxDepth+1; curDepth-- != 0;)
  3790. {
  3791. mergeAgain:
  3792. for (unsigned idx = 0; idx < graphs.ordinality(); idx++)
  3793. {
  3794. ResourceGraphInfo & cur = graphs.item(idx);
  3795. if ((cur.getDepth() == curDepth) && !cur.isDead)
  3796. {
  3797. bool tryAgain;
  3798. do
  3799. {
  3800. tryAgain = false;
  3801. for (unsigned idxSource = 0; idxSource < cur.sources.ordinality(); /*incremented in loop*/)
  3802. {
  3803. ResourceGraphLink & curLink = cur.sources.item(idxSource);
  3804. ResourceGraphInfo * source = curLink.sourceGraph;
  3805. IHqlExpression * sourceNode = curLink.sourceNode;
  3806. bool tryToMerge;
  3807. bool expandSourceInPlace = queryResourceInfo(sourceNode)->expandRatherThanSpill(false);
  3808. if (pass == 0)
  3809. tryToMerge = !expandSourceInPlace;
  3810. else
  3811. tryToMerge = expandSourceInPlace;
  3812. if (tryToMerge)
  3813. {
  3814. bool ok = true;
  3815. ResourcerInfo * sourceResourceInfo = queryResourceInfo(sourceNode);
  3816. if (sourceResourceInfo->outputToUseForSpill && (targetClusterType == HThorCluster))
  3817. {
  3818. if (curLink.sinkNode != sourceResourceInfo->outputToUseForSpill)
  3819. ok = false;
  3820. }
  3821. if (sequential && source->containsActionSink())
  3822. ok = false;
  3823. unsigned curSourceDepth = source->getDepth();
  3824. //MORE: Merging identical conditionals?
  3825. if (ok && queryMergeGraphLink(curLink) &&
  3826. !sourceResourceInfo->expandRatherThanSplit() &&
  3827. cur.mergeInSource(*source, *resourceLimit))
  3828. {
  3829. //NB: Following cannot remove sources below the current index.
  3830. replaceGraphReferences(source, &cur);
  3831. source->isDead = true;
  3832. #ifdef VERIFY_RESOURCING
  3833. checkRecursion(&cur);
  3834. #endif
  3835. unsigned newDepth = cur.getDepth();
  3836. //Unusual: The source we are merging with has just increased in depth, so any
  3837. //dependents have also increased in depth. Need to try again at different depth
  3838. //to see if one of those will merge in.
  3839. if (newDepth > curSourceDepth)
  3840. {
  3841. curDepth += (newDepth - curSourceDepth);
  3842. goto mergeAgain;
  3843. }
  3844. //depth of this element has changed, so don't check to see if it merges with any other
  3845. //sources on this iteration.
  3846. if (newDepth != curDepth)
  3847. {
  3848. tryAgain = false;
  3849. break;
  3850. }
  3851. tryAgain = true;
  3852. }
  3853. else
  3854. idxSource++;
  3855. }
  3856. else
  3857. idxSource++;
  3858. }
  3859. } while (tryAgain);
  3860. }
  3861. }
  3862. }
  3863. }
  3864. void EclResourcer::mergeSiblings()
  3865. {
  3866. unsigned maxDepth = getMaxDepth();
  3867. for (unsigned curDepth = maxDepth+1; curDepth-- != 0;)
  3868. {
  3869. for (unsigned idx = 0; idx < graphs.ordinality(); idx++)
  3870. {
  3871. ResourceGraphInfo & cur = graphs.item(idx);
  3872. if ((cur.getDepth() == curDepth) && !cur.isDead)
  3873. {
  3874. if (cur.containsActionSink())
  3875. {
  3876. if (!cur.canMergeActionAsSibling(sequential))
  3877. continue;
  3878. }
  3879. ForEachItemIn(idxSource, cur.sources)
  3880. {
  3881. ResourceGraphLink & curLink = cur.sources.item(idxSource);
  3882. ResourceGraphInfo * source = curLink.sourceGraph;
  3883. IHqlExpression * sourceNode = curLink.sourceNode;
  3884. ResourcerInfo * sourceInfo = queryResourceInfo(sourceNode);
  3885. if (sourceInfo->neverSplit || sourceInfo->expandRatherThanSplit())
  3886. continue;
  3887. for (unsigned iSink = 0; iSink < source->sinks.ordinality(); )
  3888. {
  3889. ResourceGraphLink & secondLink = source->sinks.item(iSink);
  3890. ResourceGraphInfo * sink = secondLink.sinkGraph;
  3891. bool ok = false;
  3892. if (sink && (sink != &cur) && !sink->isDead && sourceNode->queryBody() == secondLink.sourceNode->queryBody())
  3893. {
  3894. ok = true;
  3895. if (sequential && !sink->canMergeActionAsSibling(sequential))
  3896. ok = false;
  3897. if (ok && cur.mergeInSibling(*sink, *resourceLimit))
  3898. {
  3899. //NB: Following cannot remove sources below the current index.
  3900. replaceGraphReferences(sink, &cur);
  3901. sink->isDead = true;
  3902. }
  3903. else
  3904. iSink++;
  3905. }
  3906. else
  3907. iSink++;
  3908. }
  3909. }
  3910. }
  3911. }
  3912. }
  3913. }
  3914. void EclResourcer::mergeSubGraphs()
  3915. {
  3916. for (unsigned pass=0; pass < 2; pass++)
  3917. mergeSubGraphs(pass);
  3918. if (options.combineSiblings)
  3919. mergeSiblings();
  3920. ForEachItemInRev(idx2, graphs)
  3921. {
  3922. if (graphs.item(idx2).isDead)
  3923. graphs.remove(idx2);
  3924. }
  3925. }
  3926. //------------------------------------------------------------------------------------------
  3927. // PASS7: Optimize aggregates off of splitters into through aggregates.
  3928. bool EclResourcer::optimizeAggregate(IHqlExpression * expr)
  3929. {
  3930. if (!isSimpleAggregateResult(expr))
  3931. return false;
  3932. //expr is a no_extractresult
  3933. if (queryResourceInfo(expr)->childDependents.ordinality())
  3934. return false;
  3935. IHqlExpression * row2ds = expr->queryChild(0); // no_datasetfromrow
  3936. IHqlExpression * selectNth = row2ds->queryChild(0);
  3937. ResourcerInfo * row2dsInfo = queryResourceInfo(row2ds);
  3938. //If more than one set result for the same aggregate don't merge the aggregation because
  3939. //it messes up the internal count. Should really fix it and do multiple stores in the
  3940. //through aggregate.
  3941. if (row2dsInfo->numInternalUses() > 1)
  3942. return false;
  3943. //Be careful not to lose any spills...
  3944. if (row2dsInfo->numExternalUses)
  3945. return false;
  3946. ResourcerInfo * selectNthInfo = queryResourceInfo(selectNth);
  3947. if (selectNthInfo->numExternalUses)
  3948. return false;
  3949. IHqlExpression * aggregate = selectNth->queryChild(0); // no_newaggregate
  3950. IHqlExpression * parent = aggregate->queryChild(0);
  3951. ResourcerInfo * info = queryResourceInfo(parent);
  3952. if (info->numInternalUses() <= 1)
  3953. return false;
  3954. //ok, we can go ahead and merge.
  3955. info->aggregates.append(*LINK(expr));
  3956. // info->numExternalUses--;
  3957. // info->numUses--;
  3958. return true;
  3959. }
  3960. void EclResourcer::optimizeAggregates()
  3961. {
  3962. for (unsigned idx = 0; idx < graphs.ordinality(); idx++)
  3963. {
  3964. ResourceGraphInfo & cur = graphs.item(idx);
  3965. for (unsigned idxSink = 0; idxSink < cur.sinks.ordinality(); /*incremented in loop*/)
  3966. {
  3967. ResourceGraphLink & link = cur.sinks.item(idxSink);
  3968. if ((link.sinkGraph == NULL) && optimizeAggregate(link.sourceNode))
  3969. cur.sinks.remove(idxSink);
  3970. else
  3971. idxSink++;
  3972. }
  3973. }
  3974. }
  3975. //------------------------------------------------------------------------------------------
  3976. // PASS8: Improve efficiency by merging the split points slightly
  3977. IHqlExpression * EclResourcer::findPredecessor(IHqlExpression * expr, IHqlExpression * search, IHqlExpression * prev)
  3978. {
  3979. if (expr == search)
  3980. return prev;
  3981. ResourcerInfo * info = queryResourceInfo(expr);
  3982. if (info && info->containsActivity)
  3983. {
  3984. unsigned first = getFirstActivityArgument(expr);
  3985. unsigned last = first + getNumActivityArguments(expr);
  3986. for (unsigned idx=first; idx < last; idx++)
  3987. {
  3988. IHqlExpression * match = findPredecessor(expr->queryChild(idx), search, expr);
  3989. if (match)
  3990. return match;
  3991. }
  3992. }
  3993. return NULL;
  3994. }
  3995. IHqlExpression * EclResourcer::findPredecessor(ResourcerInfo * search)
  3996. {
  3997. ForEachItemIn(idx, links)
  3998. {
  3999. ResourceGraphLink & cur = links.item(idx);
  4000. if (cur.sourceGraph == search->graph)
  4001. {
  4002. IHqlExpression * match = findPredecessor(cur.sourceNode, search->original, NULL);
  4003. if (match)
  4004. return match;
  4005. }
  4006. }
  4007. return NULL;
  4008. }
  4009. void EclResourcer::moveExternalSpillPoints()
  4010. {
  4011. if (options.minimizeSpillSize == 0)
  4012. return;
  4013. //if we have a external spill point where all the external outputs reduce their data significantly
  4014. //either via a project or a filter, then it might be worth including those activities in the main
  4015. //graph and ,if all external children reduce data, then may be best to filter
  4016. ForEachItemIn(idx, links)
  4017. {
  4018. ResourceGraphLink & cur = links.item(idx);
  4019. if ((cur.linkKind == UnconditionalLink) && cur.sinkGraph)
  4020. {
  4021. while (lightweightAndReducesDatasetSize(cur.sinkNode))
  4022. {
  4023. ResourcerInfo * sourceInfo = queryResourceInfo(cur.sourceNode);
  4024. if (!sourceInfo->isExternalSpill() || (sourceInfo->numExternalUses > options.minimizeSpillSize))
  4025. break;
  4026. ResourcerInfo * sinkInfo = queryResourceInfo(cur.sinkNode);
  4027. if (sinkInfo->numInternalUses() != 1)
  4028. break;
  4029. IHqlExpression * sinkPred = findPredecessor(sinkInfo);
  4030. sinkInfo->graph.set(cur.sourceGraph);
  4031. sourceInfo->numExternalUses--;
  4032. sinkInfo->numExternalUses++;
  4033. cur.sourceNode.set(cur.sinkNode);
  4034. cur.sinkNode.set(sinkPred);
  4035. }
  4036. }
  4037. }
  4038. }
  4039. //------------------------------------------------------------------------------------------
  4040. // PASS9: Create a new expression tree representing the information
  4041. static HqlTransformerInfo childDependentReplacerInfo("ChildDependentReplacer");
  4042. class ChildDependentReplacer : public MergingHqlTransformer
  4043. {
  4044. public:
  4045. ChildDependentReplacer(const HqlExprCopyArray & _childDependents, const HqlExprArray & _replacements)
  4046. : MergingHqlTransformer(childDependentReplacerInfo), childDependents(_childDependents), replacements(_replacements)
  4047. {
  4048. }
  4049. protected:
  4050. virtual IHqlExpression * createTransformed(IHqlExpression * expr)
  4051. {
  4052. unsigned match = childDependents.find(*expr);
  4053. if (match != NotFound)
  4054. return LINK(&replacements.item(match));
  4055. return MergingHqlTransformer::createTransformed(expr);
  4056. }
  4057. protected:
  4058. const HqlExprCopyArray & childDependents;
  4059. const HqlExprArray & replacements;
  4060. };
  4061. static IHqlExpression * getScalarReplacement(CChildDependent & cur, ResourcerInfo * hoistedInfo, IHqlExpression * replacement)
  4062. {
  4063. IHqlExpression * value = cur.original;
  4064. //First skip any wrappers which are there to cause things to be hoisted.
  4065. loop
  4066. {
  4067. node_operator op = value->getOperator();
  4068. if ((op != no_globalscope) && (op != no_thisnode) && (op != no_evalonce))
  4069. break;
  4070. value = value->queryChild(0);
  4071. }
  4072. //Now modify the spilled result depending on how the spilled result was created (see EclHoistLocator::noteScalar() above)
  4073. if (value->getOperator() == no_select)
  4074. {
  4075. bool isNew;
  4076. IHqlExpression * ds = querySelectorDataset(value, isNew);
  4077. if(isNew || ds->isDatarow())
  4078. {
  4079. if (cur.hoisted != cur.projectedHoisted)
  4080. {
  4081. assertex(cur.projected);
  4082. unsigned match = hoistedInfo->projected.find(*cur.projected);
  4083. assertex(match != NotFound);
  4084. IHqlExpression * projectedRecord = cur.projectedHoisted->queryRecord();
  4085. IHqlExpression * projectedField = projectedRecord->queryChild(match);
  4086. return createNewSelectExpr(LINK(replacement), LINK(projectedField));
  4087. }
  4088. return replaceSelectorDataset(value, replacement);
  4089. }
  4090. //Very unusual - can occur when a thisnode(somefield) is extracted from an allnodes.
  4091. //It will have gone through the default case in the noteScalar() code
  4092. }
  4093. else if (value->getOperator() == no_createset)
  4094. {
  4095. IHqlExpression * record = replacement->queryRecord();
  4096. IHqlExpression * field = record->queryChild(0);
  4097. return createValue(no_createset, cur.original->getType(), LINK(replacement), createSelectExpr(LINK(replacement->queryNormalizedSelector()), LINK(field)));
  4098. }
  4099. IHqlExpression * record = replacement->queryRecord();
  4100. return createNewSelectExpr(LINK(replacement), LINK(record->queryChild(0)));
  4101. }
  4102. IHqlExpression * EclResourcer::replaceResourcedReferences(ResourcerInfo * info, IHqlExpression * expr)
  4103. {
  4104. if (!isAffectedByResourcing(expr))
  4105. return LINK(expr);
  4106. LinkedHqlExpr mapped = expr;
  4107. if (info && (info->childDependents.ordinality()))
  4108. {
  4109. HqlExprCopyArray originals;
  4110. HqlExprArray replacements;
  4111. ForEachItemIn(i, info->childDependents)
  4112. {
  4113. CChildDependent & cur = info->childDependents.item(i);
  4114. if (!isResourcedActivity(cur.projectedHoisted))
  4115. continue;
  4116. OwnedHqlExpr replacement = createResourced(cur.projectedHoisted, NULL, false, false);
  4117. IHqlExpression * original = cur.original;
  4118. if (!original->isDataset() && !original->isDatarow() && !original->isDictionary())
  4119. replacement.setown(getScalarReplacement(cur, queryResourceInfo(cur.hoisted), replacement));
  4120. originals.append(*original);
  4121. replacements.append(*replacement.getClear());
  4122. }
  4123. if (originals.ordinality())
  4124. {
  4125. ChildDependentReplacer replacer(originals, replacements);
  4126. mapped.setown(replacer.transformRoot(mapped));
  4127. }
  4128. }
  4129. return mapped.getClear();
  4130. }
  4131. IHqlExpression * EclResourcer::doCreateResourced(IHqlExpression * expr, ResourceGraphInfo * ownerGraph, bool expandInParent, bool defineSideEffect)
  4132. {
  4133. ResourcerInfo * info = queryResourceInfo(expr);
  4134. node_operator op = expr->getOperator();
  4135. HqlExprArray args;
  4136. bool same = true;
  4137. unsigned first = getFirstActivityArgument(expr);
  4138. unsigned last = first + getNumActivityArguments(expr);
  4139. OwnedHqlExpr transformed;
  4140. switch (op)
  4141. {
  4142. case no_if:
  4143. case no_choose:
  4144. case no_chooseds:
  4145. {
  4146. ForEachChild(idx, expr)
  4147. {
  4148. IHqlExpression * child = expr->queryChild(idx);
  4149. IHqlExpression * resourced;
  4150. if ((idx < first) || (idx >= last))
  4151. resourced = replaceResourcedReferences(info, child);
  4152. else
  4153. resourced = createResourced(child, ownerGraph, expandInParent, false);
  4154. if (child != resourced)
  4155. same = false;
  4156. args.append(*resourced);
  4157. }
  4158. break;
  4159. }
  4160. case no_case:
  4161. case no_map:
  4162. UNIMPLEMENTED;
  4163. case no_keyed:
  4164. return LINK(expr);
  4165. case no_compound:
  4166. if (options.convertCompoundToExecuteWhen)
  4167. {
  4168. //NB: Arguments to no_executewhen are the reverse of no_compound.
  4169. args.append(*createResourced(expr->queryChild(1), ownerGraph, expandInParent, false));
  4170. args.append(*createResourced(expr->queryChild(0), ownerGraph, expandInParent, false));
  4171. transformed.setown(createDataset(no_executewhen, args));
  4172. }
  4173. else
  4174. transformed.setown(createResourced(expr->queryChild(1), ownerGraph, expandInParent, false));
  4175. break;
  4176. case no_executewhen:
  4177. {
  4178. args.append(*createResourced(expr->queryChild(0), ownerGraph, expandInParent, false));
  4179. args.append(*createResourced(expr->queryChild(1), ownerGraph, expandInParent, false));
  4180. assertex(args.item(1).getOperator() == no_callsideeffect);
  4181. unwindChildren(args, expr, 2);
  4182. same = false;
  4183. break;
  4184. }
  4185. case no_select:
  4186. {
  4187. IHqlExpression * ds = expr->queryChild(0);
  4188. OwnedHqlExpr newDs = createResourced(ds, ownerGraph, expandInParent, false);
  4189. if (ds != newDs)
  4190. {
  4191. args.append(*LINK(newDs));
  4192. unwindChildren(args, expr, 1);
  4193. if (!expr->hasAttribute(newAtom) && isNewSelector(expr) && (newDs->getOperator() != no_select))
  4194. args.append(*LINK(queryNewSelectAttrExpr()));
  4195. same = false;
  4196. }
  4197. break;
  4198. }
  4199. case no_join:
  4200. case no_denormalize:
  4201. case no_denormalizegroup:
  4202. if (false)//if (isKeyedJoin(expr))
  4203. {
  4204. args.append(*createResourced(expr->queryChild(0), ownerGraph, expandInParent, false));
  4205. args.append(*LINK(expr->queryChild(1)));
  4206. unsigned max = expr->numChildren();
  4207. for (unsigned idx=2; idx < max; idx++)
  4208. args.append(*replaceResourcedReferences(info, expr->queryChild(idx)));
  4209. same = false;
  4210. break;
  4211. }
  4212. //fall through...
  4213. default:
  4214. {
  4215. IHqlExpression * activeTable = NULL;
  4216. // Check to see if the activity has a dataset which is in scope for the rest of its arguments.
  4217. // If so we'll need to remap references from the children.
  4218. if (hasActiveTopDataset(expr) && (first != last))
  4219. activeTable = expr->queryChild(0);
  4220. ForEachChild(idx, expr)
  4221. {
  4222. IHqlExpression * child = expr->queryChild(idx);
  4223. IHqlExpression * resourced;
  4224. if ((idx < first) || (idx >= last))
  4225. {
  4226. LinkedHqlExpr mapped = child;
  4227. if (activeTable && isAffectedByResourcing(child))
  4228. {
  4229. IHqlExpression * activeTableTransformed = &args.item(0);
  4230. if (activeTable != activeTableTransformed)
  4231. mapped.setown(scopedReplaceSelector(child, activeTable, activeTableTransformed));
  4232. }
  4233. resourced = replaceResourcedReferences(info, mapped);
  4234. }
  4235. else
  4236. resourced = createResourced(child, ownerGraph, expandInParent, false);
  4237. if (child != resourced)
  4238. same = false;
  4239. args.append(*resourced);
  4240. }
  4241. }
  4242. break;
  4243. }
  4244. if (!transformed)
  4245. transformed.setown(same ? LINK(expr) : expr->clone(args));
  4246. if (!expandInParent)
  4247. {
  4248. if (!transformed->isAction())
  4249. transformed.setown(info->createTransformedExpr(transformed));
  4250. else if (defineSideEffect)
  4251. transformed.setown(createValue(no_definesideeffect, LINK(transformed), createUniqueId()));
  4252. }
  4253. return transformed.getClear();
  4254. }
  4255. /*
  4256. Need to be careful because result should not reuse the same expression tree unless that element is a splitter.
  4257. createResourced()
  4258. {
  4259. if (!isActivity)
  4260. if (!containsActivity)
  4261. replace any refs to the activeTable with whatever it has been mapped to
  4262. else
  4263. recurse
  4264. else if (!isDefinedInSameGraph)
  4265. expand/create reader
  4266. set active table
  4267. else isSplitter and alreadyGeneratedForThisGraph
  4268. return previous result
  4269. else
  4270. create transformed
  4271. }
  4272. */
  4273. void EclResourcer::doCheckRecursion(ResourceGraphInfo * graph, PointerArray & visited)
  4274. {
  4275. visited.append(graph);
  4276. ForEachItemIn(idxD, graph->dependsOn)
  4277. checkRecursion(graph->dependsOn.item(idxD).sourceGraph, visited);
  4278. ForEachItemIn(idxS, graph->sources)
  4279. checkRecursion(graph->sources.item(idxS).sourceGraph, visited);
  4280. visited.pop();
  4281. }
  4282. void EclResourcer::checkRecursion(ResourceGraphInfo * graph, PointerArray & visited)
  4283. {
  4284. if (visited.find(graph) != NotFound)
  4285. throwUnexpected();
  4286. doCheckRecursion(graph, visited);
  4287. }
  4288. void EclResourcer::checkRecursion(ResourceGraphInfo * graph)
  4289. {
  4290. PointerArray visited;
  4291. doCheckRecursion(graph, visited);
  4292. }
  4293. IHqlExpression * EclResourcer::createResourced(IHqlExpression * expr, ResourceGraphInfo * ownerGraph, bool expandInParent, bool defineSideEffect)
  4294. {
  4295. ResourcerInfo * info = queryResourceInfo(expr);
  4296. if (!info || !info->containsActivity)
  4297. return replaceResourcedReferences(info, expr);
  4298. if (!info->isActivity)
  4299. {
  4300. assertex(!defineSideEffect);
  4301. HqlExprArray args;
  4302. bool same = true;
  4303. ForEachChild(idx, expr)
  4304. {
  4305. IHqlExpression * cur = expr->queryChild(idx);
  4306. IHqlExpression * curResourced = createResourced(cur, ownerGraph, expandInParent, false);
  4307. args.append(*curResourced);
  4308. if (cur != curResourced)
  4309. same = false;
  4310. }
  4311. if (same)
  4312. return LINK(expr);
  4313. return expr->clone(args);
  4314. }
  4315. if (info->graph != ownerGraph)
  4316. {
  4317. assertex(!defineSideEffect);
  4318. bool isShared = options.optimizeSharedInputs && ownerGraph && ownerGraph->isSharedInput(expr);
  4319. if (isShared)
  4320. {
  4321. IHqlExpression * mapped = ownerGraph->queryMappedSharedInput(expr->queryBody());
  4322. if (mapped)
  4323. return LINK(mapped);
  4324. }
  4325. IHqlExpression * source;
  4326. if (info->expandRatherThanSpill(true))
  4327. {
  4328. bool expandChildInParent = options.minimiseSpills ? expandInParent : true;
  4329. OwnedHqlExpr resourced = doCreateResourced(expr, ownerGraph, expandChildInParent, false);
  4330. if (queryAddUniqueToActivity(resourced))
  4331. source = appendUniqueAttr(resourced);
  4332. else
  4333. source = LINK(resourced);
  4334. }
  4335. else
  4336. {
  4337. if (!expr->isAction())
  4338. {
  4339. OwnedHqlExpr reason;
  4340. if (ownerGraph && options.checkResources())
  4341. {
  4342. StringBuffer reasonText;
  4343. ownerGraph->getMergeFailReason(reasonText, info->graph, *resourceLimit);
  4344. if (reasonText.length())
  4345. {
  4346. reasonText.insert(0, "Resource limit spill: ");
  4347. reason.setown(createAttribute(_spillReason_Atom, createConstant(reasonText.str())));
  4348. }
  4349. }
  4350. source = info->createSpilledRead(reason);
  4351. }
  4352. else
  4353. {
  4354. IHqlExpression * transformed = info->transformed;
  4355. if (transformed->getOperator() == no_definesideeffect)
  4356. {
  4357. IHqlExpression * uid = info->transformed->queryAttribute(_uid_Atom);
  4358. assertex(uid);
  4359. source = createValue(no_callsideeffect, makeVoidType(), LINK(uid));
  4360. }
  4361. else
  4362. source = LINK(transformed);
  4363. }
  4364. }
  4365. if (isShared)
  4366. {
  4367. if (!source->isAction())
  4368. {
  4369. source = createDatasetF(no_split, source, createAttribute(balancedAtom), createUniqueId(), NULL);
  4370. ownerGraph->addSharedInput(expr->queryBody(), source);
  4371. }
  4372. }
  4373. return source;
  4374. }
  4375. if (!expandInParent && info->transformed && info->isSplit())
  4376. {
  4377. return LINK(info->transformed);
  4378. }
  4379. OwnedHqlExpr resourced = doCreateResourced(expr, ownerGraph, expandInParent, defineSideEffect);
  4380. if (queryAddUniqueToActivity(resourced))// && !resourced->hasAttribute(_internal_Atom))
  4381. resourced.setown(appendUniqueAttr(resourced));
  4382. if (!expandInParent)
  4383. {
  4384. info->transformed = resourced;
  4385. }
  4386. return resourced.getClear();
  4387. }
  4388. void EclResourcer::createResourced(ResourceGraphInfo * graph, HqlExprArray & transformed)
  4389. {
  4390. if (graph->createdGraph || graph->isDead)
  4391. return;
  4392. if (!graph->containsActiveSinks() && (!graph->hasConditionSource))
  4393. return;
  4394. #ifdef VERIFY_RESOURCING
  4395. checkRecursion(graph);
  4396. #endif
  4397. // DBGLOG("Prepare to CreateResourced(%p)", graph);
  4398. if (graph->startedGeneratingResourced)
  4399. throwError(HQLWRN_RecursiveDependendencies);
  4400. graph->startedGeneratingResourced = true;
  4401. ForEachItemIn(idxD, graph->dependsOn)
  4402. createResourced(graph->dependsOn.item(idxD).sourceGraph, transformed);
  4403. ForEachItemIn(idxS, graph->sources)
  4404. createResourced(graph->sources.item(idxS).sourceGraph, transformed);
  4405. // DBGLOG("Create resourced %p", graph);
  4406. //First generate the graphs for all the unconditional sinks
  4407. HqlExprArray args;
  4408. ForEachItemIn(idx, graph->sinks)
  4409. {
  4410. ResourceGraphLink & sink = graph->sinks.item(idx);
  4411. IHqlExpression * sinkNode = sink.sourceNode;
  4412. ResourcerInfo * info = queryResourceInfo(sinkNode);
  4413. //If graph is unconditional, any condition sinks are forced to be generated (and spilt)
  4414. if (!info->transformed)
  4415. {
  4416. // if it is a spiller, then it will be generated from another sink
  4417. if (!info->isExternalSpill())
  4418. {
  4419. IHqlExpression * resourced = createResourced(sinkNode, graph, false, sinkNode->isAction() && sink.sinkGraph);
  4420. assertex(info->transformed);
  4421. args.append(*resourced);
  4422. }
  4423. }
  4424. }
  4425. ForEachItemIn(i2, graph->sinks)
  4426. {
  4427. ResourceGraphLink & sink = graph->sinks.item(i2);
  4428. IHqlExpression * sinkNode = sink.sourceNode;
  4429. ResourcerInfo * info = queryResourceInfo(sinkNode);
  4430. IHqlExpression * splitter = info->splitterOutput;
  4431. if (splitter && !args.contains(*splitter))
  4432. args.append(*LINK(splitter));
  4433. }
  4434. if (args.ordinality() == 0)
  4435. graph->isDead = true;
  4436. else
  4437. {
  4438. if (options.useGraphResults || options.alwaysUseGraphResults)
  4439. args.append(*createAttribute(childAtom));
  4440. graph->createdGraph.setown(createValue(no_subgraph, makeVoidType(), args));
  4441. transformed.append(*LINK(graph->createdGraph));
  4442. }
  4443. }
  4444. void EclResourcer::inheritRedundantDependencies(ResourceGraphInfo * thisGraph)
  4445. {
  4446. if (thisGraph->inheritedExpandedDependencies)
  4447. return;
  4448. thisGraph->inheritedExpandedDependencies = true;
  4449. ForEachItemIn(idx, thisGraph->sources)
  4450. {
  4451. ResourceGraphLink & cur = thisGraph->sources.item(idx);
  4452. if (cur.isRedundantLink())
  4453. {
  4454. inheritRedundantDependencies(cur.sourceGraph);
  4455. ForEachItemIn(i, cur.sourceGraph->dependsOn)
  4456. {
  4457. ResourceGraphLink & curDepend = cur.sourceGraph->dependsOn.item(i);
  4458. ResourceGraphLink * link = new ResourceGraphDependencyLink(curDepend.sourceGraph, curDepend.sourceNode, thisGraph, cur.sinkNode);
  4459. thisGraph->dependsOn.append(*link);
  4460. links.append(*link);
  4461. }
  4462. }
  4463. }
  4464. }
  4465. void EclResourcer::createResourced(HqlExprArray & transformed)
  4466. {
  4467. //Before removing null links (e.g., where the source graph is expanded inline), need to make sure
  4468. //dependencies are cloned, otherwise graphs can be generated in the wrong order
  4469. ForEachItemIn(idx1, graphs)
  4470. inheritRedundantDependencies(&graphs.item(idx1));
  4471. ForEachItemInRev(idx2, links)
  4472. {
  4473. ResourceGraphLink & cur = links.item(idx2);
  4474. if (cur.isRedundantLink())
  4475. removeLink(cur, true);
  4476. }
  4477. ForEachItemIn(idx3, graphs)
  4478. createResourced(&graphs.item(idx3), transformed);
  4479. }
  4480. static int compareGraphDepth(CInterface * * _l, CInterface * * _r)
  4481. {
  4482. ResourceGraphInfo * l = (ResourceGraphInfo *)*_l;
  4483. ResourceGraphInfo * r = (ResourceGraphInfo *)*_r;
  4484. return l->getDepth() - r->getDepth();
  4485. }
  4486. static int compareLinkDepth(CInterface * * _l, CInterface * * _r)
  4487. {
  4488. ResourceGraphLink * l = (ResourceGraphLink *)*_l;
  4489. ResourceGraphLink * r = (ResourceGraphLink *)*_r;
  4490. int diff = l->sourceGraph->getDepth() - r->sourceGraph->getDepth();
  4491. if (diff) return diff;
  4492. if (l->sinkGraph)
  4493. if (r->sinkGraph)
  4494. return l->sinkGraph->getDepth() - r->sinkGraph->getDepth();
  4495. else
  4496. return -1;
  4497. else
  4498. if (r->sinkGraph)
  4499. return +1;
  4500. else
  4501. return 0;
  4502. }
  4503. void EclResourcer::display(StringBuffer & out)
  4504. {
  4505. CIArrayOf<ResourceGraphInfo> sortedGraphs;
  4506. ForEachItemIn(j1, graphs)
  4507. sortedGraphs.append(OLINK(graphs.item(j1)));
  4508. sortedGraphs.sort(compareGraphDepth);
  4509. out.append("Graphs:\n");
  4510. ForEachItemIn(i, sortedGraphs)
  4511. {
  4512. ResourceGraphInfo & cur = sortedGraphs.item(i);
  4513. out.appendf("%d: depth(%d) uncond(%d) cond(%d) %s {%p}\n", i, cur.getDepth(), cur.isUnconditional, cur.hasConditionSource, cur.isDead ? "dead" : "", &cur);
  4514. ForEachItemIn(j, cur.sources)
  4515. {
  4516. ResourceGraphLink & link = cur.sources.item(j);
  4517. out.appendf(" Source: %p %s\n", link.sinkNode.get(), getOpString(link.sinkNode->getOperator()));
  4518. }
  4519. ForEachItemIn(k, cur.sinks)
  4520. {
  4521. ResourceGraphLink & link = cur.sinks.item(k);
  4522. IHqlExpression * sourceNode = link.sourceNode;
  4523. ResourcerInfo * sourceInfo = queryResourceInfo(sourceNode);
  4524. out.appendf(" Sink: %p %s cond(%d,%d) int(%d) ext(%d)\n", sourceNode, getOpString(sourceNode->getOperator()), sourceInfo->conditions.ordinality(), sourceInfo->conditionSourceCount, sourceInfo->numInternalUses(), sourceInfo->numExternalUses);
  4525. }
  4526. ForEachItemIn(i3, dependencySource.graphs)
  4527. {
  4528. if (&dependencySource.graphs.item(i3) == &cur)
  4529. {
  4530. StringBuffer s;
  4531. toECL(&dependencySource.search.item(i3), s);
  4532. out.appendf(" Creates: %s\n", s.str());
  4533. }
  4534. }
  4535. }
  4536. out.append("Links:\n");
  4537. CIArrayOf<ResourceGraphLink> sortedLinks;
  4538. ForEachItemIn(j2, links)
  4539. sortedLinks.append(OLINK(links.item(j2)));
  4540. sortedLinks.sort(compareLinkDepth);
  4541. ForEachItemIn(i2, sortedLinks)
  4542. {
  4543. ResourceGraphLink & link = sortedLinks.item(i2);
  4544. unsigned len = out.length();
  4545. out.appendf(" Source: %d %s", sortedGraphs.find(*link.sourceGraph), getOpString(link.sourceNode->getOperator()));
  4546. if (link.sinkNode)
  4547. {
  4548. out.padTo(len+30);
  4549. out.appendf(" Sink: %d %s", sortedGraphs.find(*link.sinkGraph), getOpString(link.sinkNode->getOperator()));
  4550. }
  4551. if (link.linkKind == SequenceLink)
  4552. out.append(" <sequence>");
  4553. out.newline();
  4554. }
  4555. }
  4556. void EclResourcer::trace()
  4557. {
  4558. StringBuffer s;
  4559. display(s);
  4560. DBGLOG("%s", s.str());
  4561. }
  4562. //---------------------------------------------------------------------------
  4563. static void expandLists(node_operator op, HqlExprArray & args, IHqlExpression * expr);
  4564. static void expandChildren(node_operator op, HqlExprArray & args, IHqlExpression * expr)
  4565. {
  4566. ForEachChild(idx, expr)
  4567. expandLists(op, args, expr->queryChild(idx));
  4568. }
  4569. static void expandLists(node_operator op, HqlExprArray & args, IHqlExpression * expr)
  4570. {
  4571. switch (expr->getOperator())
  4572. {
  4573. case no_sequential:
  4574. case no_orderedactionlist:
  4575. if (op != no_parallel)
  4576. expandChildren(no_sequential, args, expr);
  4577. else
  4578. args.append(*LINK(expr));
  4579. break;
  4580. case no_comma:
  4581. case no_compound:
  4582. case no_actionlist:
  4583. case no_parallel:
  4584. if (op != no_sequential)
  4585. expandChildren(no_parallel, args, expr);
  4586. else
  4587. args.append(*LINK(expr));
  4588. break;
  4589. case no_null:
  4590. break;
  4591. default:
  4592. args.append(*LINK(expr));
  4593. break;
  4594. }
  4595. }
  4596. void EclResourcer::resourceGraph(IHqlExpression * expr, HqlExprArray & transformed)
  4597. {
  4598. HqlExprArray exprs;
  4599. node_operator expandOp = options.isChildQuery ? no_any : no_parallel;
  4600. expandLists(expandOp, exprs, expr);
  4601. //NB: This only resources a single level of queries. SubQueries should be resourced in a separate
  4602. //pass so that commonality between different activities/subgraphs isn't introduced/messed up.
  4603. findSplitPoints(exprs);
  4604. createInitialGraphs(exprs);
  4605. markConditions(exprs);
  4606. if (options.checkResources())
  4607. resourceSubGraphs(exprs);
  4608. addDependencies(exprs);
  4609. #ifdef TRACE_RESOURCING
  4610. trace();
  4611. #endif
  4612. mergeSubGraphs();
  4613. #ifdef TRACE_RESOURCING
  4614. trace();
  4615. #endif
  4616. spotUnbalancedSplitters(exprs);
  4617. if (options.optimizeSharedInputs)
  4618. spotSharedInputs();
  4619. if (spotThroughAggregate)
  4620. optimizeAggregates();
  4621. moveExternalSpillPoints();
  4622. createResourced(transformed);
  4623. }
  4624. void EclResourcer::resourceRemoteGraph(IHqlExpression * expr, HqlExprArray & transformed)
  4625. {
  4626. HqlExprArray exprs;
  4627. expandLists(no_any, exprs, expr);
  4628. //NB: This only resources a single level of queries. SubQueries should be resourced in a separate
  4629. //pass so that commonality between different activities/subgraphs isn't introduced/messed up.
  4630. findSplitPoints(exprs);
  4631. createInitialRemoteGraphs(exprs);
  4632. markConditions(exprs);
  4633. addDependencies(exprs);
  4634. #ifdef TRACE_RESOURCING
  4635. trace();
  4636. #endif
  4637. mergeSubGraphs();
  4638. #ifdef TRACE_RESOURCING
  4639. trace();
  4640. #endif
  4641. createResourced(transformed);
  4642. }
  4643. //---------------------------------------------------------------------------
  4644. IHqlExpression * resourceThorGraph(HqlCppTranslator & translator, IHqlExpression * expr, ClusterType targetClusterType, unsigned clusterSize, IHqlExpression * graphIdExpr)
  4645. {
  4646. HqlExprArray transformed;
  4647. {
  4648. EclResourcer resourcer(translator.queryErrors(), translator.wu(), targetClusterType, clusterSize, translator.queryOptions());
  4649. if (graphIdExpr)
  4650. resourcer.setNewChildQuery(graphIdExpr, 0);
  4651. resourcer.resourceGraph(expr, transformed);
  4652. }
  4653. hoistNestedCompound(translator, transformed);
  4654. return createActionList(transformed);
  4655. }
  4656. static IHqlExpression * doResourceGraph(HqlCppTranslator & translator, HqlExprCopyArray * activeRows, IHqlExpression * expr,
  4657. ClusterType targetClusterType, unsigned clusterSize,
  4658. IHqlExpression * graphIdExpr, unsigned numResults, bool isChild, bool useGraphResults)
  4659. {
  4660. HqlExprArray transformed;
  4661. unsigned totalResults;
  4662. {
  4663. EclResourcer resourcer(translator.queryErrors(), translator.wu(), targetClusterType, clusterSize, translator.queryOptions());
  4664. if (isChild)
  4665. resourcer.setChildQuery(true);
  4666. resourcer.setNewChildQuery(graphIdExpr, numResults);
  4667. resourcer.setUseGraphResults(useGraphResults);
  4668. if (isSequentialActionList(expr))
  4669. resourcer.setSequential(true);
  4670. if (activeRows)
  4671. resourcer.tagActiveCursors(*activeRows);
  4672. resourcer.resourceGraph(expr, transformed);
  4673. totalResults = resourcer.numGraphResults();
  4674. }
  4675. hoistNestedCompound(translator, transformed);
  4676. if (totalResults == 0)
  4677. totalResults = 1;
  4678. transformed.append(*createAttribute(numResultsAtom, getSizetConstant(totalResults)));
  4679. transformed.append(*LINK(graphIdExpr));
  4680. if (isSequentialActionList(expr))
  4681. transformed.append(*createAttribute(sequentialAtom));
  4682. return createValue(no_subgraph, makeVoidType(), transformed);
  4683. }
  4684. IHqlExpression * resourceLibraryGraph(HqlCppTranslator & translator, IHqlExpression * expr, ClusterType targetClusterType, unsigned clusterSize, IHqlExpression * graphIdExpr, unsigned numResults)
  4685. {
  4686. return doResourceGraph(translator, NULL, expr, targetClusterType, clusterSize, graphIdExpr, numResults, false, true); //?? what value for isChild (e.g., thor library call). Need to gen twice?
  4687. }
  4688. IHqlExpression * resourceNewChildGraph(HqlCppTranslator & translator, HqlExprCopyArray & activeRows, IHqlExpression * expr, ClusterType targetClusterType, IHqlExpression * graphIdExpr, unsigned numResults)
  4689. {
  4690. return doResourceGraph(translator, &activeRows, expr, targetClusterType, 0, graphIdExpr, numResults, true, true);
  4691. }
  4692. IHqlExpression * resourceLoopGraph(HqlCppTranslator & translator, HqlExprCopyArray & activeRows, IHqlExpression * expr, ClusterType targetClusterType, IHqlExpression * graphIdExpr, unsigned numResults, bool insideChildQuery)
  4693. {
  4694. return doResourceGraph(translator, &activeRows, expr, targetClusterType, 0, graphIdExpr, numResults, insideChildQuery, true);
  4695. }
  4696. IHqlExpression * resourceRemoteGraph(HqlCppTranslator & translator, IHqlExpression * expr, ClusterType targetClusterType, unsigned clusterSize)
  4697. {
  4698. HqlExprArray transformed;
  4699. {
  4700. EclResourcer resourcer(translator.queryErrors(), translator.wu(), targetClusterType, clusterSize, translator.queryOptions());
  4701. resourcer.resourceRemoteGraph(expr, transformed);
  4702. }
  4703. hoistNestedCompound(translator, transformed);
  4704. return createActionList(transformed);
  4705. }
  4706. /*
  4707. Conditions:
  4708. They are nasty. We process the tree in two passes. First we tag anything which must be evaluated, and
  4709. save a list of condition statements to process later.
  4710. Second pass we tag conditionals.
  4711. a) all left and right branches of a condition are tagged. [conditionSourceCount]
  4712. b) all conditional expressions are tagged with the conditions they are evaluated for.
  4713. [if the condition lists match then it should be possible to merge the graphs]
  4714. c) The spill count for an node should ignore the number of links from conditional graphs,
  4715. but should add the number of conditions.
  4716. d) if (a, b(f1) +b(f2), c) needs to link b twice though!
  4717. */
  4718. /*
  4719. This transformer converts spill activities to no_dataset/no_output, and also converts splitters of splitters into
  4720. a single splitter.
  4721. */
  4722. class SpillActivityTransformer : public NewHqlTransformer
  4723. {
  4724. public:
  4725. SpillActivityTransformer(bool _createGraphResults);
  4726. protected:
  4727. virtual void analyseExpr(IHqlExpression * expr);
  4728. virtual IHqlExpression * createTransformed(IHqlExpression * expr);
  4729. bool isUnbalanced(IHqlExpression * body)
  4730. {
  4731. ANewTransformInfo * info = queryTransformExtra(body);
  4732. return info->spareByte1 != 0;
  4733. }
  4734. void setUnbalanced(IHqlExpression * body)
  4735. {
  4736. ANewTransformInfo * info = queryTransformExtra(body);
  4737. info->spareByte1 = true;
  4738. }
  4739. protected:
  4740. bool createGraphResults;
  4741. };
  4742. static HqlTransformerInfo spillActivityTransformerInfo("SpillActivityTransformer");
  4743. SpillActivityTransformer::SpillActivityTransformer(bool _createGraphResults)
  4744. : NewHqlTransformer(spillActivityTransformerInfo), createGraphResults(_createGraphResults)
  4745. {
  4746. }
  4747. void SpillActivityTransformer::analyseExpr(IHqlExpression * expr)
  4748. {
  4749. IHqlExpression * body = expr->queryBody();
  4750. if (alreadyVisited(body))
  4751. return;
  4752. //If splitters are commoned up ensure unbalanced splitters stay unbalanced.
  4753. if ((body->getOperator() == no_split) && !body->hasAttribute(balancedAtom))
  4754. {
  4755. IHqlExpression * splitter = NULL;
  4756. IHqlExpression * cur = body->queryChild(0);
  4757. loop
  4758. {
  4759. node_operator op = cur->getOperator();
  4760. if (op == no_split)
  4761. splitter = cur;
  4762. else if (op != no_commonspill)
  4763. break;
  4764. cur = cur->queryChild(0);
  4765. }
  4766. if (splitter)
  4767. setUnbalanced(splitter->queryBody());
  4768. }
  4769. NewHqlTransformer::analyseExpr(expr);
  4770. }
  4771. IHqlExpression * SpillActivityTransformer::createTransformed(IHqlExpression * expr)
  4772. {
  4773. IHqlExpression * annotation = queryTransformAnnotation(expr);
  4774. if (annotation)
  4775. return annotation;
  4776. switch (expr->getOperator())
  4777. {
  4778. case no_split:
  4779. {
  4780. OwnedHqlExpr input = transform(expr->queryChild(0));
  4781. if (input->getOperator() == no_split)
  4782. return input.getClear();
  4783. OwnedHqlExpr transformed = NewHqlTransformer::createTransformed(expr);
  4784. if (transformed->hasAttribute(balancedAtom) && isUnbalanced(expr))
  4785. return removeAttribute(transformed, balancedAtom);
  4786. return transformed.getClear();
  4787. }
  4788. case no_writespill:
  4789. {
  4790. HqlExprArray args;
  4791. transformChildren(expr, args);
  4792. if (createGraphResults)
  4793. return createValue(no_setgraphresult, makeVoidType(), args);
  4794. return createValue(no_output, makeVoidType(), args);
  4795. }
  4796. case no_commonspill:
  4797. return transform(expr->queryChild(0));
  4798. case no_readspill:
  4799. {
  4800. OwnedHqlExpr ds = transform(expr->queryChild(0));
  4801. node_operator readOp = createGraphResults ? no_getgraphresult : no_table;
  4802. HqlExprArray args;
  4803. if (!createGraphResults)
  4804. args.append(*transform(expr->queryChild(1)));
  4805. args.append(*LINK(ds->queryRecord()));
  4806. if (createGraphResults)
  4807. args.append(*transform(expr->queryChild(1)));
  4808. ForEachChildFrom(i, expr, 2)
  4809. {
  4810. IHqlExpression * cur = expr->queryChild(i);
  4811. args.append(*transform(cur));
  4812. }
  4813. IHqlExpression * recordCountAttr = queryRecordCountInfo(expr);
  4814. if (recordCountAttr)
  4815. args.append(*LINK(recordCountAttr));
  4816. OwnedHqlExpr ret;
  4817. if (ds->isDatarow())
  4818. ret.setown(createRow(readOp, args));
  4819. else if (ds->isDictionary())
  4820. ret.setown(createDictionary(readOp, args));
  4821. else
  4822. ret.setown(createDataset(readOp, args));
  4823. const bool loseDistribution = false;
  4824. return preserveTableInfo(ret, ds, loseDistribution, NULL);
  4825. }
  4826. }
  4827. return NewHqlTransformer::createTransformed(expr);
  4828. }
  4829. IHqlExpression * convertSpillsToActivities(IHqlExpression * expr, bool createGraphResults)
  4830. {
  4831. SpillActivityTransformer transformer(createGraphResults);
  4832. transformer.analyse(expr, 0);
  4833. return transformer.transformRoot(expr);
  4834. }