hqlhtcpp.cpp 641 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853685468556856685768586859686068616862686368646865686668676868686968706871687268736874687568766877687868796880688168826883688468856886688768886889689068916892689368946895689668976898689969006901690269036904690569066907690869096910691169126913691469156916691769186919692069216922692369246925692669276928692969306931693269336934693569366937693869396940694169426943694469456946694769486949695069516952695369546955695669576958695969606961696269636964696569666967696869696970697169726973697469756976697769786979698069816982698369846985698669876988698969906991699269936994699569966997699869997000700170027003700470057006700770087009701070117012701370147015701670177018701970207021702270237024702570267027702870297030703170327033703470357036703770387039704070417042704370447045704670477048704970507051705270537054705570567057705870597060706170627063706470657066706770687069707070717072707370747075707670777078707970807081708270837084708570867087708870897090709170927093709470957096709770987099710071017102710371047105710671077108710971107111711271137114711571167117711871197120712171227123712471257126712771287129713071317132713371347135713671377138713971407141714271437144714571467147714871497150715171527153715471557156715771587159716071617162716371647165716671677168716971707171717271737174717571767177717871797180718171827183718471857186718771887189719071917192719371947195719671977198719972007201720272037204720572067207720872097210721172127213721472157216721772187219722072217222722372247225722672277228722972307231723272337234723572367237723872397240724172427243724472457246724772487249725072517252725372547255725672577258725972607261726272637264726572667267726872697270727172727273727472757276727772787279728072817282728372847285728672877288728972907291729272937294729572967297729872997300730173027303730473057306730773087309731073117312731373147315731673177318731973207321732273237324732573267327732873297330733173327333733473357336733773387339734073417342734373447345734673477348734973507351735273537354735573567357735873597360736173627363736473657366736773687369737073717372737373747375737673777378737973807381738273837384738573867387738873897390739173927393739473957396739773987399740074017402740374047405740674077408740974107411741274137414741574167417741874197420742174227423742474257426742774287429743074317432743374347435743674377438743974407441744274437444744574467447744874497450745174527453745474557456745774587459746074617462746374647465746674677468746974707471747274737474747574767477747874797480748174827483748474857486748774887489749074917492749374947495749674977498749975007501750275037504750575067507750875097510751175127513751475157516751775187519752075217522752375247525752675277528752975307531753275337534753575367537753875397540754175427543754475457546754775487549755075517552755375547555755675577558755975607561756275637564756575667567756875697570757175727573757475757576757775787579758075817582758375847585758675877588758975907591759275937594759575967597759875997600760176027603760476057606760776087609761076117612761376147615761676177618761976207621762276237624762576267627762876297630763176327633763476357636763776387639764076417642764376447645764676477648764976507651765276537654765576567657765876597660766176627663766476657666766776687669767076717672767376747675767676777678767976807681768276837684768576867687768876897690769176927693769476957696769776987699770077017702770377047705770677077708770977107711771277137714771577167717771877197720772177227723772477257726772777287729773077317732773377347735773677377738773977407741774277437744774577467747774877497750775177527753775477557756775777587759776077617762776377647765776677677768776977707771777277737774777577767777777877797780778177827783778477857786778777887789779077917792779377947795779677977798779978007801780278037804780578067807780878097810781178127813781478157816781778187819782078217822782378247825782678277828782978307831783278337834783578367837783878397840784178427843784478457846784778487849785078517852785378547855785678577858785978607861786278637864786578667867786878697870787178727873787478757876787778787879788078817882788378847885788678877888788978907891789278937894789578967897789878997900790179027903790479057906790779087909791079117912791379147915791679177918791979207921792279237924792579267927792879297930793179327933793479357936793779387939794079417942794379447945794679477948794979507951795279537954795579567957795879597960796179627963796479657966796779687969797079717972797379747975797679777978797979807981798279837984798579867987798879897990799179927993799479957996799779987999800080018002800380048005800680078008800980108011801280138014801580168017801880198020802180228023802480258026802780288029803080318032803380348035803680378038803980408041804280438044804580468047804880498050805180528053805480558056805780588059806080618062806380648065806680678068806980708071807280738074807580768077807880798080808180828083808480858086808780888089809080918092809380948095809680978098809981008101810281038104810581068107810881098110811181128113811481158116811781188119812081218122812381248125812681278128812981308131813281338134813581368137813881398140814181428143814481458146814781488149815081518152815381548155815681578158815981608161816281638164816581668167816881698170817181728173817481758176817781788179818081818182818381848185818681878188818981908191819281938194819581968197819881998200820182028203820482058206820782088209821082118212821382148215821682178218821982208221822282238224822582268227822882298230823182328233823482358236823782388239824082418242824382448245824682478248824982508251825282538254825582568257825882598260826182628263826482658266826782688269827082718272827382748275827682778278827982808281828282838284828582868287828882898290829182928293829482958296829782988299830083018302830383048305830683078308830983108311831283138314831583168317831883198320832183228323832483258326832783288329833083318332833383348335833683378338833983408341834283438344834583468347834883498350835183528353835483558356835783588359836083618362836383648365836683678368836983708371837283738374837583768377837883798380838183828383838483858386838783888389839083918392839383948395839683978398839984008401840284038404840584068407840884098410841184128413841484158416841784188419842084218422842384248425842684278428842984308431843284338434843584368437843884398440844184428443844484458446844784488449845084518452845384548455845684578458845984608461846284638464846584668467846884698470847184728473847484758476847784788479848084818482848384848485848684878488848984908491849284938494849584968497849884998500850185028503850485058506850785088509851085118512851385148515851685178518851985208521852285238524852585268527852885298530853185328533853485358536853785388539854085418542854385448545854685478548854985508551855285538554855585568557855885598560856185628563856485658566856785688569857085718572857385748575857685778578857985808581858285838584858585868587858885898590859185928593859485958596859785988599860086018602860386048605860686078608860986108611861286138614861586168617861886198620862186228623862486258626862786288629863086318632863386348635863686378638863986408641864286438644864586468647864886498650865186528653865486558656865786588659866086618662866386648665866686678668866986708671867286738674867586768677867886798680868186828683868486858686868786888689869086918692869386948695869686978698869987008701870287038704870587068707870887098710871187128713871487158716871787188719872087218722872387248725872687278728872987308731873287338734873587368737873887398740874187428743874487458746874787488749875087518752875387548755875687578758875987608761876287638764876587668767876887698770877187728773877487758776877787788779878087818782878387848785878687878788878987908791879287938794879587968797879887998800880188028803880488058806880788088809881088118812881388148815881688178818881988208821882288238824882588268827882888298830883188328833883488358836883788388839884088418842884388448845884688478848884988508851885288538854885588568857885888598860886188628863886488658866886788688869887088718872887388748875887688778878887988808881888288838884888588868887888888898890889188928893889488958896889788988899890089018902890389048905890689078908890989108911891289138914891589168917891889198920892189228923892489258926892789288929893089318932893389348935893689378938893989408941894289438944894589468947894889498950895189528953895489558956895789588959896089618962896389648965896689678968896989708971897289738974897589768977897889798980898189828983898489858986898789888989899089918992899389948995899689978998899990009001900290039004900590069007900890099010901190129013901490159016901790189019902090219022902390249025902690279028902990309031903290339034903590369037903890399040904190429043904490459046904790489049905090519052905390549055905690579058905990609061906290639064906590669067906890699070907190729073907490759076907790789079908090819082908390849085908690879088908990909091909290939094909590969097909890999100910191029103910491059106910791089109911091119112911391149115911691179118911991209121912291239124912591269127912891299130913191329133913491359136913791389139914091419142914391449145914691479148914991509151915291539154915591569157915891599160916191629163916491659166916791689169917091719172917391749175917691779178917991809181918291839184918591869187918891899190919191929193919491959196919791989199920092019202920392049205920692079208920992109211921292139214921592169217921892199220922192229223922492259226922792289229923092319232923392349235923692379238923992409241924292439244924592469247924892499250925192529253925492559256925792589259926092619262926392649265926692679268926992709271927292739274927592769277927892799280928192829283928492859286928792889289929092919292929392949295929692979298929993009301930293039304930593069307930893099310931193129313931493159316931793189319932093219322932393249325932693279328932993309331933293339334933593369337933893399340934193429343934493459346934793489349935093519352935393549355935693579358935993609361936293639364936593669367936893699370937193729373937493759376937793789379938093819382938393849385938693879388938993909391939293939394939593969397939893999400940194029403940494059406940794089409941094119412941394149415941694179418941994209421942294239424942594269427942894299430943194329433943494359436943794389439944094419442944394449445944694479448944994509451945294539454945594569457945894599460946194629463946494659466946794689469947094719472947394749475947694779478947994809481948294839484948594869487948894899490949194929493949494959496949794989499950095019502950395049505950695079508950995109511951295139514951595169517951895199520952195229523952495259526952795289529953095319532953395349535953695379538953995409541954295439544954595469547954895499550955195529553955495559556955795589559956095619562956395649565956695679568956995709571957295739574957595769577957895799580958195829583958495859586958795889589959095919592959395949595959695979598959996009601960296039604960596069607960896099610961196129613961496159616961796189619962096219622962396249625962696279628962996309631963296339634963596369637963896399640964196429643964496459646964796489649965096519652965396549655965696579658965996609661966296639664966596669667966896699670967196729673967496759676967796789679968096819682968396849685968696879688968996909691969296939694969596969697969896999700970197029703970497059706970797089709971097119712971397149715971697179718971997209721972297239724972597269727972897299730973197329733973497359736973797389739974097419742974397449745974697479748974997509751975297539754975597569757975897599760976197629763976497659766976797689769977097719772977397749775977697779778977997809781978297839784978597869787978897899790979197929793979497959796979797989799980098019802980398049805980698079808980998109811981298139814981598169817981898199820982198229823982498259826982798289829983098319832983398349835983698379838983998409841984298439844984598469847984898499850985198529853985498559856985798589859986098619862986398649865986698679868986998709871987298739874987598769877987898799880988198829883988498859886988798889889989098919892989398949895989698979898989999009901990299039904990599069907990899099910991199129913991499159916991799189919992099219922992399249925992699279928992999309931993299339934993599369937993899399940994199429943994499459946994799489949995099519952995399549955995699579958995999609961996299639964996599669967996899699970997199729973997499759976997799789979998099819982998399849985998699879988998999909991999299939994999599969997999899991000010001100021000310004100051000610007100081000910010100111001210013100141001510016100171001810019100201002110022100231002410025100261002710028100291003010031100321003310034100351003610037100381003910040100411004210043100441004510046100471004810049100501005110052100531005410055100561005710058100591006010061100621006310064100651006610067100681006910070100711007210073100741007510076100771007810079100801008110082100831008410085100861008710088100891009010091100921009310094100951009610097100981009910100101011010210103101041010510106101071010810109101101011110112101131011410115101161011710118101191012010121101221012310124101251012610127101281012910130101311013210133101341013510136101371013810139101401014110142101431014410145101461014710148101491015010151101521015310154101551015610157101581015910160101611016210163101641016510166101671016810169101701017110172101731017410175101761017710178101791018010181101821018310184101851018610187101881018910190101911019210193101941019510196101971019810199102001020110202102031020410205102061020710208102091021010211102121021310214102151021610217102181021910220102211022210223102241022510226102271022810229102301023110232102331023410235102361023710238102391024010241102421024310244102451024610247102481024910250102511025210253102541025510256102571025810259102601026110262102631026410265102661026710268102691027010271102721027310274102751027610277102781027910280102811028210283102841028510286102871028810289102901029110292102931029410295102961029710298102991030010301103021030310304103051030610307103081030910310103111031210313103141031510316103171031810319103201032110322103231032410325103261032710328103291033010331103321033310334103351033610337103381033910340103411034210343103441034510346103471034810349103501035110352103531035410355103561035710358103591036010361103621036310364103651036610367103681036910370103711037210373103741037510376103771037810379103801038110382103831038410385103861038710388103891039010391103921039310394103951039610397103981039910400104011040210403104041040510406104071040810409104101041110412104131041410415104161041710418104191042010421104221042310424104251042610427104281042910430104311043210433104341043510436104371043810439104401044110442104431044410445104461044710448104491045010451104521045310454104551045610457104581045910460104611046210463104641046510466104671046810469104701047110472104731047410475104761047710478104791048010481104821048310484104851048610487104881048910490104911049210493104941049510496104971049810499105001050110502105031050410505105061050710508105091051010511105121051310514105151051610517105181051910520105211052210523105241052510526105271052810529105301053110532105331053410535105361053710538105391054010541105421054310544105451054610547105481054910550105511055210553105541055510556105571055810559105601056110562105631056410565105661056710568105691057010571105721057310574105751057610577105781057910580105811058210583105841058510586105871058810589105901059110592105931059410595105961059710598105991060010601106021060310604106051060610607106081060910610106111061210613106141061510616106171061810619106201062110622106231062410625106261062710628106291063010631106321063310634106351063610637106381063910640106411064210643106441064510646106471064810649106501065110652106531065410655106561065710658106591066010661106621066310664106651066610667106681066910670106711067210673106741067510676106771067810679106801068110682106831068410685106861068710688106891069010691106921069310694106951069610697106981069910700107011070210703107041070510706107071070810709107101071110712107131071410715107161071710718107191072010721107221072310724107251072610727107281072910730107311073210733107341073510736107371073810739107401074110742107431074410745107461074710748107491075010751107521075310754107551075610757107581075910760107611076210763107641076510766107671076810769107701077110772107731077410775107761077710778107791078010781107821078310784107851078610787107881078910790107911079210793107941079510796107971079810799108001080110802108031080410805108061080710808108091081010811108121081310814108151081610817108181081910820108211082210823108241082510826108271082810829108301083110832108331083410835108361083710838108391084010841108421084310844108451084610847108481084910850108511085210853108541085510856108571085810859108601086110862108631086410865108661086710868108691087010871108721087310874108751087610877108781087910880108811088210883108841088510886108871088810889108901089110892108931089410895108961089710898108991090010901109021090310904109051090610907109081090910910109111091210913109141091510916109171091810919109201092110922109231092410925109261092710928109291093010931109321093310934109351093610937109381093910940109411094210943109441094510946109471094810949109501095110952109531095410955109561095710958109591096010961109621096310964109651096610967109681096910970109711097210973109741097510976109771097810979109801098110982109831098410985109861098710988109891099010991109921099310994109951099610997109981099911000110011100211003110041100511006110071100811009110101101111012110131101411015110161101711018110191102011021110221102311024110251102611027110281102911030110311103211033110341103511036110371103811039110401104111042110431104411045110461104711048110491105011051110521105311054110551105611057110581105911060110611106211063110641106511066110671106811069110701107111072110731107411075110761107711078110791108011081110821108311084110851108611087110881108911090110911109211093110941109511096110971109811099111001110111102111031110411105111061110711108111091111011111111121111311114111151111611117111181111911120111211112211123111241112511126111271112811129111301113111132111331113411135111361113711138111391114011141111421114311144111451114611147111481114911150111511115211153111541115511156111571115811159111601116111162111631116411165111661116711168111691117011171111721117311174111751117611177111781117911180111811118211183111841118511186111871118811189111901119111192111931119411195111961119711198111991120011201112021120311204112051120611207112081120911210112111121211213112141121511216112171121811219112201122111222112231122411225112261122711228112291123011231112321123311234112351123611237112381123911240112411124211243112441124511246112471124811249112501125111252112531125411255112561125711258112591126011261112621126311264112651126611267112681126911270112711127211273112741127511276112771127811279112801128111282112831128411285112861128711288112891129011291112921129311294112951129611297112981129911300113011130211303113041130511306113071130811309113101131111312113131131411315113161131711318113191132011321113221132311324113251132611327113281132911330113311133211333113341133511336113371133811339113401134111342113431134411345113461134711348113491135011351113521135311354113551135611357113581135911360113611136211363113641136511366113671136811369113701137111372113731137411375113761137711378113791138011381113821138311384113851138611387113881138911390113911139211393113941139511396113971139811399114001140111402114031140411405114061140711408114091141011411114121141311414114151141611417114181141911420114211142211423114241142511426114271142811429114301143111432114331143411435114361143711438114391144011441114421144311444114451144611447114481144911450114511145211453114541145511456114571145811459114601146111462114631146411465114661146711468114691147011471114721147311474114751147611477114781147911480114811148211483114841148511486114871148811489114901149111492114931149411495114961149711498114991150011501115021150311504115051150611507115081150911510115111151211513115141151511516115171151811519115201152111522115231152411525115261152711528115291153011531115321153311534115351153611537115381153911540115411154211543115441154511546115471154811549115501155111552115531155411555115561155711558115591156011561115621156311564115651156611567115681156911570115711157211573115741157511576115771157811579115801158111582115831158411585115861158711588115891159011591115921159311594115951159611597115981159911600116011160211603116041160511606116071160811609116101161111612116131161411615116161161711618116191162011621116221162311624116251162611627116281162911630116311163211633116341163511636116371163811639116401164111642116431164411645116461164711648116491165011651116521165311654116551165611657116581165911660116611166211663116641166511666116671166811669116701167111672116731167411675116761167711678116791168011681116821168311684116851168611687116881168911690116911169211693116941169511696116971169811699117001170111702117031170411705117061170711708117091171011711117121171311714117151171611717117181171911720117211172211723117241172511726117271172811729117301173111732117331173411735117361173711738117391174011741117421174311744117451174611747117481174911750117511175211753117541175511756117571175811759117601176111762117631176411765117661176711768117691177011771117721177311774117751177611777117781177911780117811178211783117841178511786117871178811789117901179111792117931179411795117961179711798117991180011801118021180311804118051180611807118081180911810118111181211813118141181511816118171181811819118201182111822118231182411825118261182711828118291183011831118321183311834118351183611837118381183911840118411184211843118441184511846118471184811849118501185111852118531185411855118561185711858118591186011861118621186311864118651186611867118681186911870118711187211873118741187511876118771187811879118801188111882118831188411885118861188711888118891189011891118921189311894118951189611897118981189911900119011190211903119041190511906119071190811909119101191111912119131191411915119161191711918119191192011921119221192311924119251192611927119281192911930119311193211933119341193511936119371193811939119401194111942119431194411945119461194711948119491195011951119521195311954119551195611957119581195911960119611196211963119641196511966119671196811969119701197111972119731197411975119761197711978119791198011981119821198311984119851198611987119881198911990119911199211993119941199511996119971199811999120001200112002120031200412005120061200712008120091201012011120121201312014120151201612017120181201912020120211202212023120241202512026120271202812029120301203112032120331203412035120361203712038120391204012041120421204312044120451204612047120481204912050120511205212053120541205512056120571205812059120601206112062120631206412065120661206712068120691207012071120721207312074120751207612077120781207912080120811208212083120841208512086120871208812089120901209112092120931209412095120961209712098120991210012101121021210312104121051210612107121081210912110121111211212113121141211512116121171211812119121201212112122121231212412125121261212712128121291213012131121321213312134121351213612137121381213912140121411214212143121441214512146121471214812149121501215112152121531215412155121561215712158121591216012161121621216312164121651216612167121681216912170121711217212173121741217512176121771217812179121801218112182121831218412185121861218712188121891219012191121921219312194121951219612197121981219912200122011220212203122041220512206122071220812209122101221112212122131221412215122161221712218122191222012221122221222312224122251222612227122281222912230122311223212233122341223512236122371223812239122401224112242122431224412245122461224712248122491225012251122521225312254122551225612257122581225912260122611226212263122641226512266122671226812269122701227112272122731227412275122761227712278122791228012281122821228312284122851228612287122881228912290122911229212293122941229512296122971229812299123001230112302123031230412305123061230712308123091231012311123121231312314123151231612317123181231912320123211232212323123241232512326123271232812329123301233112332123331233412335123361233712338123391234012341123421234312344123451234612347123481234912350123511235212353123541235512356123571235812359123601236112362123631236412365123661236712368123691237012371123721237312374123751237612377123781237912380123811238212383123841238512386123871238812389123901239112392123931239412395123961239712398123991240012401124021240312404124051240612407124081240912410124111241212413124141241512416124171241812419124201242112422124231242412425124261242712428124291243012431124321243312434124351243612437124381243912440124411244212443124441244512446124471244812449124501245112452124531245412455124561245712458124591246012461124621246312464124651246612467124681246912470124711247212473124741247512476124771247812479124801248112482124831248412485124861248712488124891249012491124921249312494124951249612497124981249912500125011250212503125041250512506125071250812509125101251112512125131251412515125161251712518125191252012521125221252312524125251252612527125281252912530125311253212533125341253512536125371253812539125401254112542125431254412545125461254712548125491255012551125521255312554125551255612557125581255912560125611256212563125641256512566125671256812569125701257112572125731257412575125761257712578125791258012581125821258312584125851258612587125881258912590125911259212593125941259512596125971259812599126001260112602126031260412605126061260712608126091261012611126121261312614126151261612617126181261912620126211262212623126241262512626126271262812629126301263112632126331263412635126361263712638126391264012641126421264312644126451264612647126481264912650126511265212653126541265512656126571265812659126601266112662126631266412665126661266712668126691267012671126721267312674126751267612677126781267912680126811268212683126841268512686126871268812689126901269112692126931269412695126961269712698126991270012701127021270312704127051270612707127081270912710127111271212713127141271512716127171271812719127201272112722127231272412725127261272712728127291273012731127321273312734127351273612737127381273912740127411274212743127441274512746127471274812749127501275112752127531275412755127561275712758127591276012761127621276312764127651276612767127681276912770127711277212773127741277512776127771277812779127801278112782127831278412785127861278712788127891279012791127921279312794127951279612797127981279912800128011280212803128041280512806128071280812809128101281112812128131281412815128161281712818128191282012821128221282312824128251282612827128281282912830128311283212833128341283512836128371283812839128401284112842128431284412845128461284712848128491285012851128521285312854128551285612857128581285912860128611286212863128641286512866128671286812869128701287112872128731287412875128761287712878128791288012881128821288312884128851288612887128881288912890128911289212893128941289512896128971289812899129001290112902129031290412905129061290712908129091291012911129121291312914129151291612917129181291912920129211292212923129241292512926129271292812929129301293112932129331293412935129361293712938129391294012941129421294312944129451294612947129481294912950129511295212953129541295512956129571295812959129601296112962129631296412965129661296712968129691297012971129721297312974129751297612977129781297912980129811298212983129841298512986129871298812989129901299112992129931299412995129961299712998129991300013001130021300313004130051300613007130081300913010130111301213013130141301513016130171301813019130201302113022130231302413025130261302713028130291303013031130321303313034130351303613037130381303913040130411304213043130441304513046130471304813049130501305113052130531305413055130561305713058130591306013061130621306313064130651306613067130681306913070130711307213073130741307513076130771307813079130801308113082130831308413085130861308713088130891309013091130921309313094130951309613097130981309913100131011310213103131041310513106131071310813109131101311113112131131311413115131161311713118131191312013121131221312313124131251312613127131281312913130131311313213133131341313513136131371313813139131401314113142131431314413145131461314713148131491315013151131521315313154131551315613157131581315913160131611316213163131641316513166131671316813169131701317113172131731317413175131761317713178131791318013181131821318313184131851318613187131881318913190131911319213193131941319513196131971319813199132001320113202132031320413205132061320713208132091321013211132121321313214132151321613217132181321913220132211322213223132241322513226132271322813229132301323113232132331323413235132361323713238132391324013241132421324313244132451324613247132481324913250132511325213253132541325513256132571325813259132601326113262132631326413265132661326713268132691327013271132721327313274132751327613277132781327913280132811328213283132841328513286132871328813289132901329113292132931329413295132961329713298132991330013301133021330313304133051330613307133081330913310133111331213313133141331513316133171331813319133201332113322133231332413325133261332713328133291333013331133321333313334133351333613337133381333913340133411334213343133441334513346133471334813349133501335113352133531335413355133561335713358133591336013361133621336313364133651336613367133681336913370133711337213373133741337513376133771337813379133801338113382133831338413385133861338713388133891339013391133921339313394133951339613397133981339913400134011340213403134041340513406134071340813409134101341113412134131341413415134161341713418134191342013421134221342313424134251342613427134281342913430134311343213433134341343513436134371343813439134401344113442134431344413445134461344713448134491345013451134521345313454134551345613457134581345913460134611346213463134641346513466134671346813469134701347113472134731347413475134761347713478134791348013481134821348313484134851348613487134881348913490134911349213493134941349513496134971349813499135001350113502135031350413505135061350713508135091351013511135121351313514135151351613517135181351913520135211352213523135241352513526135271352813529135301353113532135331353413535135361353713538135391354013541135421354313544135451354613547135481354913550135511355213553135541355513556135571355813559135601356113562135631356413565135661356713568135691357013571135721357313574135751357613577135781357913580135811358213583135841358513586135871358813589135901359113592135931359413595135961359713598135991360013601136021360313604136051360613607136081360913610136111361213613136141361513616136171361813619136201362113622136231362413625136261362713628136291363013631136321363313634136351363613637136381363913640136411364213643136441364513646136471364813649136501365113652136531365413655136561365713658136591366013661136621366313664136651366613667136681366913670136711367213673136741367513676136771367813679136801368113682136831368413685136861368713688136891369013691136921369313694136951369613697136981369913700137011370213703137041370513706137071370813709137101371113712137131371413715137161371713718137191372013721137221372313724137251372613727137281372913730137311373213733137341373513736137371373813739137401374113742137431374413745137461374713748137491375013751137521375313754137551375613757137581375913760137611376213763137641376513766137671376813769137701377113772137731377413775137761377713778137791378013781137821378313784137851378613787137881378913790137911379213793137941379513796137971379813799138001380113802138031380413805138061380713808138091381013811138121381313814138151381613817138181381913820138211382213823138241382513826138271382813829138301383113832138331383413835138361383713838138391384013841138421384313844138451384613847138481384913850138511385213853138541385513856138571385813859138601386113862138631386413865138661386713868138691387013871138721387313874138751387613877138781387913880138811388213883138841388513886138871388813889138901389113892138931389413895138961389713898138991390013901139021390313904139051390613907139081390913910139111391213913139141391513916139171391813919139201392113922139231392413925139261392713928139291393013931139321393313934139351393613937139381393913940139411394213943139441394513946139471394813949139501395113952139531395413955139561395713958139591396013961139621396313964139651396613967139681396913970139711397213973139741397513976139771397813979139801398113982139831398413985139861398713988139891399013991139921399313994139951399613997139981399914000140011400214003140041400514006140071400814009140101401114012140131401414015140161401714018140191402014021140221402314024140251402614027140281402914030140311403214033140341403514036140371403814039140401404114042140431404414045140461404714048140491405014051140521405314054140551405614057140581405914060140611406214063140641406514066140671406814069140701407114072140731407414075140761407714078140791408014081140821408314084140851408614087140881408914090140911409214093140941409514096140971409814099141001410114102141031410414105141061410714108141091411014111141121411314114141151411614117141181411914120141211412214123141241412514126141271412814129141301413114132141331413414135141361413714138141391414014141141421414314144141451414614147141481414914150141511415214153141541415514156141571415814159141601416114162141631416414165141661416714168141691417014171141721417314174141751417614177141781417914180141811418214183141841418514186141871418814189141901419114192141931419414195141961419714198141991420014201142021420314204142051420614207142081420914210142111421214213142141421514216142171421814219142201422114222142231422414225142261422714228142291423014231142321423314234142351423614237142381423914240142411424214243142441424514246142471424814249142501425114252142531425414255142561425714258142591426014261142621426314264142651426614267142681426914270142711427214273142741427514276142771427814279142801428114282142831428414285142861428714288142891429014291142921429314294142951429614297142981429914300143011430214303143041430514306143071430814309143101431114312143131431414315143161431714318143191432014321143221432314324143251432614327143281432914330143311433214333143341433514336143371433814339143401434114342143431434414345143461434714348143491435014351143521435314354143551435614357143581435914360143611436214363143641436514366143671436814369143701437114372143731437414375143761437714378143791438014381143821438314384143851438614387143881438914390143911439214393143941439514396143971439814399144001440114402144031440414405144061440714408144091441014411144121441314414144151441614417144181441914420144211442214423144241442514426144271442814429144301443114432144331443414435144361443714438144391444014441144421444314444144451444614447144481444914450144511445214453144541445514456144571445814459144601446114462144631446414465144661446714468144691447014471144721447314474144751447614477144781447914480144811448214483144841448514486144871448814489144901449114492144931449414495144961449714498144991450014501145021450314504145051450614507145081450914510145111451214513145141451514516145171451814519145201452114522145231452414525145261452714528145291453014531145321453314534145351453614537145381453914540145411454214543145441454514546145471454814549145501455114552145531455414555145561455714558145591456014561145621456314564145651456614567145681456914570145711457214573145741457514576145771457814579145801458114582145831458414585145861458714588145891459014591145921459314594145951459614597145981459914600146011460214603146041460514606146071460814609146101461114612146131461414615146161461714618146191462014621146221462314624146251462614627146281462914630146311463214633146341463514636146371463814639146401464114642146431464414645146461464714648146491465014651146521465314654146551465614657146581465914660146611466214663146641466514666146671466814669146701467114672146731467414675146761467714678146791468014681146821468314684146851468614687146881468914690146911469214693146941469514696146971469814699147001470114702147031470414705147061470714708147091471014711147121471314714147151471614717147181471914720147211472214723147241472514726147271472814729147301473114732147331473414735147361473714738147391474014741147421474314744147451474614747147481474914750147511475214753147541475514756147571475814759147601476114762147631476414765147661476714768147691477014771147721477314774147751477614777147781477914780147811478214783147841478514786147871478814789147901479114792147931479414795147961479714798147991480014801148021480314804148051480614807148081480914810148111481214813148141481514816148171481814819148201482114822148231482414825148261482714828148291483014831148321483314834148351483614837148381483914840148411484214843148441484514846148471484814849148501485114852148531485414855148561485714858148591486014861148621486314864148651486614867148681486914870148711487214873148741487514876148771487814879148801488114882148831488414885148861488714888148891489014891148921489314894148951489614897148981489914900149011490214903149041490514906149071490814909149101491114912149131491414915149161491714918149191492014921149221492314924149251492614927149281492914930149311493214933149341493514936149371493814939149401494114942149431494414945149461494714948149491495014951149521495314954149551495614957149581495914960149611496214963149641496514966149671496814969149701497114972149731497414975149761497714978149791498014981149821498314984149851498614987149881498914990149911499214993149941499514996149971499814999150001500115002150031500415005150061500715008150091501015011150121501315014150151501615017150181501915020150211502215023150241502515026150271502815029150301503115032150331503415035150361503715038150391504015041150421504315044150451504615047150481504915050150511505215053150541505515056150571505815059150601506115062150631506415065150661506715068150691507015071150721507315074150751507615077150781507915080150811508215083150841508515086150871508815089150901509115092150931509415095150961509715098150991510015101151021510315104151051510615107151081510915110151111511215113151141511515116151171511815119151201512115122151231512415125151261512715128151291513015131151321513315134151351513615137151381513915140151411514215143151441514515146151471514815149151501515115152151531515415155151561515715158151591516015161151621516315164151651516615167151681516915170151711517215173151741517515176151771517815179151801518115182151831518415185151861518715188151891519015191151921519315194151951519615197151981519915200152011520215203152041520515206152071520815209152101521115212152131521415215152161521715218152191522015221152221522315224152251522615227152281522915230152311523215233152341523515236152371523815239152401524115242152431524415245152461524715248152491525015251152521525315254152551525615257152581525915260152611526215263152641526515266152671526815269152701527115272152731527415275152761527715278152791528015281152821528315284152851528615287152881528915290152911529215293152941529515296152971529815299153001530115302153031530415305153061530715308153091531015311153121531315314153151531615317153181531915320153211532215323153241532515326153271532815329153301533115332153331533415335153361533715338153391534015341153421534315344153451534615347153481534915350153511535215353153541535515356153571535815359153601536115362153631536415365153661536715368153691537015371153721537315374153751537615377153781537915380153811538215383153841538515386153871538815389153901539115392153931539415395153961539715398153991540015401154021540315404154051540615407154081540915410154111541215413154141541515416154171541815419154201542115422154231542415425154261542715428154291543015431154321543315434154351543615437154381543915440154411544215443154441544515446154471544815449154501545115452154531545415455154561545715458154591546015461154621546315464154651546615467154681546915470154711547215473154741547515476154771547815479154801548115482154831548415485154861548715488154891549015491154921549315494154951549615497154981549915500155011550215503155041550515506155071550815509155101551115512155131551415515155161551715518155191552015521155221552315524155251552615527155281552915530155311553215533155341553515536155371553815539155401554115542155431554415545155461554715548155491555015551155521555315554155551555615557155581555915560155611556215563155641556515566155671556815569155701557115572155731557415575155761557715578155791558015581155821558315584155851558615587155881558915590155911559215593155941559515596155971559815599156001560115602156031560415605156061560715608156091561015611156121561315614156151561615617156181561915620156211562215623156241562515626156271562815629156301563115632156331563415635156361563715638156391564015641156421564315644156451564615647156481564915650156511565215653156541565515656156571565815659156601566115662156631566415665156661566715668156691567015671156721567315674156751567615677156781567915680156811568215683156841568515686156871568815689156901569115692156931569415695156961569715698156991570015701157021570315704157051570615707157081570915710157111571215713157141571515716157171571815719157201572115722157231572415725157261572715728157291573015731157321573315734157351573615737157381573915740157411574215743157441574515746157471574815749157501575115752157531575415755157561575715758157591576015761157621576315764157651576615767157681576915770157711577215773157741577515776157771577815779157801578115782157831578415785157861578715788157891579015791157921579315794157951579615797157981579915800158011580215803158041580515806158071580815809158101581115812158131581415815158161581715818158191582015821158221582315824158251582615827158281582915830158311583215833158341583515836158371583815839158401584115842158431584415845158461584715848158491585015851158521585315854158551585615857158581585915860158611586215863158641586515866158671586815869158701587115872158731587415875158761587715878158791588015881158821588315884158851588615887158881588915890158911589215893158941589515896158971589815899159001590115902159031590415905159061590715908159091591015911159121591315914159151591615917159181591915920159211592215923159241592515926159271592815929159301593115932159331593415935159361593715938159391594015941159421594315944159451594615947159481594915950159511595215953159541595515956159571595815959159601596115962159631596415965159661596715968159691597015971159721597315974159751597615977159781597915980159811598215983159841598515986159871598815989159901599115992159931599415995159961599715998159991600016001160021600316004160051600616007160081600916010160111601216013160141601516016160171601816019160201602116022160231602416025160261602716028160291603016031160321603316034160351603616037160381603916040160411604216043160441604516046160471604816049160501605116052160531605416055160561605716058160591606016061160621606316064160651606616067160681606916070160711607216073160741607516076160771607816079160801608116082160831608416085160861608716088160891609016091160921609316094160951609616097160981609916100161011610216103161041610516106161071610816109161101611116112161131611416115161161611716118161191612016121161221612316124161251612616127161281612916130161311613216133161341613516136161371613816139161401614116142161431614416145161461614716148161491615016151161521615316154161551615616157161581615916160161611616216163161641616516166161671616816169161701617116172161731617416175161761617716178161791618016181161821618316184161851618616187161881618916190161911619216193161941619516196161971619816199162001620116202162031620416205162061620716208162091621016211162121621316214162151621616217162181621916220162211622216223162241622516226162271622816229162301623116232162331623416235162361623716238162391624016241162421624316244162451624616247162481624916250162511625216253162541625516256162571625816259162601626116262162631626416265162661626716268162691627016271162721627316274162751627616277162781627916280162811628216283162841628516286162871628816289162901629116292162931629416295162961629716298162991630016301163021630316304163051630616307163081630916310163111631216313163141631516316163171631816319163201632116322163231632416325163261632716328163291633016331163321633316334163351633616337163381633916340163411634216343163441634516346163471634816349163501635116352163531635416355163561635716358163591636016361163621636316364163651636616367163681636916370163711637216373163741637516376163771637816379163801638116382163831638416385163861638716388163891639016391163921639316394163951639616397163981639916400164011640216403164041640516406164071640816409164101641116412164131641416415164161641716418164191642016421164221642316424164251642616427164281642916430164311643216433164341643516436164371643816439164401644116442164431644416445164461644716448164491645016451164521645316454164551645616457164581645916460164611646216463164641646516466164671646816469164701647116472164731647416475164761647716478164791648016481164821648316484164851648616487164881648916490164911649216493164941649516496164971649816499165001650116502165031650416505165061650716508165091651016511165121651316514165151651616517165181651916520165211652216523165241652516526165271652816529165301653116532165331653416535165361653716538165391654016541165421654316544165451654616547165481654916550165511655216553165541655516556165571655816559165601656116562165631656416565165661656716568165691657016571165721657316574165751657616577165781657916580165811658216583165841658516586165871658816589165901659116592165931659416595165961659716598165991660016601166021660316604166051660616607166081660916610166111661216613166141661516616166171661816619166201662116622166231662416625166261662716628166291663016631166321663316634166351663616637166381663916640166411664216643166441664516646166471664816649166501665116652166531665416655166561665716658166591666016661166621666316664166651666616667166681666916670166711667216673166741667516676166771667816679166801668116682166831668416685166861668716688166891669016691166921669316694166951669616697166981669916700167011670216703167041670516706167071670816709167101671116712167131671416715167161671716718167191672016721167221672316724167251672616727167281672916730167311673216733167341673516736167371673816739167401674116742167431674416745167461674716748167491675016751167521675316754167551675616757167581675916760167611676216763167641676516766167671676816769167701677116772167731677416775167761677716778167791678016781167821678316784167851678616787167881678916790167911679216793167941679516796167971679816799168001680116802168031680416805168061680716808168091681016811168121681316814168151681616817168181681916820168211682216823168241682516826168271682816829168301683116832168331683416835168361683716838168391684016841168421684316844168451684616847168481684916850168511685216853168541685516856168571685816859168601686116862168631686416865168661686716868168691687016871168721687316874168751687616877168781687916880168811688216883168841688516886168871688816889168901689116892168931689416895168961689716898168991690016901169021690316904169051690616907169081690916910169111691216913169141691516916169171691816919169201692116922169231692416925169261692716928169291693016931169321693316934169351693616937169381693916940169411694216943169441694516946169471694816949169501695116952169531695416955169561695716958169591696016961169621696316964169651696616967169681696916970169711697216973169741697516976169771697816979169801698116982169831698416985169861698716988169891699016991169921699316994169951699616997169981699917000170011700217003170041700517006170071700817009170101701117012170131701417015170161701717018170191702017021170221702317024170251702617027170281702917030170311703217033170341703517036170371703817039170401704117042170431704417045170461704717048170491705017051170521705317054170551705617057170581705917060170611706217063170641706517066170671706817069170701707117072170731707417075170761707717078170791708017081170821708317084170851708617087170881708917090170911709217093170941709517096170971709817099171001710117102171031710417105171061710717108171091711017111171121711317114171151711617117171181711917120171211712217123171241712517126171271712817129171301713117132171331713417135171361713717138171391714017141171421714317144171451714617147171481714917150171511715217153171541715517156171571715817159171601716117162171631716417165171661716717168171691717017171171721717317174171751717617177171781717917180171811718217183171841718517186171871718817189171901719117192171931719417195171961719717198171991720017201172021720317204172051720617207172081720917210172111721217213172141721517216172171721817219172201722117222172231722417225172261722717228172291723017231172321723317234172351723617237172381723917240172411724217243172441724517246172471724817249172501725117252172531725417255172561725717258172591726017261172621726317264172651726617267172681726917270172711727217273172741727517276172771727817279172801728117282172831728417285172861728717288172891729017291172921729317294172951729617297172981729917300173011730217303173041730517306173071730817309173101731117312173131731417315173161731717318173191732017321173221732317324173251732617327173281732917330173311733217333173341733517336173371733817339173401734117342173431734417345173461734717348173491735017351173521735317354173551735617357173581735917360173611736217363173641736517366173671736817369173701737117372173731737417375173761737717378173791738017381173821738317384173851738617387173881738917390173911739217393173941739517396173971739817399174001740117402174031740417405174061740717408174091741017411174121741317414174151741617417174181741917420174211742217423174241742517426174271742817429174301743117432174331743417435174361743717438174391744017441174421744317444174451744617447174481744917450174511745217453174541745517456174571745817459174601746117462174631746417465174661746717468174691747017471174721747317474174751747617477174781747917480174811748217483174841748517486174871748817489174901749117492174931749417495174961749717498174991750017501175021750317504175051750617507175081750917510175111751217513175141751517516175171751817519175201752117522175231752417525175261752717528175291753017531175321753317534175351753617537175381753917540175411754217543175441754517546175471754817549175501755117552175531755417555175561755717558175591756017561175621756317564175651756617567175681756917570175711757217573175741757517576175771757817579175801758117582175831758417585175861758717588175891759017591175921759317594175951759617597175981759917600176011760217603176041760517606176071760817609176101761117612176131761417615176161761717618176191762017621176221762317624176251762617627176281762917630176311763217633176341763517636176371763817639176401764117642176431764417645176461764717648176491765017651176521765317654176551765617657176581765917660176611766217663176641766517666176671766817669176701767117672176731767417675176761767717678176791768017681176821768317684176851768617687176881768917690176911769217693176941769517696176971769817699177001770117702177031770417705177061770717708177091771017711177121771317714177151771617717177181771917720177211772217723177241772517726177271772817729177301773117732177331773417735177361773717738177391774017741177421774317744177451774617747177481774917750177511775217753177541775517756177571775817759177601776117762177631776417765177661776717768177691777017771177721777317774177751777617777177781777917780177811778217783177841778517786177871778817789177901779117792177931779417795177961779717798177991780017801178021780317804178051780617807178081780917810178111781217813178141781517816178171781817819178201782117822178231782417825178261782717828178291783017831178321783317834178351783617837178381783917840178411784217843178441784517846178471784817849178501785117852178531785417855178561785717858178591786017861178621786317864178651786617867178681786917870178711787217873178741787517876178771787817879178801788117882178831788417885178861788717888178891789017891178921789317894178951789617897178981789917900179011790217903179041790517906179071790817909179101791117912179131791417915179161791717918179191792017921179221792317924179251792617927179281792917930179311793217933179341793517936179371793817939179401794117942179431794417945179461794717948179491795017951179521795317954179551795617957179581795917960179611796217963179641796517966179671796817969179701797117972179731797417975179761797717978179791798017981179821798317984179851798617987179881798917990179911799217993179941799517996179971799817999180001800118002180031800418005180061800718008180091801018011180121801318014180151801618017180181801918020180211802218023180241802518026180271802818029180301803118032180331803418035180361803718038180391804018041180421804318044180451804618047180481804918050180511805218053180541805518056180571805818059180601806118062180631806418065180661806718068180691807018071180721807318074180751807618077180781807918080180811808218083180841808518086180871808818089180901809118092180931809418095180961809718098180991810018101181021810318104181051810618107
  1. /*##############################################################################
  2. Copyright (C) 2011 HPCC Systems.
  3. All rights reserved. This program is free software: you can redistribute it and/or modify
  4. it under the terms of the GNU Affero General Public License as
  5. published by the Free Software Foundation, either version 3 of the
  6. License, or (at your option) any later version.
  7. This program is distributed in the hope that it will be useful,
  8. but WITHOUT ANY WARRANTY; without even the implied warranty of
  9. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  10. GNU Affero General Public License for more details.
  11. You should have received a copy of the GNU Affero General Public License
  12. along with this program. If not, see <http://www.gnu.org/licenses/>.
  13. ############################################################################## */
  14. #include "jliball.hpp"
  15. #include "hql.hpp"
  16. #include "platform.h"
  17. #include "jlib.hpp"
  18. #include "jmisc.hpp"
  19. #include "jstream.ipp"
  20. #include "jdebug.hpp"
  21. #include "build-config.h"
  22. #include "hql.hpp"
  23. #include "hqlthql.hpp"
  24. #include "hqlmeta.hpp"
  25. #include "hqlutil.hpp"
  26. #include "hqlpmap.hpp"
  27. #include "hqlattr.hpp"
  28. #include "hqlerrors.hpp"
  29. #include "hqlvalid.hpp"
  30. #include "hqlerror.hpp"
  31. #include "hqlhtcpp.ipp"
  32. #include "hqlttcpp.ipp"
  33. #include "hqlwcpp.hpp"
  34. #include "hqlcpputil.hpp"
  35. #include "hqltcppc.ipp"
  36. #include "hqlopt.hpp"
  37. #include "hqlfold.hpp"
  38. #include "hqlcerrors.hpp"
  39. #include "hqlcatom.hpp"
  40. #include "hqllib.ipp"
  41. #include "hqlresource.hpp"
  42. #include "hqlregex.ipp"
  43. #include "hqlsource.ipp"
  44. #include "hqlcse.ipp"
  45. #include "hqlgraph.ipp"
  46. #include "hqlscope.hpp"
  47. #include "hqlccommon.hpp"
  48. #include "deffield.hpp"
  49. //The following are include to ensure they call compile...
  50. #include "eclhelper.hpp"
  51. #include "eclrtl_imp.hpp"
  52. #include "rtlfield_imp.hpp"
  53. #include "rtlds_imp.hpp"
  54. #include "eclhelper_base.hpp"
  55. #define MAX_ROWS_OUTPUT_TO_SDS 1000
  56. #define PERSIST_VERSION 1 // Increment when implementation is incompatible.
  57. #define MAX_SAFE_RECORD_SIZE 10000000
  58. #define DEFAULT_EXPIRY_PERIOD 7
  59. #define MAX_GRAPH_ECL_LENGTH 1000
  60. #define MAX_ROW_VALUE_TEXT_LEN 10
  61. //#define TRACE_META_TO_GRAPH
  62. //#define FLATTEN_DATASETS
  63. //#define TraceTableFields
  64. //#define TRACE_ASSIGN_MATCH
  65. //#define TRACE_DUMPTREE
  66. //#define _GATHER_USAGE_STATS
  67. //#define _SR6_
  68. #define MAX_CSV_RECORD_SIZE 4096
  69. #define MIN_HEADER_SIZE 225 // Stolen from Hozed sources
  70. #define ECLRTL_LIB "eclrtl"
  71. //===========================================================================
  72. #ifdef _GATHER_USAGE_STATS
  73. unsigned activityCounts[TAKlast][TAKlast];
  74. #endif
  75. MODULE_INIT(INIT_PRIORITY_STANDARD)
  76. {
  77. return true;
  78. }
  79. MODULE_EXIT()
  80. {
  81. dumpActivityCounts();
  82. }
  83. //===========================================================================
  84. static const char * TF[2] = {"false","true"};
  85. const char * boolToText(bool value) { return TF[value]; } // is this strictly legal?
  86. //---------------------------------------------------------------------------
  87. /*
  88. When should an activity be executed?
  89. 1. If it is used unconditionally.
  90. and
  91. 2. if
  92. a) It isn't a result of some kind.
  93. or
  94. b) It is an external result
  95. or
  96. b) It is an internal result that is used by something outside this graph.
  97. Any subgraph that contains unconditional activities should be marked with the <attr name="RootGraph" value="1"/>
  98. And activity that should not be executed unconditionally should have _internal set.
  99. */
  100. inline bool isInternalSeq(IHqlExpression * seq)
  101. {
  102. return !seq || matchesConstantValue(seq, ResultSequenceInternal);
  103. }
  104. static void markSubGraphAsRoot(IPropertyTree * tree)
  105. {
  106. if (!tree->hasProp("att[@name=\"rootGraph\"]"))
  107. addGraphAttributeBool(tree, "rootGraph", true);
  108. }
  109. //---------------------------------------------------------------------------
  110. class InternalResultTracker : public CInterface
  111. {
  112. public:
  113. InternalResultTracker(IHqlExpression * _name, IPropertyTree * _subGraphTree, unsigned _graphSeq, ActivityInstance * _definingActivity)
  114. : name(_name), subGraphTree(_subGraphTree), graphSeq(_graphSeq), definingActivity(_definingActivity)
  115. {
  116. }
  117. bool noteUse(IHqlExpression * searchName, unsigned curGraphSeq);
  118. public:
  119. LinkedHqlExpr name;
  120. Linked<IPropertyTree> subGraphTree;
  121. unsigned graphSeq;
  122. Linked<ActivityInstance> definingActivity;
  123. };
  124. bool InternalResultTracker::noteUse(IHqlExpression * searchName, unsigned curGraphSeq)
  125. {
  126. if (searchName == name)
  127. {
  128. if ((graphSeq != curGraphSeq) && subGraphTree)
  129. {
  130. markSubGraphAsRoot(subGraphTree);
  131. definingActivity->setInternalSink(false);
  132. subGraphTree.clear();
  133. }
  134. return true;
  135. }
  136. return false;
  137. }
  138. //---------------------------------------------------------------------------
  139. IHqlExpression * getMetaUniqueKey(IHqlExpression * dataset)
  140. {
  141. IHqlExpression * record = dataset->queryRecord();
  142. if (record) record = record->queryBody();
  143. LinkedHqlExpr search = record;
  144. ITypeInfo * type = dataset->queryType();
  145. if (type && type->queryGroupInfo() != NULL)
  146. search.setown(createAttribute(groupedAtom, search.getClear()));
  147. if (!search)
  148. search.setown(createValue(no_null));
  149. return search.getClear();
  150. }
  151. IHqlExpression * getNullStringPointer(bool translated)
  152. {
  153. IHqlExpression * null = createValue(no_nullptr, LINK(constUnknownVarStringType));
  154. if (translated)
  155. return createTranslatedOwned(null);
  156. return null;
  157. }
  158. //---------------------------------------------------------------------------
  159. bool canIterateTableInline(IHqlExpression * expr)
  160. {
  161. switch (expr->getOperator())
  162. {
  163. case no_filter:
  164. return canIterateTableInline(expr->queryChild(0));
  165. case no_field:
  166. case no_select:
  167. return true;
  168. case no_newaggregate:
  169. {
  170. IHqlExpression * child = expr->queryChild(0);
  171. if (!child->queryType()->queryGroupInfo())
  172. return canIterateTableInline(child);
  173. return false;
  174. }
  175. default:
  176. return false;
  177. }
  178. }
  179. static IHqlExpression * createResultName(IHqlExpression * name)
  180. {
  181. if (!name)
  182. return createQuoted("0", makeReferenceModifier(makeVarStringType(UNKNOWN_LENGTH)));
  183. switch (name->getOperator())
  184. {
  185. case no_constant:
  186. return LINK(name);
  187. default:
  188. return LINK(name);
  189. UNIMPLEMENTED;
  190. break;
  191. }
  192. return NULL;
  193. }
  194. //---------------------------------------------------------------------------
  195. void extractAtmostArgs(IHqlExpression * atmost, OwnedHqlExpr & atmostCond, OwnedHqlExpr & atmostLimit)
  196. {
  197. atmostLimit.set(queryZero());
  198. if (atmost)
  199. {
  200. IHqlExpression * arg0 = atmost->queryChild(0);
  201. if (arg0->isBoolean())
  202. {
  203. atmostCond.set(arg0);
  204. atmostLimit.set(atmost->queryChild(1));
  205. }
  206. else
  207. atmostLimit.set(arg0);
  208. }
  209. }
  210. static bool matchesAtmostCondition(IHqlExpression * cond, HqlExprArray & atConds, unsigned & numMatched)
  211. {
  212. if (atConds.find(*cond) != NotFound)
  213. {
  214. numMatched++;
  215. return true;
  216. }
  217. if (cond->getOperator() != no_assertkeyed)
  218. return false;
  219. unsigned savedMatched = numMatched;
  220. HqlExprArray conds;
  221. cond->queryChild(0)->unwindList(conds, no_and);
  222. ForEachItemIn(i, conds)
  223. {
  224. if (!matchesAtmostCondition(&conds.item(i), atConds, numMatched))
  225. {
  226. numMatched = savedMatched;
  227. return false;
  228. }
  229. }
  230. return true;
  231. }
  232. void HqlCppTranslator::splitFuzzyCondition(IHqlExpression * condition, IHqlExpression * atmostCond, OwnedHqlExpr & fuzzy, OwnedHqlExpr & hard)
  233. {
  234. if (atmostCond)
  235. {
  236. //If join condition has evaluated to a constant then allow any atmost condition.
  237. if (!condition->isConstant())
  238. {
  239. HqlExprArray conds, atConds;
  240. condition->unwindList(conds, no_and);
  241. atmostCond->unwindList(atConds, no_and);
  242. unsigned numAtmostMatched = 0;
  243. ForEachItemIn(i, conds)
  244. {
  245. IHqlExpression & cur = conds.item(i);
  246. if (matchesAtmostCondition(&cur, atConds, numAtmostMatched))
  247. extendConditionOwn(hard, no_and, LINK(&cur));
  248. else
  249. extendConditionOwn(fuzzy, no_and, LINK(&cur));
  250. }
  251. if (atConds.ordinality() != numAtmostMatched)
  252. {
  253. StringBuffer s;
  254. getExprECL(atmostCond, s);
  255. throwError1(HQLERR_AtmostFailMatchCondition, s.str());
  256. }
  257. }
  258. }
  259. else
  260. hard.set(condition);
  261. }
  262. //---------------------------------------------------------------------------
  263. ColumnToOffsetMap * RecordOffsetMap::queryMapping(IHqlExpression * record, unsigned maxRecordSize)
  264. {
  265. ColumnToOffsetMap * match = find(record);
  266. if (!match)
  267. {
  268. match = new ColumnToOffsetMap(record, 1, maxRecordSize, false);
  269. match->init(*this);
  270. addOwn(*match);
  271. if (maxRecordSize == 0)
  272. match->checkValidMaxSize();
  273. }
  274. return match;
  275. }
  276. //---------------------------------------------------------------------------
  277. MemberFunction::MemberFunction(HqlCppTranslator & _translator, BuildCtx & classctx, const char * text, unsigned _flags) : translator(_translator), ctx(classctx)
  278. {
  279. ctx.addQuotedCompound(text);
  280. translator.pushMemberFunction(*this);
  281. flags = _flags;
  282. }
  283. MemberFunction::MemberFunction(HqlCppTranslator & _translator, BuildCtx & classctx, StringBuffer & text, unsigned _flags) : translator(_translator), ctx(classctx)
  284. {
  285. ctx.addQuotedCompound(text);
  286. translator.pushMemberFunction(*this);
  287. flags = _flags;
  288. }
  289. MemberFunction::~MemberFunction()
  290. {
  291. translator.popMemberFunction();
  292. }
  293. //---------------------------------------------------------------------------
  294. class ChildSpotterInfo : public HoistingTransformInfo
  295. {
  296. public:
  297. ChildSpotterInfo(IHqlExpression * _original) : HoistingTransformInfo(_original) { spareByte2 = false; }
  298. inline bool getHoist() { return spareByte2 != 0; }
  299. inline void setHoist() { spareByte2 = true; }
  300. private:
  301. using HoistingTransformInfo::spareByte2; //prevent derived classes from also using this spare byte
  302. };
  303. static HqlTransformerInfo childDatasetSpotterInfo("ChildDatasetSpotter");
  304. class ChildDatasetSpotter : public HoistingHqlTransformer
  305. {
  306. public:
  307. ChildDatasetSpotter(HqlCppTranslator & _translator, BuildCtx & _ctx)
  308. : HoistingHqlTransformer(childDatasetSpotterInfo, HTFnoteallconditionals), translator(_translator), ctx(_ctx)
  309. {
  310. candidate = false;
  311. }
  312. virtual ANewTransformInfo * createTransformInfo(IHqlExpression * expr)
  313. {
  314. return CREATE_NEWTRANSFORMINFO(ChildSpotterInfo, expr);
  315. }
  316. inline ChildSpotterInfo * queryExtra(IHqlExpression * expr) { return static_cast<ChildSpotterInfo *>(queryTransformExtra(expr)); }
  317. virtual void analyseExpr(IHqlExpression * expr)
  318. {
  319. if (pass == 0)
  320. {
  321. if (!analyseThis(expr))
  322. return;
  323. switch (expr->getOperator())
  324. {
  325. case no_if:
  326. case no_and:
  327. case no_which:
  328. case no_rejected:
  329. case no_or:
  330. case no_map:
  331. {
  332. analyseExpr(expr->queryChild(0));
  333. conditionDepth++;
  334. ForEachChildFrom(i, expr, 1)
  335. analyseExpr(expr->queryChild(i));
  336. conditionDepth--;
  337. break;
  338. }
  339. case no_choose:
  340. case no_case:
  341. {
  342. analyseExpr(expr->queryChild(0));
  343. analyseExpr(expr->queryChild(1));
  344. conditionDepth++;
  345. ForEachChildFrom(i, expr, 2)
  346. analyseExpr(expr->queryChild(i));
  347. conditionDepth--;
  348. break;
  349. }
  350. default:
  351. doAnalyseExpr(expr);
  352. break;
  353. }
  354. }
  355. else
  356. {
  357. if (!alreadyVisited(expr))
  358. markHoistPoints(expr);
  359. }
  360. }
  361. //MORE: This is a bit of a hack, and should be improved (share code with resource child hoist?)
  362. inline bool walkFurtherDownTree(IHqlExpression * expr)
  363. {
  364. //There are operators which can occur down the tree which may contain datasets
  365. //This should match the analyse code above
  366. switch (expr->getOperator())
  367. {
  368. case no_createrow:
  369. case no_inlinetable:
  370. //The expressions in the transform may contain datasets
  371. case no_addfiles:
  372. case no_datasetfromrow:
  373. case no_alias_scope:
  374. //child datasets may have something worth creating a graph for
  375. case no_if:
  376. //The condition may be worth hoisting (and some of the inputs)
  377. return true;
  378. }
  379. return false;
  380. }
  381. void markHoistPoints(IHqlExpression * expr)
  382. {
  383. node_operator op = expr->getOperator();
  384. switch (op)
  385. {
  386. case no_transform:
  387. case no_assign:
  388. case no_assignall:
  389. break;
  390. default:
  391. if (!containsNonActiveDataset(expr))
  392. return;
  393. break;
  394. }
  395. if (expr->isDataset() || (expr->isDatarow() && (op != no_select)))
  396. {
  397. if (isUsedUnconditionallyEnough(expr) && !translator.canAssignInline(&ctx, expr))
  398. {
  399. candidate = true;
  400. queryExtra(expr)->setHoist();
  401. return;
  402. }
  403. if (walkFurtherDownTree(expr))
  404. doAnalyseExpr(expr);
  405. return;
  406. }
  407. switch (op)
  408. {
  409. case no_countfile:
  410. case no_countindex:
  411. case no_createset:
  412. //not really good enough - need to prevent anything within these from being hoisted.
  413. return;
  414. case no_select:
  415. case no_if:
  416. case no_choose:
  417. case no_case:
  418. case no_and:
  419. case no_which:
  420. case no_rejected:
  421. case no_or:
  422. case no_range:
  423. case no_rangeto:
  424. case no_rangefrom:
  425. case no_rangecommon:
  426. case no_list:
  427. case no_selectnth:
  428. case no_mul:
  429. case no_div:
  430. case no_modulus:
  431. case no_negate:
  432. case no_add:
  433. case no_sub:
  434. case no_exp:
  435. case no_power:
  436. case no_round:
  437. case no_roundup:
  438. case no_ln:
  439. case no_log10:
  440. case no_sin:
  441. case no_cos:
  442. case no_tan:
  443. case no_asin:
  444. case no_acos:
  445. case no_atan:
  446. case no_atan2:
  447. case no_sinh:
  448. case no_cosh:
  449. case no_tanh:
  450. case no_sqrt:
  451. case no_truncate:
  452. case no_cast:
  453. case no_implicitcast:
  454. case no_abs:
  455. case no_charlen:
  456. case no_sizeof:
  457. case no_offsetof:
  458. case no_band:
  459. case no_bor:
  460. case no_bxor:
  461. case no_bnot:
  462. case no_order: //?? also a comparison
  463. case no_rank:
  464. case no_ranked:
  465. case no_hash:
  466. case no_typetransfer:
  467. case no_lshift:
  468. case no_rshift:
  469. case no_crc:
  470. case no_random:
  471. case no_counter:
  472. case no_address:
  473. case no_hash32:
  474. case no_hash64:
  475. case no_wuid:
  476. case no_existslist:
  477. case no_countlist:
  478. case no_maxlist:
  479. case no_minlist:
  480. case no_sumlist:
  481. case no_unicodeorder:
  482. case no_assertkeyed:
  483. case no_hashmd5:
  484. case no_concat:
  485. case no_substring:
  486. case no_asstring:
  487. case no_intformat:
  488. case no_realformat:
  489. case no_trim:
  490. case no_fromunicode:
  491. case no_tounicode:
  492. case no_keyunicode:
  493. case no_rowdiff:
  494. case no_regex_find:
  495. case no_regex_replace:
  496. case no_eq:
  497. case no_ne:
  498. case no_lt:
  499. case no_le:
  500. case no_gt:
  501. case no_ge:
  502. case no_not:
  503. case no_notnot:
  504. case no_xor:
  505. case no_notin:
  506. case no_in:
  507. case no_notbetween:
  508. case no_between:
  509. case no_is_valid:
  510. case no_alias:
  511. case no_transform:
  512. case no_assign:
  513. case no_alias_scope:
  514. case no_getenv:
  515. case no_assignall:
  516. doAnalyseExpr(expr);
  517. break;
  518. default:
  519. doAnalyseExpr(expr);
  520. break;
  521. }
  522. }
  523. inline bool isUsedUnconditionallyEnough(IHqlExpression * expr)
  524. {
  525. IHqlExpression * search = expr;
  526. loop
  527. {
  528. if (isUsedUnconditionally(search))
  529. return true;
  530. switch (search->getOperator())
  531. {
  532. case no_selectnth:
  533. case no_newaggregate:
  534. case no_filter:
  535. break;
  536. case no_select:
  537. if (isNewSelector(search))
  538. break;
  539. return false;
  540. default:
  541. return false;
  542. }
  543. search = search->queryChild(0);
  544. }
  545. }
  546. IHqlExpression * createTransformed(IHqlExpression * expr)
  547. {
  548. OwnedHqlExpr transformed = HoistingHqlTransformer::createTransformed(expr);
  549. updateOrphanedSelectors(transformed, expr);
  550. if ((expr->isDataset() || expr->isDatarow()) && (expr->getOperator() != no_select))
  551. {
  552. if (queryExtra(expr)->getHoist() && !translator.canAssignInline(&ctx, transformed))
  553. {
  554. if (!builder)
  555. builder.setown(new ChildGraphBuilder(translator));
  556. return builder->addDataset(expr);
  557. }
  558. }
  559. return transformed.getClear();
  560. }
  561. virtual IHqlExpression * transformIndependent(IHqlExpression * expr) { return LINK(expr); }
  562. bool hasCandidate() const { return candidate; }
  563. ChildGraphBuilder * getBuilder() { return builder.getClear(); };
  564. protected:
  565. HqlCppTranslator & translator;
  566. Owned<ChildGraphBuilder> builder;
  567. BuildCtx & ctx;
  568. bool candidate;
  569. };
  570. class StatementCollection : public HqlExprArray
  571. {
  572. public:
  573. //Combine multiple conditional assigns, where the guard condition is the same.
  574. //Combine either IF() or CHOOSE()
  575. void combineConditions()
  576. {
  577. unsigned max = ordinality();
  578. for (unsigned first=0; first+1 < max; first++)
  579. {
  580. IHqlExpression & cur = item(first);
  581. if (cur.getOperator() == no_assign)
  582. {
  583. IHqlExpression * rhs = cur.queryChild(1);
  584. if (isCast(rhs))
  585. rhs = rhs->queryChild(0);
  586. node_operator firstOp = rhs->getOperator();
  587. unsigned numFirstChildren = rhs->numChildren();
  588. //Don't combine choose() operators with large numbers of constant values since an array lookup is probably more efficient.
  589. if ((firstOp == no_if) ||
  590. ((firstOp == no_choose) && (numFirstChildren <= 3 || !allBranchesAreConstant(rhs))))
  591. {
  592. IHqlExpression * cond = rhs->queryChild(0);
  593. unsigned next = first+1;
  594. while (next != max)
  595. {
  596. IHqlExpression & nextAssign = item(next);
  597. if (nextAssign.getOperator() != no_assign)
  598. break;
  599. IHqlExpression * nextRhs = nextAssign.queryChild(1);
  600. if (isCast(nextRhs))
  601. nextRhs = nextRhs->queryChild(0);
  602. if (nextRhs->getOperator() != firstOp)
  603. break;
  604. if (nextRhs->queryChild(0) != cond)
  605. break;
  606. if (nextRhs->numChildren() != numFirstChildren)
  607. break;
  608. next++;
  609. }
  610. if (next != first+1)
  611. {
  612. OwnedHqlExpr combined = combineConditionRange(first, next);
  613. replace(*combined.getClear(), first);
  614. unsigned num = (next - first - 1);
  615. removen(first+1, num);
  616. max -= num;
  617. }
  618. }
  619. //MORE: Combine no_case and no_map - but need to be careful,
  620. //because they don't currently have an implementation for actions, so either need to implement, or convert
  621. //to ifs, but also need to be careful we don't make the implementation worse!
  622. }
  623. }
  624. }
  625. void replaceAssignment(IHqlExpression & search, IHqlExpression & newAssign)
  626. {
  627. unsigned match = find(search);
  628. replace(OLINK(newAssign), match);
  629. }
  630. bool onlyOccursOnce(IHqlExpression * expr)
  631. {
  632. return (getNumOccurences(*this, expr, 2) == 1);
  633. }
  634. protected:
  635. bool allBranchesAreConstant(IHqlExpression * expr)
  636. {
  637. ForEachChildFrom(i, expr, 1)
  638. {
  639. IHqlExpression * cur = expr->queryChild(i);
  640. if (!cur->queryValue())
  641. return false;
  642. }
  643. return true;
  644. }
  645. IHqlExpression * extractBranches(unsigned from, unsigned to, unsigned child)
  646. {
  647. StatementCollection assigns;
  648. for (unsigned i=from; i < to; i++)
  649. {
  650. IHqlExpression & cur = item(i);
  651. IHqlExpression * lhs = cur.queryChild(0);
  652. IHqlExpression * rhs = cur.queryChild(1);
  653. OwnedHqlExpr newRhs;
  654. if (isCast(rhs))
  655. {
  656. IHqlExpression * branch = rhs->queryChild(0)->queryChild(child);
  657. newRhs.setown(ensureExprType(branch, rhs->queryType()));
  658. }
  659. else
  660. newRhs.set(rhs->queryChild(child));
  661. assigns.append(*createAssign(LINK(lhs), LINK(newRhs)));
  662. }
  663. assigns.combineConditions();
  664. return createActionList(assigns);
  665. }
  666. IHqlExpression * combineConditionRange(unsigned from, unsigned to)
  667. {
  668. IHqlExpression * firstRhs = item(from).queryChild(1);
  669. IHqlExpression * conditionExpr = isCast(firstRhs) ? firstRhs->queryChild(0) : firstRhs;
  670. HqlExprArray args;
  671. args.append(*LINK(conditionExpr->queryChild(0)));
  672. ForEachChildFrom(i, conditionExpr, 1)
  673. args.append(*extractBranches(from, to, i));
  674. return createValue(conditionExpr->getOperator(), makeVoidType(), args);
  675. }
  676. };
  677. class DelayedStatementExecutor
  678. {
  679. public:
  680. DelayedStatementExecutor(HqlCppTranslator & _translator, BuildCtx & _ctx)
  681. : translator(_translator), buildctx(_ctx)
  682. {
  683. processed = false;
  684. }
  685. void processAssign(BuildCtx & ctx, IHqlExpression * stmt)
  686. {
  687. pending.append(*LINK(stmt));
  688. }
  689. void processAlias(BuildCtx & ctx, IHqlExpression * stmt)
  690. {
  691. pending.append(*LINK(stmt));
  692. }
  693. void clear()
  694. {
  695. builder.clear();
  696. pending.kill();
  697. processed = false;
  698. }
  699. bool hasGraphPending()
  700. {
  701. spotChildDatasets();
  702. return builder != NULL;
  703. }
  704. void flush(BuildCtx & ctx)
  705. {
  706. spotChildDatasets();
  707. if (builder)
  708. {
  709. builder->generateGraph(ctx);
  710. builder.clear();
  711. }
  712. combineConditions();
  713. optimizeAssigns();
  714. ForEachItemIn(i, pending)
  715. translator.buildStmt(ctx, &pending.item(i));
  716. pending.kill();
  717. }
  718. void generatePrefetch(BuildCtx & ctx, OwnedHqlExpr * retGraphExpr, OwnedHqlExpr * retResultsExpr)
  719. {
  720. builder->generatePrefetchGraph(ctx, retGraphExpr, retResultsExpr);
  721. builder.clear();
  722. }
  723. protected:
  724. //Combine multiple conditional assigns, where the guard condition is the same.
  725. void combineConditions()
  726. {
  727. pending.combineConditions();
  728. }
  729. void spotChildDatasets()
  730. {
  731. if (!processed && translator.queryCommonUpChildGraphs())
  732. {
  733. ChildDatasetSpotter spotter(translator, buildctx);
  734. for (unsigned pass=0; pass < 2; pass++)
  735. {
  736. ForEachItemIn(i, pending)
  737. {
  738. IHqlExpression & cur = pending.item(i);
  739. IHqlExpression * value = &cur;
  740. switch (value->getOperator())
  741. {
  742. case no_assign:
  743. value = value->queryChild(1);
  744. break;
  745. case no_alias:
  746. case no_skip:
  747. value = value->queryChild(0);
  748. break;
  749. }
  750. if (value)
  751. spotter.analyse(value, pass);
  752. }
  753. }
  754. if (spotter.hasCandidate())
  755. {
  756. HqlExprArray replacement;
  757. ForEachItemIn(i, pending)
  758. replacement.append(*spotter.transformRoot(&pending.item(i)));
  759. replaceArray(pending, replacement);
  760. builder.setown(spotter.getBuilder());
  761. }
  762. processed = true;
  763. }
  764. }
  765. virtual void optimizeAssigns() {}
  766. protected:
  767. HqlCppTranslator & translator;
  768. BuildCtx buildctx;
  769. StatementCollection pending;
  770. Owned<ChildGraphBuilder> builder;
  771. bool processed;
  772. };
  773. //---------------------------------------------------------------------------
  774. static IHqlExpression * getExtractMatchingAssign(HqlExprArray & assigns, IHqlExpression * search, unsigned & expectedIndex, IHqlExpression * selfSelector)
  775. {
  776. if (assigns.isItem(expectedIndex))
  777. {
  778. IHqlExpression & candidate = assigns.item(expectedIndex);
  779. IHqlExpression * lhs = candidate.queryChild(0);
  780. IHqlExpression * candidateField = lhs->queryChild(1);
  781. if (candidateField == search)
  782. {
  783. OwnedHqlExpr ret;
  784. if (lhs->queryChild(0) == selfSelector)
  785. ret.set(&candidate);
  786. else
  787. {
  788. IHqlExpression * rhs = candidate.queryChild(1);
  789. ret.setown(createAssign(createSelectExpr(LINK(selfSelector), LINK(candidateField)), LINK(rhs)));
  790. }
  791. expectedIndex++;
  792. return ret.getClear();
  793. }
  794. }
  795. ForEachItemIn(idx, assigns)
  796. {
  797. IHqlExpression & assign = assigns.item(idx);
  798. #ifdef TRACE_ASSIGN_MATCH
  799. PrintLog("Next comparison:");
  800. x.clear().append("target(").append((unsigned)assign.queryChild(0)->queryChild(0)).append(":");
  801. x.appendf("%p", assign.queryChild(0)->queryChild(1)).append(") ");
  802. assign.queryChild(0)->toString(x);
  803. PrintLog(x.str());
  804. x.clear().append("search(").appendf("%p", search).append(") ");
  805. search->toString(x);
  806. PrintLog(x.str());
  807. #endif
  808. IHqlExpression * lhs = assign.queryChild(0);
  809. IHqlExpression * candidateField = lhs->queryChild(1);
  810. if (candidateField == search)
  811. {
  812. OwnedHqlExpr ret;
  813. if (lhs->queryChild(0) == selfSelector)
  814. ret.set(&assign);
  815. else
  816. {
  817. IHqlExpression * rhs = assign.queryChild(1);
  818. ret.setown(createAssign(createSelectExpr(LINK(selfSelector), LINK(candidateField)), LINK(rhs)));
  819. }
  820. expectedIndex = idx+1;
  821. return ret.getClear();
  822. }
  823. }
  824. return NULL;
  825. }
  826. class TransformBuilder : public DelayedStatementExecutor
  827. {
  828. public:
  829. TransformBuilder(HqlCppTranslator & _translator, BuildCtx & _ctx, IHqlExpression * _record, BoundRow * _self, HqlExprArray & _assigns) :
  830. DelayedStatementExecutor(_translator, _ctx), assigns(_assigns), record(_record), self(_self)
  831. {
  832. expectedIndex = 0;
  833. if (translator.recordContainsIfBlock(record))
  834. mapper.setown(new NestedHqlMapTransformer);
  835. }
  836. TransformBuilder(const TransformBuilder & other, BuildCtx & _ctx) :
  837. DelayedStatementExecutor(other.translator, _ctx), mapper(other.mapper), assigns(other.assigns)
  838. {
  839. expectedIndex = 0;
  840. }
  841. void doTransform(BuildCtx & ctx, IHqlExpression * transform, BoundRow * self);
  842. void buildTransformChildren(BuildCtx & ctx, IHqlExpression * record, IHqlExpression * parentSelector);
  843. protected:
  844. virtual void checkAssigned() { }
  845. virtual void onIfBlock(IHqlExpression * expr) { }
  846. virtual void onMissingAssignment(IHqlExpression * expr)
  847. {
  848. StringBuffer s;
  849. expr->toString(s);
  850. throwError2(HQLERR_MissingTransformAssignXX, s.str(), expr);
  851. }
  852. void pushCondition(IHqlExpression * cond)
  853. {
  854. mapper->beginNestedScope();
  855. }
  856. void popCondition()
  857. {
  858. mapper->endNestedScope();
  859. }
  860. void buildTransform(BuildCtx & ctx, IHqlExpression * expr, IHqlExpression * parentSelector);
  861. void doBuildTransformChildren(BuildCtx & ctx, IHqlExpression * record, IHqlExpression * parentSelector);
  862. public:
  863. Linked<NestedHqlMapTransformer> mapper;
  864. HqlExprArray & assigns;
  865. LinkedHqlExpr record;
  866. BoundRow * self;
  867. unsigned expectedIndex;
  868. };
  869. void TransformBuilder::buildTransform(BuildCtx & ctx, IHqlExpression * expr, IHqlExpression * parentSelector)
  870. {
  871. switch (expr->getOperator())
  872. {
  873. case no_ifblock:
  874. {
  875. onIfBlock(expr);
  876. flush(ctx);
  877. assertex(mapper);
  878. OwnedHqlExpr test = replaceSelector(expr->queryChild(0), querySelfReference(), parentSelector);
  879. OwnedHqlExpr foldedTest = mapper->transformRoot(test);
  880. foldedTest.setown(foldHqlExpression(foldedTest)); // can only contain references to self, so don't need to worry about other datasets in scope being messed up.
  881. OwnedHqlExpr sizeOfIfBlock = createValue(no_sizeof, makeIntType(4,false), createSelectExpr(LINK(parentSelector), LINK(expr)));
  882. IValue * mappedValue = foldedTest->queryValue();
  883. BuildCtx subctx(ctx);
  884. bool include = true;
  885. if (mappedValue)
  886. {
  887. //Associate the ifblock condition to avoid it being evaluated later when calculating the field offsets
  888. ctx.associateExpr(test, foldedTest);
  889. if (!mappedValue->getBoolValue())
  890. include = false;
  891. }
  892. else
  893. pushCondition(foldedTest);
  894. if (include)
  895. {
  896. translator.buildFilter(subctx, foldedTest);
  897. TransformBuilder childBuilder(*this, subctx);
  898. childBuilder.buildTransformChildren(subctx, expr->queryChild(1), parentSelector);
  899. childBuilder.flush(subctx);
  900. //This calculates the size of the previous block. It means that subsequent uses of the
  901. //offsets are cached - even if they are inside another ifblock().
  902. CHqlBoundExpr bound;
  903. translator.buildCachedExpr(ctx, sizeOfIfBlock, bound);
  904. }
  905. else
  906. {
  907. //This calculates the size of the previous block. It means that subsequent uses of the
  908. //offsets are cached - even if they are inside another ifblock().
  909. OwnedHqlExpr zero = getSizetConstant(0);
  910. ctx.associateExpr(sizeOfIfBlock, zero);
  911. }
  912. if (!mappedValue)
  913. popCondition();
  914. }
  915. break;
  916. case no_record:
  917. doBuildTransformChildren(ctx, expr, parentSelector);
  918. break;
  919. case no_field:
  920. {
  921. OwnedHqlExpr match = getExtractMatchingAssign(assigns, expr, expectedIndex, parentSelector);
  922. if (match)
  923. {
  924. processAssign(ctx, match);
  925. if (mapper && (match->getOperator() == no_assign))
  926. {
  927. IHqlExpression * rhs = match->queryChild(1);
  928. if (rhs->queryValue())
  929. {
  930. IHqlExpression * lhs = match->queryChild(0);
  931. OwnedHqlExpr cast = ensureExprType(rhs, lhs->queryType());
  932. mapper->setMapping(lhs, cast);
  933. }
  934. }
  935. return;
  936. }
  937. onMissingAssignment(expr);
  938. }
  939. break;
  940. case no_attr:
  941. case no_attr_expr:
  942. case no_attr_link:
  943. break;
  944. default:
  945. UNIMPLEMENTED;
  946. }
  947. }
  948. void TransformBuilder::doBuildTransformChildren(BuildCtx & ctx, IHqlExpression * record, IHqlExpression * parentSelector)
  949. {
  950. ForEachChild(idx, record)
  951. buildTransform(ctx, record->queryChild(idx), parentSelector);
  952. }
  953. void TransformBuilder::buildTransformChildren(BuildCtx & ctx, IHqlExpression * record, IHqlExpression * parentSelector)
  954. {
  955. assertex(parentSelector);
  956. expectedIndex = 0;
  957. doBuildTransformChildren(ctx, record, parentSelector);
  958. }
  959. void TransformBuilder::doTransform(BuildCtx & ctx, IHqlExpression * transform, BoundRow * self)
  960. {
  961. IHqlExpression * body = transform->queryBody(true);
  962. if (transform != body)
  963. {
  964. switch (transform->getAnnotationKind())
  965. {
  966. case annotate_meta:
  967. translator.queryWarningProcessor().processMetaAnnotation(transform);
  968. break;
  969. case annotate_symbol:
  970. {
  971. WarningProcessor::OnWarningStateSymbolBlock saved(translator.queryWarningProcessor(), transform);
  972. doTransform(ctx, body, self);
  973. return;
  974. }
  975. }
  976. doTransform(ctx, body, self);
  977. return;
  978. }
  979. if (!isKnownTransform(transform))
  980. {
  981. translator.doUserTransform(ctx, transform, self);
  982. return;
  983. }
  984. translator.filterExpandAssignments(ctx, this, assigns, transform);
  985. buildTransformChildren(ctx, self->queryRecord(), self->querySelector());
  986. flush(ctx);
  987. checkAssigned();
  988. }
  989. class UpdateTransformBuilder : public TransformBuilder
  990. {
  991. friend class UnsafeSelectorReplacer;
  992. public:
  993. UpdateTransformBuilder(HqlCppTranslator & _translator, BuildCtx & _ctx, IHqlExpression * record, BoundRow * _self, IHqlExpression * _prevSelector, HqlExprArray & _assigns, bool _canRemoveLeadingAssigns) :
  994. TransformBuilder(_translator, _ctx, record, _self, _assigns), prevSelector(_prevSelector)
  995. {
  996. aliasInsertPos = 0;
  997. needToReassignAll = false;
  998. canRemoveLeadingAssigns = _canRemoveLeadingAssigns;
  999. }
  1000. void ensureAlias(IHqlExpression * expr);
  1001. inline bool isUnsafeSelector(IHqlExpression * expr) const
  1002. {
  1003. return needToReassignAll || unsafeSelectors.contains(*expr);
  1004. }
  1005. protected:
  1006. virtual void checkAssigned() { }
  1007. virtual void onIfBlock(IHqlExpression * expr) { throwUnexpected(); }
  1008. virtual void onMissingAssignment(IHqlExpression * expr) {}
  1009. virtual void optimizeAssigns();
  1010. bool isSpecialAssignment(IHqlExpression * assign, node_operator op, IHqlExpression * previous) const;
  1011. void optimizeAssigns(IHqlExpression * expr, IHqlExpression * parentSelector);
  1012. void optimizeRecordAssigns(IHqlExpression * record, IHqlExpression * parentSelector);
  1013. IHqlExpression * replaceUnsafeSelectors(IHqlExpression * rhs);
  1014. void optimizeSpecialAssignments(IHqlExpression * expr, IHqlExpression * parentSelector);
  1015. void optimizeRecordSpecialAssignments(IHqlExpression * expr, IHqlExpression * parentSelector);
  1016. void protectAgainstLeaks(IHqlExpression * expr);
  1017. void protectRecordAgainstLeaks(IHqlExpression * expr);
  1018. protected:
  1019. LinkedHqlExpr prevSelector;
  1020. HqlExprArray unsafeSelectors;
  1021. HqlExprArray preservedSelectors;
  1022. unsigned aliasInsertPos;
  1023. bool needToReassignAll;
  1024. bool canRemoveLeadingAssigns;
  1025. };
  1026. void UpdateTransformBuilder::ensureAlias(IHqlExpression * expr)
  1027. {
  1028. if (!pending.contains(*expr))
  1029. pending.add(*LINK(expr), aliasInsertPos++);
  1030. }
  1031. bool UpdateTransformBuilder::isSpecialAssignment(IHqlExpression * assign, node_operator op, IHqlExpression * previous) const
  1032. {
  1033. IHqlExpression * rhs = assign->queryChild(1);
  1034. if (rhs->getOperator() != op)
  1035. return false;
  1036. if (rhs->queryChild(0) != previous)
  1037. return false;
  1038. return true;
  1039. }
  1040. void UpdateTransformBuilder::optimizeSpecialAssignments(IHqlExpression * expr, IHqlExpression * parentSelector)
  1041. {
  1042. switch (expr->getOperator())
  1043. {
  1044. case no_record:
  1045. optimizeRecordSpecialAssignments(expr, parentSelector);
  1046. break;
  1047. case no_field:
  1048. {
  1049. //check for SELF.x := RIGHT.x <add-file> f(LEFT)
  1050. node_operator newOp = no_none;
  1051. OwnedHqlExpr previous = createSelectExpr(LINK(prevSelector), LINK(expr));
  1052. OwnedHqlExpr match;
  1053. if (expr->isDataset())
  1054. {
  1055. match.setown(getExtractMatchingAssign(pending, expr, expectedIndex, parentSelector));
  1056. if (match && isSpecialAssignment(match, no_addfiles, previous))
  1057. newOp = no_assign_addfiles;
  1058. }
  1059. if (newOp && pending.onlyOccursOnce(previous))
  1060. {
  1061. IHqlExpression * rhs = match->queryChild(1);
  1062. OwnedHqlExpr newAssign = createValue(newOp, makeVoidType(), LINK(match->queryChild(0)), LINK(rhs->queryChild(1)));
  1063. pending.replaceAssignment(*match, *newAssign);
  1064. preservedSelectors.append(*LINK(previous));
  1065. }
  1066. }
  1067. break;
  1068. }
  1069. }
  1070. void UpdateTransformBuilder::optimizeRecordSpecialAssignments(IHqlExpression * record, IHqlExpression * parentSelector)
  1071. {
  1072. ForEachChild(idx, record)
  1073. optimizeSpecialAssignments(record->queryChild(idx), parentSelector);
  1074. }
  1075. //Ensure all fields from SELF are cloned before they are overwritten.
  1076. //If only fixed width fields have been updated so far then any fields not yet assigned can be used
  1077. //If a variable width field has been updated all fields updated so far are invalid, and all field that
  1078. //follow may have been overwritten => it will need an alias. For simplicity all fields are assumed tainted.
  1079. void UpdateTransformBuilder::optimizeAssigns(IHqlExpression * expr, IHqlExpression * parentSelector)
  1080. {
  1081. switch (expr->getOperator())
  1082. {
  1083. case no_ifblock:
  1084. throwUnexpected();
  1085. case no_record:
  1086. optimizeRecordAssigns(expr, parentSelector);
  1087. break;
  1088. case no_field:
  1089. {
  1090. OwnedHqlExpr match = getExtractMatchingAssign(pending, expr, expectedIndex, parentSelector);
  1091. if (match)
  1092. {
  1093. IHqlExpression * source = match->queryChild(1);
  1094. OwnedHqlExpr previous = createSelectExpr(LINK(prevSelector), LINK(expr));
  1095. bool usesPrevious = (source != previous) && exprReferencesDataset(source, prevSelector);
  1096. bool retainAssign = !canRemoveLeadingAssigns || needToReassignAll || usesPrevious || (match->getOperator() != no_assign);
  1097. if (retainAssign)
  1098. {
  1099. IHqlExpression * target = match->queryChild(0);
  1100. //Variable offset => all subsequent fields need to be reassigned.
  1101. ITypeInfo * type = expr->queryType();
  1102. bool hasVariableSize = (type->getSize() == UNKNOWN_LENGTH);
  1103. //Potential problems with fixed length strings. Otherwise it should be safe, or go via a temporary
  1104. bool safeToAccessSelf = hasVariableSize || !isTypePassedByAddress(type); // not true for some fixed length strings, what else?
  1105. //Any access to this field now needs to go via a temporary
  1106. if (!safeToAccessSelf)
  1107. unsafeSelectors.append(*LINK(previous));
  1108. OwnedHqlExpr safeRhs = replaceUnsafeSelectors(source);
  1109. if (safeToAccessSelf)
  1110. unsafeSelectors.append(*LINK(previous));
  1111. if (hasVariableSize)
  1112. needToReassignAll = true;
  1113. if (safeRhs != source)
  1114. {
  1115. //MORE: Create no_plusequals, no_concat_equals
  1116. OwnedHqlExpr newAssign = createAssign(LINK(target), LINK(safeRhs));
  1117. unsigned pos = pending.find(*match);
  1118. assertex(pos != NotFound);
  1119. pending.replace(*newAssign.getClear(), pos);
  1120. }
  1121. }
  1122. else
  1123. {
  1124. preservedSelectors.append(*LINK(previous));
  1125. pending.zap(*match);
  1126. }
  1127. return;
  1128. }
  1129. }
  1130. break;
  1131. case no_attr:
  1132. case no_attr_expr:
  1133. case no_attr_link:
  1134. break;
  1135. default:
  1136. UNIMPLEMENTED;
  1137. }
  1138. }
  1139. void UpdateTransformBuilder::optimizeRecordAssigns(IHqlExpression * record, IHqlExpression * parentSelector)
  1140. {
  1141. ForEachChild(idx, record)
  1142. optimizeAssigns(record->queryChild(idx), parentSelector);
  1143. }
  1144. void UpdateTransformBuilder::protectAgainstLeaks(IHqlExpression * expr)
  1145. {
  1146. switch (expr->getOperator())
  1147. {
  1148. case no_record:
  1149. protectRecordAgainstLeaks(expr);
  1150. break;
  1151. case no_field:
  1152. {
  1153. if (hasLinkCountedModifier(expr))
  1154. {
  1155. OwnedHqlExpr previous = createSelectExpr(LINK(prevSelector), LINK(expr));
  1156. if (!preservedSelectors.contains(*previous))
  1157. {
  1158. OwnedHqlExpr owned = createAliasOwn(ensureOwned(previous), NULL);
  1159. ensureAlias(owned);
  1160. }
  1161. }
  1162. }
  1163. break;
  1164. }
  1165. }
  1166. void UpdateTransformBuilder::protectRecordAgainstLeaks(IHqlExpression * record)
  1167. {
  1168. ForEachChild(idx, record)
  1169. protectAgainstLeaks(record->queryChild(idx));
  1170. }
  1171. void UpdateTransformBuilder::optimizeAssigns()
  1172. {
  1173. expectedIndex = 0;
  1174. optimizeRecordSpecialAssignments(record, self->querySelector());
  1175. expectedIndex = 0;
  1176. optimizeRecordAssigns(record, self->querySelector());
  1177. protectRecordAgainstLeaks(record);
  1178. }
  1179. static HqlTransformerInfo unsafeSelectorReplacerInfo("UnsafeSelectorReplacer");
  1180. class UnsafeSelectorReplacer : public NewHqlTransformer
  1181. {
  1182. public:
  1183. UnsafeSelectorReplacer(UpdateTransformBuilder & _builder, IHqlExpression * _searchSelector)
  1184. : NewHqlTransformer(unsafeSelectorReplacerInfo), builder(_builder), searchSelector(_searchSelector)
  1185. {
  1186. }
  1187. inline IHqlExpression * getReplacement(IHqlExpression * expr)
  1188. {
  1189. //Yuk if the whole row is referenced the code is not going to be good!
  1190. if (expr == searchSelector)
  1191. return ensureAliased(expr);
  1192. if ((expr->getOperator() == no_select) && (expr->queryChild(0) == searchSelector))
  1193. {
  1194. if (builder.isUnsafeSelector(expr))
  1195. return ensureAliased(expr);
  1196. return LINK(expr);
  1197. }
  1198. return NULL;
  1199. }
  1200. virtual IHqlExpression * createTransformed(IHqlExpression * expr)
  1201. {
  1202. IHqlExpression * body = expr->queryBody(true);
  1203. if (expr == body)
  1204. {
  1205. IHqlExpression * replacement = getReplacement(expr);
  1206. if (replacement)
  1207. return replacement;
  1208. return NewHqlTransformer::createTransformed(expr);
  1209. }
  1210. OwnedHqlExpr transformed = transform(body);
  1211. return expr->cloneAnnotation(transformed);
  1212. }
  1213. virtual IHqlExpression * createTransformedSelector(IHqlExpression * expr)
  1214. {
  1215. IHqlExpression * replacement = getReplacement(expr);
  1216. if (replacement)
  1217. return replacement;
  1218. return NewHqlTransformer::createTransformedSelector(expr);
  1219. }
  1220. IHqlExpression * ensureAliased(IHqlExpression * expr)
  1221. {
  1222. LinkedHqlExpr value = expr;
  1223. if (expr->getOperator() == no_select)
  1224. value.setown(ensureOwned(value));
  1225. OwnedHqlExpr alias = createAlias(value, NULL);
  1226. builder.ensureAlias(alias);
  1227. return alias.getClear();
  1228. }
  1229. protected:
  1230. UpdateTransformBuilder & builder;
  1231. IHqlExpression * searchSelector;
  1232. };
  1233. IHqlExpression * UpdateTransformBuilder::replaceUnsafeSelectors(IHqlExpression * rhs)
  1234. {
  1235. UnsafeSelectorReplacer replacer(*this, prevSelector);
  1236. return replacer.transformRoot(rhs);
  1237. }
  1238. //---------------------------------------------------------------------------
  1239. void HqlCppTranslator::doFilterAssignment(BuildCtx & ctx, TransformBuilder * builder, HqlExprArray & assigns, IHqlExpression * cur)
  1240. {
  1241. node_operator op = cur->getOperator();
  1242. switch (op)
  1243. {
  1244. case no_assignall:
  1245. case no_newtransform:
  1246. case no_transform:
  1247. case no_alias_scope:
  1248. doFilterAssignments(ctx, builder, assigns, cur);
  1249. break;
  1250. case no_assign:
  1251. assigns.append(*LINK(cur));
  1252. break;
  1253. case no_assert:
  1254. case no_skip:
  1255. case no_alias:
  1256. if (builder)
  1257. builder->processAlias(ctx, cur);
  1258. else
  1259. buildStmt(ctx, cur);
  1260. break;
  1261. case no_attr:
  1262. case no_attr_link:
  1263. case no_attr_expr:
  1264. break;
  1265. default:
  1266. UNIMPLEMENTED;
  1267. }
  1268. }
  1269. void HqlCppTranslator::doFilterAssignments(BuildCtx & ctx, TransformBuilder * builder, HqlExprArray & assigns, IHqlExpression * expr)
  1270. {
  1271. if (expr->getOperator() == no_alias_scope)
  1272. {
  1273. ForEachChildFrom(i, expr, 1)
  1274. doFilterAssignment(ctx, builder, assigns, expr->queryChild(i));
  1275. doFilterAssignment(ctx, builder, assigns, expr->queryChild(0));
  1276. }
  1277. else
  1278. {
  1279. ForEachChild(i, expr)
  1280. doFilterAssignment(ctx, builder, assigns, expr->queryChild(i));
  1281. }
  1282. }
  1283. void HqlCppTranslator::filterExpandAssignments(BuildCtx & ctx, TransformBuilder * builder, HqlExprArray & assigns, IHqlExpression * rawExpr)
  1284. {
  1285. LinkedHqlExpr expr = rawExpr;
  1286. if (options.spotCSE)
  1287. expr.setown(spotScalarCSE(expr));
  1288. traceExpression("transform cse", expr);
  1289. // expandAliases(ctx, expr);
  1290. doFilterAssignments(ctx, builder, assigns, expr);
  1291. }
  1292. void HqlCppTranslator::associateCounter(BuildCtx & ctx, IHqlExpression * counterExpr, const char * name)
  1293. {
  1294. if (counterExpr)
  1295. {
  1296. OwnedHqlExpr counterVar = createVariable(name, LINK(counterType));
  1297. ctx.associateExpr(counterExpr, counterVar);
  1298. }
  1299. }
  1300. unsigned HqlCppTranslator::getConsistentUID(IHqlExpression * ptr)
  1301. {
  1302. if (!ptr)
  1303. return 0;
  1304. // Allocate consistent numbers helps to regression check the generated code
  1305. if (recordIndexCache.find(ptr) == NotFound)
  1306. recordIndexCache.append(ptr);
  1307. return recordIndexCache.find(ptr)+1;
  1308. }
  1309. unsigned HqlCppTranslator::cppIndexNextActivity(bool isChildActivity)
  1310. {
  1311. activitiesThisCpp++;
  1312. if (activitiesThisCpp > options.activitiesPerCpp)
  1313. {
  1314. //Allow 25% over the default number of activities per child in order to reduce the number of activities moved into
  1315. //the header file.
  1316. if (!isChildActivity || activitiesThisCpp > (options.activitiesPerCpp * 5 / 4))
  1317. {
  1318. curCppFile++;
  1319. activitiesThisCpp = 1;
  1320. }
  1321. }
  1322. return curCppFile;
  1323. }
  1324. //---------------------------------------------------------------------------
  1325. static IHqlExpression * createResultAttribute(IHqlExpression * seq, IHqlExpression * name)
  1326. {
  1327. //if a named user output then set seq to the name so that workunit reads from the named symbol get commoned up correctly
  1328. if (name && !name->queryType()->isInteger() && seq->queryValue()->getIntValue() >= 0)
  1329. seq = name;
  1330. return createAttribute(resultAtom, LINK(seq), LINK(name));
  1331. }
  1332. static void associateRemoteResult(BuildCtx & ctx, ABoundActivity * table, IHqlExpression * seq, IHqlExpression * name)
  1333. {
  1334. OwnedHqlExpr attr = createResultAttribute(seq, name);
  1335. OwnedHqlExpr unknown = createUnknown(no_attr, NULL, NULL, LINK(table));
  1336. ctx.associateExpr(attr, unknown);
  1337. }
  1338. void HqlCppTranslator::associateRemoteResult(ActivityInstance & instance, IHqlExpression * seq, IHqlExpression * name)
  1339. {
  1340. ::associateRemoteResult(*activeGraphCtx, instance.table, seq, name);
  1341. if (name && targetRoxie())
  1342. {
  1343. OwnedHqlExpr attr = createResultAttribute(seq, name);
  1344. globalFiles.append(* new GlobalFileTracker(attr, instance.graphNode));
  1345. }
  1346. }
  1347. void HqlCppTranslator::queryAddResultDependancy(ABoundActivity & whoAmIActivity, IHqlExpression * seq, IHqlExpression * name)
  1348. {
  1349. if (activeGraphCtx)
  1350. {
  1351. //Because of extend, we need to find all the possible matches, not just the last one.
  1352. AssociationIterator iter(*activeGraphCtx);
  1353. OwnedHqlExpr attr = createResultAttribute(seq, name);
  1354. ForEach(iter)
  1355. {
  1356. HqlExprAssociation & cur = iter.get();
  1357. if (cur.represents == attr)
  1358. {
  1359. ABoundActivity * match = (ABoundActivity *)cur.queryExpr()->queryUnknownExtra();
  1360. IHqlExpression * whoAmI = whoAmIActivity.queryBound();
  1361. OwnedHqlExpr dependency = createAttribute(dependencyAtom, LINK(whoAmI), LINK(match->queryBound()));
  1362. if (!activeGraphCtx->queryMatchExpr(dependency))
  1363. {
  1364. activeGraphCtx->associateExpr(dependency, dependency);
  1365. addDependency(*activeGraphCtx, match, &whoAmIActivity, dependencyAtom);
  1366. }
  1367. }
  1368. }
  1369. if (name && targetRoxie())
  1370. registerGlobalUsage(attr);
  1371. }
  1372. }
  1373. bool HqlCppTranslator::tempRowRequiresFinalize(IHqlExpression * record) const
  1374. {
  1375. if (recordRequiresSerialization(record) || options.finalizeAllRows)
  1376. return true;
  1377. if (options.finalizeAllVariableRows && isVariableSizeRecord(record))
  1378. return true;
  1379. return false;
  1380. }
  1381. BoundRow * HqlCppTranslator::createRowBuilder(BuildCtx & ctx, BoundRow * targetRow)
  1382. {
  1383. IHqlExpression * record = targetRow->queryRecord();
  1384. IHqlExpression * boundTarget = targetRow->queryBound();
  1385. bool targetIsOwnedRow = hasWrapperModifier(boundTarget->queryType());
  1386. StringBuffer builderName, rowName;
  1387. getUniqueId(builderName.append("b"));
  1388. rowName.append(builderName).append(".row()");
  1389. if ((!targetIsOwnedRow && isFixedWidthDataset(record) && !options.alwaysCreateRowBuilder) || !options.supportDynamicRows)
  1390. {
  1391. LinkedHqlExpr targetArg = boundTarget;
  1392. if (targetIsOwnedRow)
  1393. {
  1394. OwnedHqlExpr allocator = createRowAllocator(ctx, record);
  1395. StringBuffer valueText;
  1396. valueText.append("(byte *)");
  1397. generateExprCpp(valueText, allocator).append("->createRow()");
  1398. StringBuffer setText;
  1399. generateExprCpp(setText, boundTarget);
  1400. setText.append(".setown(").append(valueText).append(");");
  1401. ctx.addQuoted(setText);
  1402. targetArg.setown(getPointer(boundTarget));
  1403. }
  1404. BoundRow * self = bindSelf(ctx, targetRow->queryDataset(), targetArg, NULL);
  1405. return LINK(self);
  1406. }
  1407. if (targetIsOwnedRow)
  1408. {
  1409. OwnedHqlExpr allocator = createRowAllocator(ctx, record);
  1410. StringBuffer s;
  1411. s.clear().append("RtlDynamicRowBuilder ").append(builderName).append("(");
  1412. generateExprCpp(s, allocator).append(");");
  1413. ctx.addQuoted(s);
  1414. BoundRow * self = bindSelf(ctx, targetRow->queryDataset(), builderName);
  1415. return LINK(self);
  1416. }
  1417. else
  1418. {
  1419. StringBuffer s;
  1420. s.clear().append("RtlStaticRowBuilder ").append(builderName).append("(");
  1421. generateExprCpp(s, boundTarget);
  1422. s.append(",").append(getMaxRecordSize(record)).append(");");
  1423. ctx.addQuoted(s);
  1424. OwnedHqlExpr builder = createVariable(builderName, makeBoolType());
  1425. BoundRow * self = bindSelf(ctx, targetRow->queryDataset(), builderName);
  1426. return LINK(self);
  1427. }
  1428. }
  1429. BoundRow * HqlCppTranslator::declareLinkedRow(BuildCtx & ctx, IHqlExpression * expr, bool isMember)
  1430. {
  1431. assertex(expr->isDatarow());
  1432. StringBuffer rowName;
  1433. getUniqueId(rowName.append('r'));
  1434. IHqlExpression * record = expr->queryRecord();
  1435. Owned<ITypeInfo> rowType = makeRowType(record->getType());
  1436. rowType.setown(makeAttributeModifier(makeWrapperModifier(rowType.getClear()), getLinkCountedAttr()));
  1437. if (isMember)
  1438. rowType.setown(makeModifier(rowType.getClear(), typemod_member, NULL));
  1439. OwnedHqlExpr boundRow = createVariable(rowName, LINK(rowType));
  1440. //Ugly, but necessary. Conditional temporary rows will be accessed in a lifetime outside of the scope they are
  1441. //evaluated in - so the declaration needs to be in a scope where they will not be freed. For the moment make
  1442. //this the outer most scope (within a function)
  1443. ctx.setNextPriority(BuildCtx::OutermostScopePrio);
  1444. ctx.addDeclare(boundRow);
  1445. return createBoundRow(expr, boundRow);
  1446. }
  1447. BoundRow * HqlCppTranslator::declareStaticRow(BuildCtx & ctx, IHqlExpression * expr)
  1448. {
  1449. assertex(expr->isDatarow());
  1450. IHqlExpression * record = expr->queryRecord();
  1451. unsigned maxRecordSize = getMaxRecordSize(record);
  1452. StringBuffer rowName;
  1453. getUniqueId(rowName.append('r'));
  1454. Owned<ITypeInfo> rowType = makeRowType(record->getType());
  1455. BuildCtx * declarectx = &ctx;
  1456. if (maxRecordSize > options.maxLocalRowSize)
  1457. getInvariantMemberContext(ctx, &declarectx, NULL, false, false);
  1458. if (declarectx != &ctx)
  1459. rowType.setown(makeModifier(rowType.getClear(), typemod_member, NULL));
  1460. else
  1461. declarectx->setNextPriority(BuildCtx::OutermostScopePrio);
  1462. OwnedHqlExpr boundRow = createVariable(rowName, LINK(rowType));
  1463. StringBuffer s;
  1464. declarectx->addQuoted(s.append("byte ").append(rowName).append("[").append(maxRecordSize).append("];"));
  1465. return createBoundRow(expr, boundRow);
  1466. }
  1467. BoundRow * HqlCppTranslator::declareTempRow(BuildCtx & ctx, BuildCtx & codectx, IHqlExpression * expr)
  1468. {
  1469. assertex(expr->isDatarow());
  1470. IHqlExpression * record = expr->queryRecord();
  1471. //if maxRecordSize is too large, and cannot store it in a class, then allocate a pointer to it dynamically.
  1472. unsigned maxRecordSize = getMaxRecordSize(record);
  1473. bool createRowDynamically = tempRowRequiresFinalize(record) || ((maxRecordSize > options.maxStaticRowSize) && (maxRecordSize > options.maxLocalRowSize));
  1474. if (createRowDynamically)
  1475. {
  1476. return declareLinkedRow(ctx, expr, &ctx != &codectx);
  1477. }
  1478. else
  1479. {
  1480. return declareStaticRow(ctx, expr);
  1481. }
  1482. }
  1483. BoundRow * HqlCppTranslator::declareTempAnonRow(BuildCtx & ctx, BuildCtx & codectx, IHqlExpression * record)
  1484. {
  1485. OwnedHqlExpr anon = createRow(no_self, LINK(record->queryRecord()), createUniqueId());
  1486. return declareTempRow(ctx, codectx, anon);
  1487. }
  1488. void HqlCppTranslator::finalizeTempRow(BuildCtx & ctx, BoundRow * row, BoundRow * builder)
  1489. {
  1490. IHqlExpression * targetRow = row->queryBound();
  1491. IHqlExpression * rowBuilder = builder->queryBound();
  1492. bool targetIsOwnedRow = hasWrapperModifier(targetRow->queryType());
  1493. IHqlExpression * record = rowBuilder->queryRecord();
  1494. if (builder->queryBuilder() && targetIsOwnedRow)
  1495. {
  1496. OwnedHqlExpr createdRowSize = getRecordSize(builder->querySelector());
  1497. HqlExprArray args;
  1498. args.append(*LINK(builder->queryBuilder()));
  1499. args.append(*LINK(createdRowSize));
  1500. OwnedHqlExpr call = bindFunctionCall(finalizeRowClearAtom, args, targetRow->queryType());
  1501. CHqlBoundTarget target;
  1502. target.expr.set(targetRow);
  1503. buildExprAssign(ctx, target, call);
  1504. CHqlBoundExpr bound;
  1505. buildExpr(ctx, createdRowSize, bound);
  1506. OwnedHqlExpr sizeofTarget = createSizeof(row->querySelector());
  1507. ctx.associateExpr(sizeofTarget, bound);
  1508. }
  1509. ctx.removeAssociation(builder);
  1510. }
  1511. //---------------------------------------------------------------------------
  1512. bool GlobalFileTracker::checkMatch(IHqlExpression * searchFilename)
  1513. {
  1514. if (searchFilename == filename.get())
  1515. {
  1516. usageCount++;
  1517. return true;
  1518. }
  1519. return false;
  1520. }
  1521. void GlobalFileTracker::writeToGraph()
  1522. {
  1523. if (usageCount && graphNode)
  1524. addGraphAttributeInt(graphNode, "_globalUsageCount", usageCount);
  1525. }
  1526. //---------------------------------------------------------------------------
  1527. MetaInstance::MetaInstance(HqlCppTranslator & translator, IHqlExpression * _dataset)
  1528. {
  1529. setDataset(translator, _dataset);
  1530. }
  1531. IHqlExpression * MetaInstance::queryRecord()
  1532. {
  1533. return dataset->queryRecord();
  1534. }
  1535. void MetaInstance::setDataset(HqlCppTranslator & translator, IHqlExpression * _dataset)
  1536. {
  1537. StringBuffer s,recordBase;
  1538. dataset = _dataset;
  1539. searchKey.setown(::getMetaUniqueKey(dataset));
  1540. appendUniqueId(recordBase, translator.getConsistentUID(searchKey));
  1541. metaName.set(s.clear().append("mi").append(recordBase).str());
  1542. instanceName.set(s.clear().append("mx").append(recordBase).str());
  1543. //MORE: This function is only used by the fvsource code - getResultRecordSizeEntry
  1544. //It seems a bit of a waste generating it for something used infrequently.
  1545. metaFactoryName.set(s.clear().append("mf").append(recordBase).str());
  1546. }
  1547. //---------------------------------------------------------------------------
  1548. unsigned LocationArray::findLocation(IHqlExpression * location)
  1549. {
  1550. ISourcePath * sourcePath = location->querySourcePath();
  1551. unsigned line = location->getStartLine();
  1552. ForEachItem(i)
  1553. {
  1554. IHqlExpression & cur = item(i);
  1555. if ((cur.querySourcePath() == sourcePath) && (cur.getStartLine() == line))
  1556. return i;
  1557. }
  1558. return NotFound;
  1559. }
  1560. bool LocationArray::queryNewLocation(IHqlExpression * location)
  1561. {
  1562. if (findLocation(location) != NotFound)
  1563. return false;
  1564. append(*LINK(location));
  1565. return true;
  1566. }
  1567. IHqlDelayedCodeGenerator * ABoundActivity::createOutputCountCallback()
  1568. {
  1569. return new DelayedUnsignedGenerator(outputCount);
  1570. }
  1571. enum { createPrio = 1000, inputPrio = 3000, readyPrio = 4000, goPrio = 5000, donePrio = 7000, destroyPrio = 9000 };
  1572. ActivityInstance::ActivityInstance(HqlCppTranslator & _translator, BuildCtx & ctx, ThorActivityKind _kind, IHqlExpression * _dataset, const char * _activityArgName) :
  1573. HqlExprAssociation(activeActivityMarkerExpr),
  1574. translator(_translator), classctx(ctx), startctx(ctx), createctx(ctx), nestedctx(ctx), onstartctx(ctx)
  1575. {
  1576. dataset.set(_dataset);
  1577. kind = _kind;
  1578. node_operator op = dataset->getOperator();
  1579. isGrouped = isGroupedActivity(dataset);
  1580. isLocal = !isGrouped && isLocalActivity(dataset) && localChangesActivity(dataset) && !translator.targetHThor();
  1581. implementationClassName = NULL;
  1582. activityArgName.set(_activityArgName);
  1583. IHqlExpression * outputDataset = dataset;
  1584. if (outputDataset->isAction() && (getNumChildTables(outputDataset) == 1))
  1585. outputDataset = dataset->queryChild(0);
  1586. if (translator.targetRoxie())
  1587. {
  1588. if ((op == no_output) && dataset->hasProperty(_spill_Atom) && queryRealChild(dataset, 1))
  1589. outputDataset = dataset->queryChild(0);
  1590. }
  1591. if ((op == no_setgraphresult) && translator.queryOptions().minimizeActivityClasses)
  1592. outputDataset = dataset->queryChild(0);
  1593. meta.setDataset(translator, outputDataset);
  1594. activityId = translator.nextActivityId();
  1595. StringBuffer s;
  1596. className.set(s.clear().append("cAc").append(activityId).str());
  1597. factoryName.set(s.clear().append("fAc").append(activityId).str());
  1598. instanceName.set(s.clear().append("iAc").append(activityId).str());
  1599. argsName.set(s.clear().append("oAc").append(activityId).str());
  1600. OwnedHqlExpr boundName = createVariable(instanceName, dataset->getType());
  1601. table = new ThorBoundActivity(dataset, boundName, activityId, translator.curSubGraphId(ctx), kind);
  1602. isMember = false;
  1603. instanceIsLocal = false;
  1604. classStmt = NULL;
  1605. classGroupStmt = NULL;
  1606. hasChildActivity = false;
  1607. includedInHeader = false;
  1608. isCoLocal = true;
  1609. executedRemotely = translator.targetThor();// && !translator.isNeverDistributed(dataset);
  1610. containerActivity = NULL;
  1611. subgraph = queryActiveSubGraph(ctx);
  1612. onCreateStmt = NULL;
  1613. onCreateMarker = 0;
  1614. //count index and count disk need to be swapped to the new (much simpler) mechanism
  1615. //until then, they need to be special cased.
  1616. activityLocalisation = GraphCoLocal;
  1617. if (kind != TAKcountindex && kind != TAKcountdisk)
  1618. {
  1619. containerActivity = translator.queryCurrentActivity(ctx);
  1620. parentEvalContext.set(translator.queryEvalContext(ctx));
  1621. parentExtract.set(static_cast<ParentExtract*>(ctx.queryFirstAssociation(AssocExtract)));
  1622. if (parentExtract)
  1623. {
  1624. if (!translator.isAlwaysCoLocal())
  1625. {
  1626. GraphLocalisation localisation = parentExtract->queryLocalisation();
  1627. activityLocalisation = queryActivityLocalisation(dataset);
  1628. executedRemotely = ((activityLocalisation == GraphNonLocal) || (localisation == GraphRemote));
  1629. isCoLocal = !executedRemotely && (localisation != GraphNonLocal) && (activityLocalisation != GraphNoAccess); // if we supported GraphNonCoLocal the last test would not be needed
  1630. //if top level activity within a query library then need to force access to the parent extract
  1631. if (!containerActivity && translator.insideLibrary())
  1632. {
  1633. //there should be no colocal activity (container = null)
  1634. if (activityLocalisation != GraphNoAccess)
  1635. activityLocalisation = GraphNonLocal;
  1636. }
  1637. if (activityLocalisation == GraphNoAccess)
  1638. parentExtract.clear();
  1639. }
  1640. if (isCoLocal && containerActivity)
  1641. colocalMember.setown(createVariable("colocal", makeVoidType()));
  1642. }
  1643. else
  1644. {
  1645. if (executedRemotely)
  1646. {
  1647. GraphLocalisation localisation = queryActivityLocalisation(dataset);
  1648. if ((kind == TAKsimpleaction) || (localisation == GraphNoAccess))
  1649. executedRemotely = false;
  1650. }
  1651. }
  1652. }
  1653. else
  1654. executedRemotely = !translator.isAlwaysCoLocal();
  1655. if (!parentExtract && (translator.getTargetClusterType() == RoxieCluster))
  1656. executedRemotely = isNonLocal(dataset);
  1657. if (containerActivity)
  1658. containerActivity->hasChildActivity = true;
  1659. }
  1660. ActivityInstance::~ActivityInstance()
  1661. {
  1662. table->Release();
  1663. }
  1664. void ActivityInstance::addBaseClass(const char * name, bool needLinkOverride)
  1665. {
  1666. baseClassExtra.append(", public ").append(name);
  1667. }
  1668. ABoundActivity * ActivityInstance::getBoundActivity()
  1669. {
  1670. return LINK(table);
  1671. }
  1672. bool ActivityInstance::isExternal()
  1673. {
  1674. return !isMember && !instanceIsLocal;
  1675. }
  1676. void ActivityInstance::addAttribute(const char * name, const char * value)
  1677. {
  1678. addGraphAttribute(graphNode, name, value);
  1679. }
  1680. void ActivityInstance::addAttributeInt(const char * name, __int64 value)
  1681. {
  1682. addGraphAttributeInt(graphNode, name, value);
  1683. }
  1684. void ActivityInstance::addAttributeBool(const char * name, bool value)
  1685. {
  1686. addGraphAttributeBool(graphNode, name, value);
  1687. }
  1688. void ActivityInstance::addLocationAttribute(IHqlExpression * location)
  1689. {
  1690. if (!translator.queryOptions().reportLocations)
  1691. return;
  1692. unsigned line = location->getStartLine();
  1693. if (line == 0)
  1694. return;
  1695. if (!locations.queryNewLocation(location))
  1696. return;
  1697. ISourcePath * sourcePath = location->querySourcePath();
  1698. unsigned column = location->getStartColumn();
  1699. StringBuffer s;
  1700. s.append(sourcePath->str()).append("(").append(line);
  1701. if (column)
  1702. s.append(",").append(column);
  1703. s.append(")");
  1704. addAttribute("definition", s.str());
  1705. }
  1706. void ActivityInstance::addNameAttribute(IHqlExpression * symbol)
  1707. {
  1708. //Not so sure about adding a location for a named symbol if there are other locations already present....
  1709. //We should probably perform some deduping instead.
  1710. addLocationAttribute(symbol);
  1711. _ATOM name = symbol->queryName();
  1712. if (!name)
  1713. return;
  1714. ForEachItemIn(i, names)
  1715. {
  1716. if (names.item(i).queryName() == name)
  1717. return;
  1718. }
  1719. names.append(*symbol);
  1720. addAttribute("name", name->str());
  1721. }
  1722. void ActivityInstance::removeAttribute(const char * name)
  1723. {
  1724. removeGraphAttribute(graphNode, name);
  1725. }
  1726. static void expandHintValue(StringBuffer & s, IHqlExpression * expr)
  1727. {
  1728. switch (expr->getOperator())
  1729. {
  1730. case no_constant:
  1731. expr->queryValue()->getStringValue(s);
  1732. break;
  1733. case no_comma:
  1734. expandHintValue(s, expr->queryChild(0));
  1735. expandHintValue(s.append(","), expr->queryChild(1));
  1736. break;
  1737. case no_range:
  1738. expandHintValue(s, expr->queryChild(0));
  1739. expandHintValue(s.append(".."), expr->queryChild(1));
  1740. break;
  1741. case no_rangefrom:
  1742. expandHintValue(s, expr->queryChild(0));
  1743. s.append("..");
  1744. break;
  1745. case no_rangeto:
  1746. expandHintValue(s.append(".."), expr->queryChild(0));
  1747. break;
  1748. case no_list:
  1749. {
  1750. s.append("[");
  1751. ForEachChild(i, expr)
  1752. {
  1753. if (i)
  1754. s.append(",");
  1755. expandHintValue(s, expr->queryChild(i));
  1756. }
  1757. s.append("]");
  1758. break;
  1759. }
  1760. default:
  1761. s.append("?");
  1762. break;
  1763. }
  1764. }
  1765. void ActivityInstance::processAnnotation(IHqlExpression * annotation)
  1766. {
  1767. switch (annotation->getAnnotationKind())
  1768. {
  1769. case annotate_meta:
  1770. {
  1771. unsigned i=0;
  1772. IHqlExpression * cur;
  1773. while ((cur = annotation->queryAnnotationParameter(i++)) != NULL)
  1774. {
  1775. _ATOM name = cur->queryName();
  1776. if (name == sectionAtom)
  1777. processSection(cur);
  1778. else if (name == hintAtom)
  1779. processHints(cur);
  1780. }
  1781. }
  1782. }
  1783. }
  1784. void ActivityInstance::processAnnotations(IHqlExpression * expr)
  1785. {
  1786. ForEachChild(iHint, expr)
  1787. {
  1788. IHqlExpression * cur = expr->queryChild(iHint);
  1789. if ((cur->queryName() == hintAtom) && cur->isAttribute())
  1790. processHints(cur);
  1791. }
  1792. IHqlExpression * cur = expr;
  1793. loop
  1794. {
  1795. IHqlExpression * body = cur->queryBody(true);
  1796. if (cur == body)
  1797. break;
  1798. processAnnotation(cur);
  1799. cur = body;
  1800. }
  1801. }
  1802. void ActivityInstance::processHint(IHqlExpression * attr)
  1803. {
  1804. _ATOM name = attr->queryName();
  1805. StringBuffer value;
  1806. ForEachChild(i, attr)
  1807. {
  1808. if (i)
  1809. value.append(",");
  1810. expandHintValue(value, attr->queryChild(i));
  1811. }
  1812. if (value.length() == 0)
  1813. value.append("1");
  1814. IPropertyTree * att = createPTree();
  1815. att->setProp("@name", name->str());
  1816. att->setProp("@value", value.str());
  1817. graphNode->addPropTree("hint", att);
  1818. }
  1819. void ActivityInstance::processSection(IHqlExpression * section)
  1820. {
  1821. StringBuffer sectionName;
  1822. getStringValue(sectionName, section->queryChild(0));
  1823. addAttribute("section", sectionName);
  1824. }
  1825. void ActivityInstance::processHints(IHqlExpression * hintAttr)
  1826. {
  1827. ForEachChild(i, hintAttr)
  1828. processHint(hintAttr->queryChild(i));
  1829. }
  1830. void ActivityInstance::changeActivityKind(ThorActivityKind newKind)
  1831. {
  1832. kind = newKind;
  1833. if (graphNode)
  1834. {
  1835. removeGraphAttribute(graphNode, "_kind");
  1836. addAttributeInt("_kind", kind);
  1837. }
  1838. if (table)
  1839. table->updateActivityKind(kind);
  1840. }
  1841. void ActivityInstance::setInternalSink(bool value)
  1842. {
  1843. if (value)
  1844. addAttributeBool("_internal", true);
  1845. else
  1846. removeAttribute("_internal");
  1847. }
  1848. void ActivityInstance::createGraphNode(IPropertyTree * defaultSubGraph, bool alwaysExecuted)
  1849. {
  1850. IPropertyTree * parentGraphNode = subgraph ? subgraph->tree.get() : defaultSubGraph;
  1851. if (!parentGraphNode)
  1852. return;
  1853. assertex(kind < TAKlast);
  1854. graphNode.set(parentGraphNode->addPropTree("node", createPTree()));
  1855. graphNode->setPropInt64("@id", activityId);
  1856. StringBuffer label;
  1857. if (isGrouped)
  1858. label.append("Grouped ");
  1859. else if (isLocal)
  1860. label.append("Local ");
  1861. label.append(getActivityText(kind));
  1862. graphNode->setProp("@label", graphLabel ? graphLabel.get() : label.str());
  1863. IHqlExpression * cur = dataset;
  1864. loop
  1865. {
  1866. IHqlExpression * body = cur->queryBody(true);
  1867. if (cur == body)
  1868. break;
  1869. switch (cur->getAnnotationKind())
  1870. {
  1871. case annotate_symbol:
  1872. addNameAttribute(cur);
  1873. break;
  1874. case annotate_location:
  1875. addLocationAttribute(cur);
  1876. break;
  1877. }
  1878. cur = body;
  1879. }
  1880. addAttributeInt("_kind", kind);
  1881. addAttributeBool("grouped", isGrouped);
  1882. addAttributeBool("local", isLocal);
  1883. #ifdef _DEBUG
  1884. assertex(dataset->isAction() == isActivitySink(kind));
  1885. #endif
  1886. if (dataset->isAction())
  1887. {
  1888. if (alwaysExecuted)
  1889. markSubGraphAsRoot(parentGraphNode);
  1890. else
  1891. addAttributeBool("_internal", true);
  1892. }
  1893. if (containerActivity)
  1894. addAttributeInt("_parentActivity", containerActivity->activityId);
  1895. if (parentExtract && isCoLocal)
  1896. addAttributeBool("coLocal", true);
  1897. if (graphEclText.length() == 0)
  1898. toECL(dataset->queryBody(), graphEclText, false, true);
  1899. if (graphEclText.length() > MAX_GRAPH_ECL_LENGTH)
  1900. {
  1901. graphEclText.setLength(MAX_GRAPH_ECL_LENGTH);
  1902. graphEclText.append("...");
  1903. }
  1904. if (strcmp(graphEclText.str(), "<>") != 0)
  1905. addAttribute("ecl", graphEclText.str());
  1906. if (translator.queryOptions().includeHelperInGraph)
  1907. addAttribute("helper", factoryName);
  1908. if (translator.queryOptions().showMetaInGraph)
  1909. {
  1910. StringBuffer s;
  1911. ITypeInfo * type = dataset->queryType();
  1912. if (translator.targetThor())
  1913. {
  1914. IHqlExpression * distribution = queryDistribution(type);
  1915. if (distribution && distribution->queryName() != localAtom)
  1916. addAttribute("metaDistribution", getExprECL(distribution, s.clear(), true).str());
  1917. }
  1918. IHqlExpression * grouping = (IHqlExpression *)type->queryGroupInfo();
  1919. if (grouping)
  1920. addAttribute("metaGrouping", getExprECL(grouping, s.clear(), true).str());
  1921. if (translator.targetThor())
  1922. {
  1923. IHqlExpression * globalSortOrder = (IHqlExpression *)type->queryGlobalSortInfo();
  1924. if (globalSortOrder)
  1925. addAttribute("metaGlobalSortOrder", getExprECL(globalSortOrder, s.clear(), true).str());
  1926. }
  1927. IHqlExpression * localSortOrder = (IHqlExpression *)type->queryLocalUngroupedSortInfo();
  1928. if (localSortOrder)
  1929. addAttribute("metaLocalSortOrder", getExprECL(localSortOrder, s.clear(), true).str());
  1930. IHqlExpression * groupSortOrder = (IHqlExpression *)type->queryGroupSortInfo();
  1931. if (groupSortOrder)
  1932. addAttribute("metaGroupSortOrder", getExprECL(groupSortOrder, s.clear(), true).str());
  1933. }
  1934. if (translator.queryOptions().noteRecordSizeInGraph)
  1935. {
  1936. IHqlExpression * record = dataset->queryRecord();
  1937. if (!record && (getNumChildTables(dataset) == 1))
  1938. record = dataset->queryChild(0)->queryRecord();
  1939. if (record)
  1940. {
  1941. size32_t maxSize = getMaxRecordSize(record, translator.getDefaultMaxRecordSize());
  1942. if (isVariableSizeRecord(record))
  1943. {
  1944. size32_t minSize = getMinRecordSize(record);
  1945. size32_t expectedSize = getExpectedRecordSize(record);
  1946. StringBuffer temp;
  1947. temp.append(minSize).append("..").append(maxSize).append("(").append(expectedSize).append(")");
  1948. addAttribute("recordSize", temp.str());
  1949. }
  1950. else
  1951. addAttributeInt("recordSize", maxSize);
  1952. }
  1953. }
  1954. if (translator.queryOptions().showRecordCountInGraph && !dataset->isAction())
  1955. {
  1956. StringBuffer text;
  1957. getRecordCountText(text, dataset);
  1958. addAttribute("recordCount", text);
  1959. }
  1960. processAnnotations(dataset);
  1961. }
  1962. void ActivityInstance::moveDefinitionToHeader()
  1963. {
  1964. if (!includedInHeader)
  1965. {
  1966. //remove this class from the c++ file and include it in the header file instead
  1967. includedInHeader = true;
  1968. classGroupStmt->setIncluded(false);
  1969. BuildCtx headerctx(*translator.code, parentHelpersAtom);
  1970. headerctx.addAlias(classStmt);
  1971. }
  1972. }
  1973. void ActivityInstance::noteChildActivityLocation(IHqlExpression * pass)
  1974. {
  1975. if (containerActivity && colocalMember)
  1976. containerActivity->noteChildActivityLocation(pass);
  1977. //A child helper has been generated in a different module =>
  1978. //remove this class from the c++ file and include it in the header file instead
  1979. if (translator.queryOptions().spanMultipleCpp && !includedInHeader && (pass != sourceFileSequence))
  1980. moveDefinitionToHeader();
  1981. }
  1982. void ActivityInstance::buildPrefix()
  1983. {
  1984. StringBuffer s;
  1985. sourceFileSequence.setown(getSizetConstant(translator.cppIndexNextActivity(isChildActivity())));
  1986. if (containerActivity && colocalMember)
  1987. containerActivity->noteChildActivityLocation(sourceFileSequence);
  1988. classctx.set(helperAtom);
  1989. //Count index functions are referenced from the calling helper => they need to be defined earlier.
  1990. if ((kind == TAKcountindex) || (kind == TAKcountdisk))
  1991. classctx.setNextPriority(BuildCtx::EarlyPrio);
  1992. classGroupStmt = classctx.addGroupPass(sourceFileSequence);
  1993. classctx.associate(*this);
  1994. classctx.addGroup();
  1995. if (!implementationClassName)
  1996. {
  1997. s.clear().append("struct ").append(className).append(" : public CThor").append(activityArgName).append("Arg").append(baseClassExtra);
  1998. classStmt = classctx.addQuotedCompound(s, ";");
  1999. if (subgraph)
  2000. classctx.associate(*subgraph);
  2001. //Generate functions in the order i) always callable ii) after create iii) after start
  2002. nestedctx.set(classctx);
  2003. nestedctx.addGroup();
  2004. createctx.set(classctx);
  2005. createctx.setNextDestructor();
  2006. createctx.addGroup();
  2007. startctx.set(createctx);
  2008. startctx.setNextDestructor();
  2009. startctx.addGroup();
  2010. createctx.associateExpr(codeContextMarkerExpr, codeContextMarkerExpr);
  2011. evalContext.setown(new ActivityEvalContext(translator, this, parentExtract, parentEvalContext, colocalMember, createctx, startctx));
  2012. classctx.associate(*evalContext);
  2013. //virtual void onCreate(ICodeContext * ctx, IHThorArg * colocalParent, MemoryBuffer * serializedCreate)
  2014. BuildCtx oncreatectx(createctx);
  2015. if (parentExtract && isCoLocal && containerActivity)
  2016. {
  2017. oncreatectx.addQuotedCompound("virtual void onCreate(ICodeContext * _ctx, IHThorArg * _colocal, MemoryBuffer * in)");
  2018. oncreatectx.addQuoted(s.clear().append("colocal = (").append(containerActivity->className).append("*)_colocal;"));
  2019. }
  2020. else
  2021. {
  2022. onCreateStmt = oncreatectx.addQuotedCompound("virtual void onCreate(ICodeContext * _ctx, IHThorArg *, MemoryBuffer * in)");
  2023. }
  2024. oncreatectx.associateExpr(insideOnCreateMarker, NULL);
  2025. oncreatectx.addQuoted("ctx = _ctx;");
  2026. evalContext->onCreate.createFunctionStructure(translator, oncreatectx, true, executedRemotely ? "serializeCreateContext" : NULL);
  2027. if (onCreateStmt)
  2028. onCreateMarker = calcTotalChildren(onCreateStmt);
  2029. onstartctx.set(startctx);
  2030. if (parentExtract)
  2031. {
  2032. onstartctx.addQuotedCompoundOpt("virtual void onStart(const byte * pe, MemoryBuffer * in)");
  2033. parentExtract->beginChildActivity(startctx, onstartctx, activityLocalisation, colocalMember, false, false, containerActivity);
  2034. }
  2035. else
  2036. {
  2037. onstartctx.addQuotedCompoundOpt("virtual void onStart(const byte *, MemoryBuffer * in)");
  2038. }
  2039. onstartctx.associateExpr(insideOnStartMarker, NULL);
  2040. evalContext->onStart.createFunctionStructure(translator, onstartctx, true, executedRemotely ? "serializeStartContext" : NULL);
  2041. if (colocalMember)
  2042. {
  2043. s.clear().append(containerActivity->className).append(" * colocal;");
  2044. classctx.addQuoted(s);
  2045. }
  2046. if (baseClassExtra.length())
  2047. {
  2048. nestedctx.addQuoted(s.clear().append("virtual void Link() const { CThor").append(activityArgName).append("Arg::Link(); }"));
  2049. nestedctx.addQuoted(s.clear().append("virtual bool Release() const { return CThor").append(activityArgName).append("Arg::Release(); }"));
  2050. }
  2051. // if (!isMember)
  2052. // classGroupStmt->setIncomplete(true);
  2053. if (translator.queryOptions().spanMultipleCpp && ((kind == TAKcountindex) || (kind == TAKcountdisk)))
  2054. moveDefinitionToHeader();
  2055. }
  2056. else
  2057. {
  2058. s.clear().append("// use library for ").append(className);
  2059. classctx.addQuoted(s);
  2060. assertex(isExternal());
  2061. }
  2062. }
  2063. void ActivityInstance::buildSuffix()
  2064. {
  2065. //If onCreate() doesn't do anything special, then use an implementation in the base
  2066. if (onCreateStmt && (calcTotalChildren(onCreateStmt) == onCreateMarker))
  2067. onCreateStmt->setIncluded(false);
  2068. const HqlCppOptions & options = translator.queryOptions();
  2069. if (classStmt && (options.spotComplexClasses || options.showActivitySizeInGraph))
  2070. {
  2071. unsigned approxSize = calcTotalChildren(classStmt);
  2072. if (options.spotComplexClasses && (approxSize >= options.complexClassesThreshold))
  2073. {
  2074. if ((options.complexClassesActivityFilter == 0) || (kind == options.complexClassesActivityFilter))
  2075. translator.WARNING2(HQLWRN_ComplexHelperClass, activityId, approxSize);
  2076. }
  2077. if (options.showActivitySizeInGraph)
  2078. addAttributeInt("approxClassSize", approxSize);
  2079. }
  2080. // if (!isMember)
  2081. // classGroupStmt->setIncomplete(false);
  2082. if (parentExtract)
  2083. parentExtract->endChildActivity();
  2084. classctx.removeAssociation(this);
  2085. locations.kill();
  2086. names.kill();
  2087. if (isExternal())
  2088. {
  2089. BuildCtx globalctx(*translator.code, helperAtom);
  2090. globalctx.addGroupPass(sourceFileSequence);
  2091. if (implementationClassName)
  2092. {
  2093. //Meta is always the last parameter...
  2094. addConstructorMetaParameter();
  2095. StringBuffer s;
  2096. s.append("extern \"C\" ECL_API IHThorArg * ").append(factoryName).append("()");
  2097. globalctx.addQuotedCompound(s);
  2098. OwnedHqlExpr call = translator.bindFunctionCall(implementationClassName, constructorArgs);
  2099. //Don't call buildReturn because we're lying about the return type, and we don't want a boolean temporary created.
  2100. CHqlBoundExpr bound;
  2101. translator.buildExpr(globalctx, call, bound);
  2102. globalctx.addReturn(bound.expr);
  2103. //translator.buildReturn(globalctx, call);
  2104. }
  2105. else
  2106. {
  2107. StringBuffer s;
  2108. s.append("extern \"C\" ECL_API IHThorArg * ").append(factoryName).append("() { return new ").append(className).append("; }");
  2109. globalctx.addQuoted(s);
  2110. }
  2111. }
  2112. }
  2113. void ActivityInstance::buildMetaMember()
  2114. {
  2115. if (implementationClassName)
  2116. return;
  2117. translator.buildMetaInfo(meta);
  2118. IHqlExpression * dataset = meta.dataset;
  2119. ITypeInfo * type = dataset->queryType();
  2120. if (type && type->getTypeCode() != type_void)
  2121. {
  2122. StringBuffer s;
  2123. s.append("virtual IOutputMetaData * queryOutputMeta() { return &").append(meta.queryInstanceObject()).append("; }");
  2124. classctx.addQuoted(s);
  2125. }
  2126. }
  2127. void ActivityInstance::addConstructorMetaParameter()
  2128. {
  2129. translator.buildMetaInfo(meta);
  2130. ITypeInfo * type = meta.dataset->queryType();
  2131. if (type && type->getTypeCode() != type_void)
  2132. {
  2133. StringBuffer s;
  2134. s.append("&").append(meta.queryInstanceObject());
  2135. OwnedHqlExpr metaExpr = createQuoted(s.str(), makeBoolType());
  2136. constructorArgs.append(*metaExpr.getClear());
  2137. }
  2138. }
  2139. ParentExtract * ActivityInstance::createNestedExtract()
  2140. {
  2141. if (!nestedExtract)
  2142. {
  2143. nestedExtract.setown(new ParentExtract(translator, GraphCoLocal, evalContext));
  2144. nestedExtract->beginNestedExtract(startctx);
  2145. }
  2146. return LINK(nestedExtract);
  2147. }
  2148. //----------------------------------------------------------------------------------------------------
  2149. StringBuffer &expandLiteral(StringBuffer &s, const char *f)
  2150. {
  2151. const char * startLine = f;
  2152. char c;
  2153. s.append('"');
  2154. while ((c = *f++) != 0)
  2155. {
  2156. switch (c)
  2157. {
  2158. case '\t':
  2159. s.append("\\t");
  2160. break;
  2161. case '\r':
  2162. s.append("\\r");
  2163. break;
  2164. case '\n':
  2165. s.append("\\n\"\n\t\t\"");
  2166. startLine = f;
  2167. break;
  2168. case ',':
  2169. s.append(c);
  2170. if (f - startLine > 60)
  2171. {
  2172. s.append("\"\n\t\t\"");
  2173. startLine = f;
  2174. }
  2175. break;
  2176. case '\\':
  2177. case '\"':
  2178. case '\'':
  2179. s.append('\\');
  2180. // fall into...
  2181. default:
  2182. s.append(c);
  2183. break;
  2184. }
  2185. if (f - startLine > 120)
  2186. {
  2187. s.append("\"\n\t\t\"");
  2188. startLine = f;
  2189. }
  2190. }
  2191. return s.append("\"");
  2192. }
  2193. //---------------------------------------------------------------------------
  2194. ReferenceSelector::ReferenceSelector(HqlCppTranslator & _translator) : translator(_translator)
  2195. {
  2196. }
  2197. //---------------------------------------------------------------------------
  2198. /* In parms: _dataset, _path are NOT linked */
  2199. DatasetSelector::DatasetSelector(HqlCppTranslator & _translator, BoundRow * _row, IHqlExpression * _path)
  2200. : ReferenceSelector(_translator)
  2201. {
  2202. row = LINK(_row);
  2203. matchedDataset = false;
  2204. column = row->queryRootColumn();
  2205. column->Link();
  2206. path.set(_path);
  2207. if (!_path)
  2208. path.set(row->querySelector());
  2209. parent = NULL;
  2210. }
  2211. /* All in parms: NOT linked */
  2212. DatasetSelector::DatasetSelector(DatasetSelector * _parent, BoundRow * _row, AColumnInfo * _column, IHqlExpression * _path)
  2213. : ReferenceSelector(_parent->translator)
  2214. {
  2215. parent = LINK(_parent);
  2216. matchedDataset = _parent->matchedDataset;
  2217. column = LINK(_column);
  2218. path.set(_path);
  2219. row = LINK(_row);
  2220. }
  2221. DatasetSelector::~DatasetSelector()
  2222. {
  2223. ::Release(column);
  2224. ::Release(parent);
  2225. ::Release(row);
  2226. }
  2227. void DatasetSelector::assignTo(BuildCtx & ctx, const CHqlBoundTarget & target)
  2228. {
  2229. OwnedHqlExpr mapped = row->getMappedSelector(ctx, this);
  2230. if (mapped)
  2231. translator.buildExprAssign(ctx, target, mapped);
  2232. else
  2233. column->buildAssign(translator, ctx, this, target);
  2234. }
  2235. void DatasetSelector::buildAddress(BuildCtx & ctx, CHqlBoundExpr & target)
  2236. {
  2237. OwnedHqlExpr mapped = row->getMappedSelector(ctx, this);
  2238. if (mapped)
  2239. translator.buildAddress(ctx, mapped, target);
  2240. else
  2241. column->buildAddress(translator, ctx, this, target);
  2242. }
  2243. void DatasetSelector::buildClear(BuildCtx & ctx, int direction)
  2244. {
  2245. assertex(row->isModifyable());
  2246. column->buildClear(translator, ctx, this, direction);
  2247. }
  2248. bool DatasetSelector::isBinary()
  2249. {
  2250. return row->isBinary();
  2251. }
  2252. bool DatasetSelector::isRoot()
  2253. {
  2254. return parent == NULL;
  2255. }
  2256. /* _newCursor, _newColumn: not linked. newPathOwn: linked */
  2257. DatasetSelector * DatasetSelector::createChild(BoundRow * _newCursor, AColumnInfo * newColumn, IHqlExpression * newPath)
  2258. {
  2259. return new DatasetSelector(this, _newCursor, newColumn, newPath);
  2260. }
  2261. void DatasetSelector::get(BuildCtx & ctx, CHqlBoundExpr & bound)
  2262. {
  2263. OwnedHqlExpr mapped = row->getMappedSelector(ctx, this);
  2264. if (mapped)
  2265. translator.buildAnyExpr(ctx, mapped, bound);
  2266. else
  2267. column->buildExpr(translator, ctx, this, bound);
  2268. }
  2269. void DatasetSelector::getOffset(BuildCtx & ctx, CHqlBoundExpr & bound)
  2270. {
  2271. //OwnedHqlExpr mapped = row->getMappedSelector(ctx, this);
  2272. assertex(!row->isNonLocal());
  2273. column->buildOffset(translator, ctx, this, bound);
  2274. }
  2275. size32_t DatasetSelector::getContainerTrailingFixed()
  2276. {
  2277. assertex(!row->isNonLocal());
  2278. return column->getContainerTrailingFixed();
  2279. }
  2280. void DatasetSelector::getSize(BuildCtx & ctx, CHqlBoundExpr & bound)
  2281. {
  2282. //OwnedHqlExpr mapped = row->getMappedSelector(ctx, this);
  2283. assertex(!row->isNonLocal());
  2284. column->buildSizeOf(translator, ctx, this, bound);
  2285. }
  2286. bool DatasetSelector::isDataset()
  2287. {
  2288. switch (column->queryType()->getTypeCode())
  2289. {
  2290. case type_table:
  2291. case type_groupedtable:
  2292. return true;
  2293. }
  2294. return false;
  2295. }
  2296. bool DatasetSelector::isConditional()
  2297. {
  2298. return row->isConditional() || column->isConditional();
  2299. }
  2300. AColumnInfo * DatasetSelector::queryColumn()
  2301. {
  2302. return column;
  2303. }
  2304. BoundRow * DatasetSelector::queryRootRow()
  2305. {
  2306. return row;
  2307. }
  2308. IHqlExpression * DatasetSelector::queryExpr()
  2309. {
  2310. return path;
  2311. }
  2312. ITypeInfo * DatasetSelector::queryType()
  2313. {
  2314. if (parent)
  2315. return path->queryType();
  2316. return row->queryDataset()->queryType();
  2317. }
  2318. IHqlExpression * DatasetSelector::resolveChildDataset(IHqlExpression * searchDataset) const
  2319. {
  2320. searchDataset = searchDataset->queryNormalizedSelector();
  2321. IHqlExpression * compare = row->queryDataset()->queryNormalizedSelector();
  2322. if (searchDataset == compare)
  2323. return compare;
  2324. return NULL;
  2325. }
  2326. AColumnInfo * DatasetSelector::resolveField(IHqlExpression * search) const
  2327. {
  2328. AColumnInfo * nextColumn = column->lookupColumn(search);
  2329. if (nextColumn)
  2330. return nextColumn;
  2331. return NULL;
  2332. }
  2333. void DatasetSelector::set(BuildCtx & ctx, IHqlExpression * expr)
  2334. {
  2335. assertex(row->isModifyable());
  2336. ITypeInfo * type = column->queryType();
  2337. if (!hasReferenceModifier(type))
  2338. {
  2339. switch (type->getTypeCode())
  2340. {
  2341. case type_row:
  2342. {
  2343. if (queryRecordType(queryType()) == expr->queryRecordType())
  2344. {
  2345. translator.buildRowAssign(ctx, this, expr);
  2346. }
  2347. else
  2348. {
  2349. Owned<IReferenceSelector> sourceRef = translator.buildNewRow(ctx, expr);
  2350. setRow(ctx, sourceRef);
  2351. }
  2352. return;
  2353. }
  2354. }
  2355. }
  2356. column->setColumn(translator, ctx, this, expr);
  2357. }
  2358. void DatasetSelector::setRow(BuildCtx & ctx, IReferenceSelector * rhs)
  2359. {
  2360. assertex(row->isModifyable());
  2361. column->setRow(translator, ctx, this, rhs);
  2362. }
  2363. void DatasetSelector::modifyOp(BuildCtx & ctx, IHqlExpression * expr, node_operator op)
  2364. {
  2365. assertex(row->isModifyable());
  2366. // IReferenceSelector * aliasedSelector = row->queryAlias();
  2367. // assertex(aliasedSelector);
  2368. if (column->modifyColumn(translator, ctx, this, expr, op))
  2369. return;
  2370. // OwnedHqlExpr sourceValue = aliasedSelector->queryRootRow()->bindToRow(path, row->querySelector());
  2371. OwnedHqlExpr sourceValue = LINK(path);
  2372. OwnedHqlExpr result;
  2373. switch (op)
  2374. {
  2375. case no_assign_addfiles:
  2376. result.setown(createDataset(no_addfiles, ensureOwned(sourceValue), LINK(expr)));
  2377. break;
  2378. #ifdef _THE_FOLLOWING_ARENT_YET_IMPLEMENTED_BUT_WOULD_BE_USEFUL
  2379. case no_assign_concat:
  2380. result.setown(createValue(no_concat, path->getType(), LINK(sourceValue), LINK(expr)));
  2381. break;
  2382. case no_assign_add:
  2383. result.setown(createValue(no_add, path->getType(), LINK(sourceValue), LINK(expr)));
  2384. break;
  2385. #endif
  2386. default:
  2387. throwError1(HQLERR_UnknownCompoundAssign, getOpString(op));
  2388. }
  2389. set(ctx, result);
  2390. }
  2391. void DatasetSelector::buildDeserialize(BuildCtx & ctx, IHqlExpression * helper)
  2392. {
  2393. column->buildDeserialize(translator, ctx, this, helper);
  2394. }
  2395. void DatasetSelector::buildSerialize(BuildCtx & ctx, IHqlExpression * helper)
  2396. {
  2397. column->buildSerialize(translator, ctx, this, helper);
  2398. }
  2399. /* In selector: not linked. Return: linked */
  2400. IReferenceSelector * DatasetSelector::select(BuildCtx & ctx, IHqlExpression * selectExpr)
  2401. {
  2402. // assertex(!isDataset());
  2403. OwnedHqlExpr selected;
  2404. //Optimize so don't create so many select expressions that already exist.
  2405. if (selectExpr->getOperator() != no_select)
  2406. selected.setown(createSelectExpr(LINK(path), LINK(selectExpr)));
  2407. else if (selectExpr->queryChild(0) == path)
  2408. selected.set(selectExpr);
  2409. else
  2410. selected.setown(createSelectExpr(LINK(path), LINK(selectExpr->queryChild(1))));
  2411. IHqlExpression * selectedField = selected->queryChild(1);
  2412. AColumnInfo * newColumn = resolveField(selectedField);
  2413. if (!newColumn)
  2414. {
  2415. //could be a dataset selector
  2416. IHqlExpression * selected = NULL;
  2417. if (!matchedDataset)
  2418. selected = resolveChildDataset(selectedField);
  2419. if (!selected)
  2420. {
  2421. #ifdef TraceTableFields
  2422. IHqlExpression * searchDataset = row->queryDataset(); // MORE what when children selected?
  2423. IHqlExpression * record = searchDataset->queryRecord()->queryBody();
  2424. unsigned numFields = record->numChildren();
  2425. unsigned idxc;
  2426. StringBuffer fields;
  2427. PrintLog("Fields:");
  2428. for (idxc = 0; idxc < numFields; idxc++)
  2429. {
  2430. IHqlExpression * field = record->queryChild(idxc);
  2431. _ATOM name = field->queryName();
  2432. fields.clear();
  2433. fields.appendf(" %20s [@%lx := %lx] ", name->str(), field, field->queryChild(0));
  2434. PrintLog(fields.str());
  2435. }
  2436. PrintLog("Search: %20s [@%lx])", selectedField->queryName()->str(),selectedField);
  2437. #endif
  2438. StringBuffer searchName, datasetName;
  2439. selectedField->toString(searchName);
  2440. if (path->queryName())
  2441. datasetName.append(path->queryName());
  2442. else
  2443. path->toString(datasetName);
  2444. throwError2(HQLERR_XDoesNotContainExpressionY, datasetName.str(), searchName.str());
  2445. }
  2446. DatasetSelector * next = createChild(row, column, selected);
  2447. next->matchedDataset = true;
  2448. return next;
  2449. }
  2450. return createChild(row, newColumn, selected);
  2451. }
  2452. BoundRow * DatasetSelector::getRow(BuildCtx & ctx)
  2453. {
  2454. if (isRoot())
  2455. return LINK(row);
  2456. CHqlBoundExpr bound;
  2457. buildAddress(ctx, bound);
  2458. Owned<ITypeInfo> type = makeReferenceModifier(LINK(queryType()));
  2459. OwnedHqlExpr address = createValue(no_implicitcast, type.getClear(), LINK(bound.expr));
  2460. return translator.createBoundRow(path, address);
  2461. }
  2462. IReferenceSelector * HqlCppTranslator::createSelfSelect(BuildCtx & ctx, IReferenceSelector * target, IHqlExpression * expr, IHqlExpression * rootSelector)
  2463. {
  2464. if (expr == rootSelector)
  2465. return LINK(target);
  2466. assertex(expr->getOperator() == no_select);
  2467. Owned<IReferenceSelector> parent = createSelfSelect(ctx, target, expr->queryChild(0), rootSelector);
  2468. return parent->select(ctx, expr);
  2469. }
  2470. void initBoundStringTarget(CHqlBoundTarget & target, ITypeInfo * type, const char * lenName, const char * dataName)
  2471. {
  2472. if (type->getSize() == UNKNOWN_LENGTH)
  2473. target.length.setown(createVariable(lenName, LINK(sizetType)));
  2474. target.expr.setown(createVariable(dataName, makeReferenceModifier(LINK(type))));
  2475. }
  2476. //---------------------------------------------------------------------------
  2477. GlobalClassBuilder::GlobalClassBuilder(HqlCppTranslator & _translator, BuildCtx & _ctx, const char * _className, const char * _baseName, const char * _accessorInterface)
  2478. : translator(_translator), classctx(_ctx), nestedctx(_ctx), startctx(_ctx), createctx(_ctx)
  2479. {
  2480. className.set(_className);
  2481. baseName.set(_baseName);
  2482. accessorInterface.set(_accessorInterface);
  2483. if (accessorInterface)
  2484. {
  2485. StringBuffer s;
  2486. accessorName.set(s.clear().append("cr").append(className).str());
  2487. }
  2488. onCreateStmt = NULL;
  2489. }
  2490. void GlobalClassBuilder::buildClass(unsigned priority)
  2491. {
  2492. StringBuffer s;
  2493. s.append("struct ").append(className);
  2494. if (baseName)
  2495. s.append(" : public ").append(baseName);
  2496. classctx.set(declareAtom);
  2497. if (priority)
  2498. classctx.setNextPriority(priority);
  2499. classStmt = classctx.addQuotedCompound(s, ";");
  2500. if (!baseName)
  2501. classctx.addQuoted("ICodeContext * ctx;");
  2502. classctx.associateExpr(codeContextMarkerExpr, codeContextMarkerExpr);
  2503. //Generate functions in the order i) always callable ii) after create iii) after start
  2504. nestedctx.set(classctx);
  2505. nestedctx.addGroup();
  2506. createctx.set(classctx);
  2507. createctx.setNextDestructor();
  2508. createctx.addGroup();
  2509. startctx.set(createctx);
  2510. evalContext.setown(new GlobalClassEvalContext(translator, parentExtract, parentEvalContext, createctx, startctx));
  2511. classctx.associate(*evalContext);
  2512. //virtual void onCreate(ICodeContext * ctx, IHThorArg * colocalParent, MemoryBuffer * serializedCreate)
  2513. BuildCtx oncreatectx(createctx);
  2514. onCreateStmt = oncreatectx.addQuotedCompound("void onCreate(ICodeContext * _ctx)");
  2515. oncreatectx.associateExpr(insideOnCreateMarker, NULL);
  2516. oncreatectx.addQuoted("ctx = _ctx;");
  2517. evalContext->onCreate.createFunctionStructure(translator, oncreatectx, true, NULL);
  2518. onCreateMarker = calcTotalChildren(onCreateStmt);
  2519. }
  2520. void GlobalClassBuilder::completeClass(unsigned priority)
  2521. {
  2522. if (onCreateStmt && (calcTotalChildren(onCreateStmt) == onCreateMarker))
  2523. onCreateStmt->setIncluded(false);
  2524. //MORE: This should be generated from a system function prototype somehow - so we can extend it to user functions later.
  2525. //arguments and parameters should also be configured similarly.
  2526. if (accessorInterface)
  2527. {
  2528. StringBuffer s, prototype;
  2529. prototype.append("extern ECL_API ").append(accessorInterface).append(" * ").append(accessorName).append("(ICodeContext * ctx, unsigned activityId)");
  2530. BuildCtx accessctx(classctx);
  2531. accessctx.set(declareAtom);
  2532. if (priority)
  2533. accessctx.setNextPriority(priority);
  2534. accessctx.addQuotedCompound(prototype);
  2535. accessctx.addQuoted(s.clear().append(className).append("* p = new ").append(className).append("(activityId); "));
  2536. accessctx.addQuoted("p->onCreate(ctx);");
  2537. accessctx.addQuoted("return p;");
  2538. if (translator.queryOptions().spanMultipleCpp)
  2539. {
  2540. BuildCtx protoctx(*translator.queryCode(), mainprototypesAtom);
  2541. protoctx.addQuoted(s.clear().append(prototype).append(";"));
  2542. }
  2543. }
  2544. }
  2545. //---------------------------------------------------------------------------
  2546. static bool expandThisPass(IHqlExpression * name, unsigned pass)
  2547. {
  2548. if (!name)
  2549. return pass == 1;
  2550. StringBuffer temp;
  2551. name->queryValue()->getStringValue(temp);
  2552. if (temp.length() && temp.charAt(0) =='@')
  2553. return pass == 0;
  2554. return pass == 1;
  2555. }
  2556. static bool anyXmlGeneratedForPass(IHqlExpression * expr, unsigned pass)
  2557. {
  2558. switch (expr->getOperator())
  2559. {
  2560. case no_field:
  2561. {
  2562. OwnedHqlExpr name;
  2563. extractXmlName(name, NULL, NULL, expr, NULL, false);
  2564. ITypeInfo * type = expr->queryType()->queryPromotedType();
  2565. switch (type->getTypeCode())
  2566. {
  2567. case type_row:
  2568. if (name)
  2569. return (pass == 1);
  2570. return anyXmlGeneratedForPass(queryRecord(type), pass);
  2571. case type_set:
  2572. return (pass==1);
  2573. case type_table:
  2574. case type_groupedtable:
  2575. return (pass == 1);
  2576. default:
  2577. return expandThisPass(name, pass);
  2578. }
  2579. break;
  2580. }
  2581. case no_ifblock:
  2582. return anyXmlGeneratedForPass(expr->queryChild(1), pass);
  2583. case no_record:
  2584. {
  2585. ForEachChild(idx, expr)
  2586. if (anyXmlGeneratedForPass(expr->queryChild(idx), pass))
  2587. return true;
  2588. return false;
  2589. }
  2590. case no_attr:
  2591. case no_attr_expr:
  2592. case no_attr_link:
  2593. return false;
  2594. default:
  2595. UNIMPLEMENTED;
  2596. }
  2597. }
  2598. //---------------------------------------------------------------------------
  2599. bool HqlCppTranslator::insideOnCreate(BuildCtx & ctx)
  2600. {
  2601. return ctx.queryMatchExpr(insideOnCreateMarker) != NULL;
  2602. }
  2603. bool HqlCppTranslator::getInvariantMemberContext(BuildCtx & ctx, BuildCtx * * declarectx, BuildCtx * * initctx, bool isIndependentMaybeShared, bool invariantEachStart)
  2604. {
  2605. EvalContext * instance = queryEvalContext(ctx);
  2606. if (instance)
  2607. return instance->getInvariantMemberContext(&ctx, declarectx, initctx, isIndependentMaybeShared, invariantEachStart);
  2608. return false;
  2609. }
  2610. void getMemberClassName(StringBuffer & className, const char * member)
  2611. {
  2612. className.append((char)toupper(member[0])).append(member+1).append("Class");
  2613. }
  2614. void HqlCppTranslator::beginNestedClass(BuildCtx & ctx, const char * member, const char * bases, const char * memberExtra, ParentExtract * extract)
  2615. {
  2616. // ActivityInstance * activity = queryCurrentActivity(ctx);
  2617. // Owned<ParentExtract> nestedUse;
  2618. // if (activity)
  2619. // nestedUse.setown(extract ? LINK(extract) : activity->createNestedExtract());
  2620. StringBuffer begin,end;
  2621. StringBuffer className;
  2622. getMemberClassName(className, member);
  2623. begin.append("struct ").append((char)toupper(member[0])).append(member+1).append("Class");
  2624. if (bases)
  2625. begin.append(" : public ").append(bases);
  2626. end.append(" ").append(member).append(memberExtra).append(";");
  2627. ctx.addQuotedCompound(begin.str(), end.str());
  2628. OwnedHqlExpr colocalName = createVariable("activity", makeVoidType());
  2629. ActivityInstance * activity = queryCurrentActivity(ctx);
  2630. if (activity)
  2631. {
  2632. Owned<ParentExtract> nestedUse = extract ? LINK(extract) : activity->createNestedExtract();
  2633. NestedEvalContext * nested = new NestedEvalContext(*this, member, nestedUse, queryEvalContext(ctx), colocalName, ctx, ctx);
  2634. ctx.associateOwn(*nested);
  2635. nested->initContext();
  2636. }
  2637. }
  2638. void HqlCppTranslator::endNestedClass()
  2639. {
  2640. }
  2641. void HqlCppTranslator::pushMemberFunction(MemberFunction & func)
  2642. {
  2643. }
  2644. void HqlCppTranslator::popMemberFunction()
  2645. {
  2646. }
  2647. void HqlCppTranslator::doBuildFunctionReturn(BuildCtx & ctx, ITypeInfo * type, IHqlExpression * value)
  2648. {
  2649. bool returnByReference = false;
  2650. CHqlBoundTarget target;
  2651. switch (type->getTypeCode())
  2652. {
  2653. case type_varstring:
  2654. case type_varunicode:
  2655. if (type->getSize() == UNKNOWN_LENGTH)
  2656. break;
  2657. //fall through
  2658. case type_qstring:
  2659. case type_string:
  2660. case type_data:
  2661. case type_unicode:
  2662. case type_utf8:
  2663. case type_table:
  2664. case type_groupedtable:
  2665. case type_row:
  2666. initBoundStringTarget(target, type, "__lenResult", "__result");
  2667. returnByReference = true;
  2668. break;
  2669. case type_set:
  2670. target.isAll.setown(createVariable("__isAllResult", makeBoolType()));
  2671. target.length.setown(createVariable("__lenResult", LINK(sizetType)));
  2672. target.expr.setown(createVariable("__result", makeReferenceModifier(LINK(type))));
  2673. returnByReference = true;
  2674. break;
  2675. }
  2676. if (returnByReference)
  2677. buildExprAssign(ctx, target, value);
  2678. else
  2679. buildReturn(ctx, value, type);
  2680. }
  2681. //MORE: Should have a generalized doBuildFunctionHeader(ctx, name, retType, HqlExprArray & parameters)
  2682. // for when we generate functions for aliases
  2683. void HqlCppTranslator::doBuildBoolFunction(BuildCtx & ctx, const char * name, bool value)
  2684. {
  2685. StringBuffer s;
  2686. s.append("virtual bool ").append(name).append("() { return ").append(TF[value]).append("; }");
  2687. ctx.addQuoted(s);
  2688. }
  2689. void HqlCppTranslator::doBuildDataFunction(BuildCtx & ctx, const char * name, IHqlExpression * value)
  2690. {
  2691. doBuildFunction(ctx, unknownDataType, name, value);
  2692. }
  2693. void HqlCppTranslator::doBuildStringFunction(BuildCtx & ctx, const char * name, IHqlExpression * value)
  2694. {
  2695. doBuildFunction(ctx, unknownStringType, name, value);
  2696. }
  2697. void HqlCppTranslator::doBuildVarStringFunction(BuildCtx & ctx, const char * name, IHqlExpression * value)
  2698. {
  2699. StringBuffer s;
  2700. BuildCtx funcctx(ctx);
  2701. funcctx.addQuotedCompound(s.append("virtual const char * ").append(name).append("()"));
  2702. if (value)
  2703. buildReturn(funcctx, value, constUnknownVarStringType);
  2704. else
  2705. funcctx.addReturn(queryQuotedNullExpr());
  2706. }
  2707. void HqlCppTranslator::doBuildBoolFunction(BuildCtx & ctx, const char * name, IHqlExpression * value)
  2708. {
  2709. doBuildFunction(ctx, queryBoolType(), name, value);
  2710. }
  2711. void HqlCppTranslator::doBuildUnsignedFunction(BuildCtx & ctx, const char * name, unsigned value)
  2712. {
  2713. StringBuffer s;
  2714. s.append("virtual unsigned ").append(name).append("() { return ").append(value).append("; }");
  2715. ctx.addQuoted(s);
  2716. }
  2717. void HqlCppTranslator::doBuildSizetFunction(BuildCtx & ctx, const char * name, size32_t value)
  2718. {
  2719. StringBuffer s;
  2720. s.append("virtual size32_t ").append(name).append("() { return ").append(value).append("; }");
  2721. ctx.addQuoted(s);
  2722. }
  2723. void HqlCppTranslator::doBuildUnsignedFunction(BuildCtx & ctx, const char * name, IHqlExpression * value)
  2724. {
  2725. doBuildFunction(ctx, unsignedType, name, value);
  2726. }
  2727. void HqlCppTranslator::doBuildUnsigned64Function(BuildCtx & ctx, const char * name, IHqlExpression * value)
  2728. {
  2729. Owned<ITypeInfo> type = makeIntType(8, false);
  2730. doBuildFunction(ctx, type, name, value);
  2731. }
  2732. void HqlCppTranslator::doBuildUnsignedFunction(BuildCtx & ctx, const char * name, const char * value)
  2733. {
  2734. if (value)
  2735. {
  2736. StringBuffer s;
  2737. s.append("virtual unsigned ").append(name).append("() { return ").append(value).append("; }");
  2738. ctx.addQuoted(s);
  2739. }
  2740. }
  2741. void HqlCppTranslator::doBuildSignedFunction(BuildCtx & ctx, const char * name, IHqlExpression * value)
  2742. {
  2743. doBuildFunction(ctx, signedType, name, value);
  2744. }
  2745. void HqlCppTranslator::doBuildFunction(BuildCtx & ctx, ITypeInfo * type, const char * name, IHqlExpression * value)
  2746. {
  2747. if (value)
  2748. {
  2749. LinkedHqlExpr cseValue = value;
  2750. if (options.spotCSE)
  2751. cseValue.setown(spotScalarCSE(cseValue));
  2752. StringBuffer s, returnParameters;
  2753. s.append("virtual ");
  2754. expandFunctionReturnType(s, returnParameters, type, NULL);
  2755. s.append(" ").append(name).append("(").append(returnParameters).append(")");
  2756. BuildCtx funcctx(ctx);
  2757. funcctx.addQuotedCompound(s);
  2758. doBuildFunctionReturn(funcctx, type, cseValue);
  2759. }
  2760. }
  2761. void HqlCppTranslator::addFilenameConstructorParameter(ActivityInstance & instance, const char * name, IHqlExpression * expr)
  2762. {
  2763. OwnedHqlExpr folded = foldHqlExpression(expr);
  2764. instance.addConstructorParameter(folded);
  2765. noteFilename(instance, name, folded, false);
  2766. }
  2767. void HqlCppTranslator::buildFilenameFunction(ActivityInstance & instance, BuildCtx & classctx, const char * name, IHqlExpression * expr, bool isDynamic)
  2768. {
  2769. OwnedHqlExpr folded = foldHqlExpression(expr);
  2770. doBuildVarStringFunction(classctx, name, folded);
  2771. noteFilename(instance, name, folded, isDynamic);
  2772. }
  2773. void HqlCppTranslator::noteFilename(ActivityInstance & instance, const char * name, IHqlExpression * expr, bool isDynamic)
  2774. {
  2775. if (options.addFilesnamesToGraph)
  2776. {
  2777. StringBuffer propName;
  2778. const char * propNameBase = name;
  2779. if (memicmp(propNameBase, "get", 3) == 0)
  2780. propNameBase += 3;
  2781. else if (memicmp(propNameBase, "query", 3) == 0)
  2782. propNameBase += 5;
  2783. propName.append("_").append((char)tolower(*propNameBase)).append(propNameBase+1);
  2784. OwnedHqlExpr folded = foldHqlExpression(expr);
  2785. if (folded)
  2786. {
  2787. if (!folded->queryValue())
  2788. {
  2789. if (!isDynamic && !options.allowVariableRoxieFilenames && targetRoxie())
  2790. {
  2791. StringBuffer x;
  2792. folded->toString(x);
  2793. throwError1(HQLERR_RoxieExpectedConstantFilename, x.str());
  2794. }
  2795. }
  2796. else
  2797. {
  2798. StringBuffer propValue;
  2799. folded->queryValue()->getStringValue(propValue);
  2800. instance.addAttribute(propName, propValue);
  2801. }
  2802. }
  2803. if (isDynamic)
  2804. {
  2805. unsigned len = propName.length();
  2806. if ((len > 4) && (memicmp(propName.str() + len-4, "name", 4) == 0))
  2807. propName.setLength(len-4);
  2808. propName.append("_dynamic");
  2809. instance.addAttributeBool(propName.str(), true);
  2810. }
  2811. }
  2812. }
  2813. void HqlCppTranslator::buildRefFilenameFunction(ActivityInstance & instance, BuildCtx & classctx, const char * name, IHqlExpression * expr)
  2814. {
  2815. IHqlExpression * table = queryPhysicalRootTable(expr);
  2816. IHqlExpression * filename = NULL;
  2817. if (table)
  2818. {
  2819. switch (table->getOperator())
  2820. {
  2821. case no_keyindex:
  2822. filename = table->queryChild(2);
  2823. break;
  2824. case no_newkeyindex:
  2825. filename = table->queryChild(3);
  2826. break;
  2827. case no_table:
  2828. filename = table->queryChild(0);
  2829. break;
  2830. }
  2831. }
  2832. buildFilenameFunction(instance, classctx, name, filename, hasDynamicFilename(table));
  2833. }
  2834. void HqlCppTranslator::buildConnectInputOutput(BuildCtx & ctx, ActivityInstance * instance, ABoundActivity * table, unsigned outputIndex, unsigned inputIndex, const char * label, bool nWay)
  2835. {
  2836. #ifdef _GATHER_USAGE_STATS
  2837. activityCounts[table->queryActivityKind()][instance->kind]++;
  2838. #endif
  2839. outputIndex = table->nextOutputCount();
  2840. logGraphEdge(instance->querySubgraphNode(), table->queryActivityId(), instance->activityId, outputIndex, inputIndex, label, nWay);
  2841. }
  2842. void HqlCppTranslator::buildInstancePrefix(ActivityInstance * instance)
  2843. {
  2844. instance->buildPrefix();
  2845. activeActivities.append(*instance->getBoundActivity());
  2846. }
  2847. void HqlCppTranslator::buildInstanceSuffix(ActivityInstance * instance)
  2848. {
  2849. instance->buildMetaMember();
  2850. instance->buildSuffix();
  2851. activeActivities.pop();
  2852. }
  2853. IHqlExpression * HqlCppTranslator::getRecordSize(IHqlExpression * dataset)
  2854. {
  2855. IHqlExpression * ds = dataset->queryNormalizedSelector();
  2856. return createValue(no_sizeof, makeIntType(4, false), ensureActiveRow(ds));
  2857. }
  2858. /* In parms: not linked */
  2859. void HqlCppTranslator::getRecordSize(BuildCtx & ctx, IHqlExpression * dataset, CHqlBoundExpr & bound)
  2860. {
  2861. OwnedHqlExpr size = getRecordSize(dataset);
  2862. buildExpr(ctx, size, bound);
  2863. }
  2864. unsigned HqlCppTranslator::getMaxRecordSize(IHqlExpression * record)
  2865. {
  2866. ColumnToOffsetMap * map = queryRecordOffsetMap(record);
  2867. if (!map)
  2868. return 0;
  2869. return map->getMaxSize();
  2870. }
  2871. unsigned HqlCppTranslator::getCsvMaxLength(IHqlExpression * csvAttr)
  2872. {
  2873. if (options.testIgnoreMaxLength)
  2874. return 1;
  2875. HqlExprArray attrs;
  2876. if (csvAttr)
  2877. {
  2878. ForEachChild(idx, csvAttr)
  2879. csvAttr->queryChild(idx)->unwindList(attrs, no_comma);
  2880. }
  2881. IHqlExpression * maxLength = queryProperty(maxLengthAtom, attrs);
  2882. if (maxLength)
  2883. return (unsigned)getIntValue(maxLength->queryChild(0), 0);
  2884. return MAX_CSV_RECORD_SIZE;
  2885. }
  2886. bool HqlCppTranslator::isFixedWidthDataset(IHqlExpression * dataset)
  2887. {
  2888. IHqlExpression * record = dataset->queryRecord();
  2889. return queryRecordOffsetMap(record)->isFixedWidth();
  2890. }
  2891. void HqlCppTranslator::createAccessFunctions(StringBuffer & helperFunc, BuildCtx & ctx, unsigned prio, const char * interfaceName, const char * object)
  2892. {
  2893. helperFunc.append("q").append(object);
  2894. StringBuffer s;
  2895. s.clear().append(interfaceName).append(" & ").append(helperFunc).append("() { ");
  2896. s.append("return ").append(object).append("; ");
  2897. s.append("}");
  2898. if (prio)
  2899. ctx.setNextPriority(prio);
  2900. ctx.addQuoted(s);
  2901. s.clear().append("extern ").append(interfaceName).append(" & ").append(helperFunc).append("();");
  2902. BuildCtx protoctx(*code, mainprototypesAtom);
  2903. protoctx.addQuoted(s);
  2904. }
  2905. void HqlCppTranslator::ensureRowAllocator(StringBuffer & allocatorName, BuildCtx & ctx, IHqlExpression * record, IHqlExpression * activityId)
  2906. {
  2907. OwnedHqlExpr marker = createAttribute(rowAllocatorMarkerAtom, LINK(record->queryBody()), LINK(activityId));
  2908. HqlExprAssociation * match = ctx.queryMatchExpr(marker);
  2909. if (match)
  2910. {
  2911. generateExprCpp(allocatorName, match->queryExpr());
  2912. return;
  2913. }
  2914. StringBuffer uid;
  2915. getUniqueId(uid.append("alloc"));
  2916. BuildCtx * declarectx = &ctx;
  2917. BuildCtx * callctx = &ctx;
  2918. getInvariantMemberContext(ctx, &declarectx, &callctx, true, false);
  2919. StringBuffer s;
  2920. s.append("Owned<IEngineRowAllocator> ").append(uid).append(";");
  2921. declarectx->addQuoted(s);
  2922. StringBuffer decl;
  2923. s.clear().append(uid).append(".setown(ctx->getRowAllocator(&");
  2924. buildMetaForRecord(s, record);
  2925. s.append(",");
  2926. generateExprCpp(s, activityId).append(")");
  2927. s.append(");");
  2928. callctx->addQuoted(s);
  2929. OwnedHqlExpr value = createVariable(uid.str(), makeBoolType());
  2930. declarectx->associateExpr(marker, value);
  2931. allocatorName.append(uid);
  2932. }
  2933. IHqlExpression * HqlCppTranslator::createRowAllocator(BuildCtx & ctx, IHqlExpression * record)
  2934. {
  2935. StringBuffer allocatorName;
  2936. OwnedHqlExpr curActivityId = getCurrentActivityId(ctx);
  2937. ensureRowAllocator(allocatorName, ctx, record, curActivityId);
  2938. return createQuoted(allocatorName, makeBoolType());
  2939. }
  2940. void HqlCppTranslator::buildMetaSerializerClass(BuildCtx & ctx, IHqlExpression * record, const char * serializerName)
  2941. {
  2942. StringBuffer s;
  2943. GlobalClassBuilder serializer(*this, ctx, serializerName, "COutputRowSerializer", "IOutputRowSerializer");
  2944. serializer.buildClass(RowMetaPrio);
  2945. serializer.setIncomplete(true);
  2946. BuildCtx & classctx = serializer.classctx;
  2947. s.clear().append("inline ").append(serializerName).append("(unsigned _activityId) : COutputRowSerializer(_activityId) {}");
  2948. classctx.addQuoted(s);
  2949. OwnedHqlExpr id = createVariable("activityId", LINK(sizetType));
  2950. serializer.classctx.associateExpr(queryActivityIdMarker(), id);
  2951. OwnedHqlExpr dataset = createDataset(no_null, LINK(record));
  2952. {
  2953. BuildCtx serializectx(serializer.startctx);
  2954. serializectx.addQuotedCompound("virtual void serialize(IRowSerializerTarget & out, const byte * self)");
  2955. OwnedHqlExpr helper = createVariable("out", makeBoolType());
  2956. BoundRow * row = bindTableCursor(serializectx, dataset, "self");
  2957. OwnedHqlExpr size = getRecordSize(row->querySelector());
  2958. CHqlBoundExpr boundSize;
  2959. buildExpr(serializectx, size, boundSize);
  2960. if (recordRequiresSerialization(record))
  2961. {
  2962. Owned<IReferenceSelector> selector = buildActiveRow(serializectx, row->querySelector());
  2963. selector->buildSerialize(serializectx, helper);
  2964. }
  2965. else
  2966. {
  2967. HqlExprArray args;
  2968. args.append(*LINK(helper));
  2969. args.append(*LINK(boundSize.expr));
  2970. args.append(*LINK(row->queryBound()));
  2971. OwnedHqlExpr call = bindTranslatedFunctionCall(serializerPutAtom, args);
  2972. serializectx.addExpr(call);
  2973. }
  2974. }
  2975. serializer.setIncomplete(false);
  2976. serializer.completeClass(RowMetaPrio);
  2977. }
  2978. void HqlCppTranslator::buildMetaDeserializerClass(BuildCtx & ctx, IHqlExpression * record, const char * deserializerName)
  2979. {
  2980. StringBuffer s;
  2981. GlobalClassBuilder deserializer(*this, ctx, deserializerName, "COutputRowDeserializer", "IOutputRowDeserializer");
  2982. deserializer.buildClass(RowMetaPrio);
  2983. deserializer.setIncomplete(true);
  2984. BuildCtx & classctx = deserializer.classctx;
  2985. s.clear().append("inline ").append(deserializerName).append("(unsigned _activityId) : COutputRowDeserializer(_activityId) {}");
  2986. classctx.addQuoted(s);
  2987. OwnedHqlExpr id = createVariable("activityId", LINK(sizetType));
  2988. deserializer.classctx.associateExpr(queryActivityIdMarker(), id);
  2989. OwnedHqlExpr dataset = createDataset(no_null, LINK(record));
  2990. {
  2991. BuildCtx deserializectx(deserializer.startctx);
  2992. deserializectx.addQuotedCompound("virtual size32_t deserialize(ARowBuilder & crSelf, IRowDeserializerSource & in)");
  2993. BoundRow * row = bindSelf(deserializectx, dataset, "crSelf");
  2994. ensureRowAllocated(deserializectx, "crSelf");
  2995. OwnedHqlExpr helper = createVariable("in", makeBoolType());
  2996. Owned<IReferenceSelector> selector = buildActiveRow(deserializectx, row->querySelector());
  2997. selector->buildDeserialize(deserializectx, helper);
  2998. buildReturnRecordSize(deserializectx, row);
  2999. }
  3000. deserializer.setIncomplete(false);
  3001. deserializer.completeClass(RowMetaPrio);
  3002. }
  3003. bool HqlCppTranslator::buildMetaPrefetcherClass(BuildCtx & ctx, IHqlExpression * record, const char * prefetcherName)
  3004. {
  3005. StringBuffer s;
  3006. GlobalClassBuilder prefetcher(*this, ctx, prefetcherName, "CSourceRowPrefetcher", NULL);
  3007. prefetcher.buildClass(RowMetaPrio);
  3008. prefetcher.setIncomplete(true);
  3009. BuildCtx & classctx = prefetcher.classctx;
  3010. s.clear().append("inline ").append(prefetcherName).append("(unsigned _activityId) : CSourceRowPrefetcher(_activityId) {}");
  3011. classctx.addQuoted(s);
  3012. OwnedHqlExpr id = createVariable("activityId", LINK(sizetType));
  3013. prefetcher.classctx.associateExpr(queryActivityIdMarker(), id);
  3014. OwnedHqlExpr dataset = createDataset(no_null, LINK(record));
  3015. bool ok;
  3016. {
  3017. BuildCtx prefetchctx(prefetcher.startctx);
  3018. IHqlStmt * stmt = prefetchctx.addQuotedCompound("virtual void readAhead(IRowDeserializerSource & in)");
  3019. OwnedHqlExpr helper = createVariable("in", makeBoolType());
  3020. ok = queryRecordOffsetMap(record)->buildReadAhead(*this, prefetchctx, helper);
  3021. }
  3022. if (ok)
  3023. {
  3024. prefetcher.setIncomplete(false);
  3025. prefetcher.completeClass(RowMetaPrio);
  3026. }
  3027. else
  3028. prefetcher.setIncluded(false);
  3029. return ok;
  3030. }
  3031. IHqlExpression * HqlCppTranslator::getRtlFieldKey(IHqlExpression * expr, IHqlExpression * rowRecord)
  3032. {
  3033. /*
  3034. Most field information is context independent - which make life much easier, there are a few exceptions though:
  3035. type_bitfield. The offset within the bitfield, and whether the bitfield is the last in the block depend on the other adjacent bitfields.
  3036. type_alien. Because it can refer to self in the parameters to the type definition it is dependent on the containing record
  3037. no_ifblock: Again because it references no_self, it is context dependent.
  3038. Theoretically with an inline record definition for a field it might be possible to make an ifblock dependent on something other than the most
  3039. immediate parent record, but it would be extremely pathological, and probably wouldn't work in lots of other ways.
  3040. */
  3041. bool contextDependent = false;
  3042. switch (expr->getOperator())
  3043. {
  3044. case no_field:
  3045. switch (expr->queryType()->getTypeCode())
  3046. {
  3047. case type_bitfield:
  3048. {
  3049. ColumnToOffsetMap * map = queryRecordOffsetMap(rowRecord);
  3050. AColumnInfo * root = map->queryRootColumn();
  3051. CBitfieldInfo * resolved = static_cast<CBitfieldInfo *>(root->lookupColumn(expr));
  3052. assertex(resolved);
  3053. unsigned offset = resolved->queryBitfieldOffset();
  3054. bool isLastBitfield = resolved->queryIsLastBitfield();
  3055. Linked<ITypeInfo> fieldType = expr->queryType();
  3056. fieldType.setown(makeAttributeModifier(LINK(fieldType), createAttribute(bitfieldOffsetAtom, getSizetConstant(offset))));
  3057. if (isLastBitfield)
  3058. fieldType.setown(makeAttributeModifier(LINK(fieldType), createAttribute(isLastBitfieldAtom)));
  3059. HqlExprArray args;
  3060. unwindChildren(args, expr);
  3061. return createField(expr->queryName(), LINK(fieldType), args);
  3062. }
  3063. break;
  3064. case type_alien:
  3065. //actually too strict - some alien data types are not context dependent.
  3066. contextDependent = true;
  3067. break;
  3068. }
  3069. break;
  3070. case no_ifblock:
  3071. contextDependent = true;
  3072. break;
  3073. }
  3074. if (contextDependent)
  3075. return createAttribute(rtlFieldKeyMarkerAtom, LINK(expr), LINK(rowRecord));
  3076. return LINK(expr);
  3077. }
  3078. unsigned HqlCppTranslator::buildRtlField(StringBuffer * instanceName, IHqlExpression * fieldKey)
  3079. {
  3080. BuildCtx declarectx(*code, declareAtom);
  3081. HqlExprAssociation * match = declarectx.queryMatchExpr(fieldKey);
  3082. if (match)
  3083. {
  3084. IHqlExpression * mapped = match->queryExpr();
  3085. if (instanceName)
  3086. mapped->queryChild(0)->toString(*instanceName);
  3087. return (unsigned)getIntValue(mapped->queryChild(1));
  3088. }
  3089. IHqlExpression * field = fieldKey;
  3090. IHqlExpression * rowRecord = NULL;
  3091. if (field->isAttribute())
  3092. {
  3093. field = fieldKey->queryChild(0);
  3094. rowRecord = fieldKey->queryChild(1);
  3095. }
  3096. StringBuffer name;
  3097. unsigned typeFlags = 0;
  3098. if (field->getOperator() == no_ifblock)
  3099. {
  3100. typeFlags = buildRtlIfBlockField(name, field, rowRecord);
  3101. }
  3102. else
  3103. {
  3104. Linked<ITypeInfo> fieldType = field->queryType();
  3105. switch (field->queryType()->getTypeCode())
  3106. {
  3107. case type_alien:
  3108. //MORE:::
  3109. break;
  3110. case type_row:
  3111. //Backward compatibility - should revisit
  3112. fieldType.set(fieldType->queryChildType());
  3113. break;
  3114. }
  3115. StringBuffer typeName;
  3116. typeFlags = buildRtlType(typeName, fieldType);
  3117. StringBuffer lowerName;
  3118. lowerName.append(field->queryName()).toLowerCase();
  3119. if (options.debugGeneratedCpp)
  3120. {
  3121. name.append("rf_");
  3122. convertToValidLabel(name, lowerName.str(), lowerName.length());
  3123. name.append("_").append(++nextFieldId);
  3124. }
  3125. else
  3126. name.append("rf").append(++nextFieldId);
  3127. StringBuffer xpathName, xpathItem;
  3128. switch (fieldType->getTypeCode())
  3129. {
  3130. case type_set:
  3131. extractXmlName(xpathName, &xpathItem, NULL, field, "Item", false);
  3132. break;
  3133. case type_table:
  3134. case type_groupedtable:
  3135. extractXmlName(xpathName, &xpathItem, NULL, field, "Row", false);
  3136. //Following should be in the type processing, and the type should include the information
  3137. if (field->hasProperty(sizeAtom) || field->hasProperty(countAtom))
  3138. typeFlags |= RFTMinvalidxml;
  3139. break;
  3140. default:
  3141. extractXmlName(xpathName, NULL, NULL, field, NULL, false);
  3142. break;
  3143. }
  3144. if (xpathName.length() && (xpathName.charAt(0) == '@'))
  3145. typeFlags |= RFTMhasxmlattr;
  3146. //Format of the xpath field is (nested-item 0x01 repeated-item)
  3147. StringBuffer xpathFull, xpathCppText;
  3148. xpathFull.append(xpathName);
  3149. if (xpathItem.length())
  3150. xpathFull.append(xpathCompoundSeparatorChar).append(xpathItem);
  3151. if (strcmp(lowerName, xpathFull) != 0)
  3152. appendStringAsQuotedCPP(xpathCppText, xpathFull.length(), xpathFull.str(), false);
  3153. else
  3154. xpathCppText.append("NULL");
  3155. StringBuffer definition;
  3156. definition.append("const RtlFieldStrInfo ").append(name).append("(\"").append(lowerName).append("\",").append(xpathCppText).append(",&").append(typeName).append(");");
  3157. BuildCtx fieldctx(declarectx);
  3158. fieldctx.setNextPriority(TypeInfoPrio);
  3159. fieldctx.addQuoted(definition);
  3160. name.insert(0, "&");
  3161. }
  3162. OwnedHqlExpr nameExpr = createVariable(name.str(), makeBoolType());
  3163. OwnedHqlExpr mapped = createAttribute(fieldAtom, LINK(nameExpr), getSizetConstant(typeFlags));
  3164. declarectx.associateExpr(fieldKey, mapped);
  3165. if (instanceName)
  3166. instanceName->append(name);
  3167. return typeFlags;
  3168. }
  3169. unsigned HqlCppTranslator::buildRtlIfBlockField(StringBuffer & instanceName, IHqlExpression * ifblock, IHqlExpression * rowRecord)
  3170. {
  3171. StringBuffer typeName, s;
  3172. BuildCtx declarectx(*code, declareAtom);
  3173. //First generate a pseudo type entry for an ifblock.
  3174. unsigned fieldType = type_ifblock|RFTMcontainsifblock;
  3175. {
  3176. unsigned length = 0;
  3177. StringBuffer childTypeName;
  3178. unsigned childType = buildRtlRecordFields(childTypeName, ifblock->queryChild(1), rowRecord);
  3179. fieldType |= (childType & (RFTMcontainsunknown|RFTMinvalidxml|RFTMhasxmlattr));
  3180. StringBuffer className;
  3181. typeName.append("ty").append(++nextTypeId);
  3182. className.append("tyc").append(nextFieldId);
  3183. //The ifblock needs a unique instance of the class to evaluate the test
  3184. BuildCtx fieldclassctx(declarectx);
  3185. fieldclassctx.setNextPriority(TypeInfoPrio);
  3186. fieldclassctx.addQuotedCompound(s.clear().append("struct ").append(className).append(" : public RtlIfBlockTypeInfo"), ";");
  3187. fieldclassctx.addQuoted(s.clear().append(className).append("() : RtlIfBlockTypeInfo(0x").appendf("%x", fieldType).append(",").append(0).append(",").append(childTypeName).append(") {}"));
  3188. OwnedHqlExpr anon = createDataset(no_anon, LINK(rowRecord));
  3189. BuildCtx condctx(fieldclassctx);
  3190. condctx.addQuotedCompound("virtual bool getCondition(const byte * self) const");
  3191. BoundRow * self = bindTableCursor(condctx, anon, "self");
  3192. OwnedHqlExpr cond = self->bindToRow(ifblock->queryChild(0), querySelfReference());
  3193. buildReturn(condctx, cond);
  3194. s.clear().append("const ").append(className).append(" ").append(typeName).append(";");
  3195. BuildCtx typectx(declarectx);
  3196. typectx.setNextPriority(TypeInfoPrio);
  3197. typectx.addQuoted(s);
  3198. }
  3199. StringBuffer name;
  3200. name.append("rf").append(++nextFieldId);
  3201. //Now generate a pseudo field for the ifblock
  3202. s.clear().append("const RtlFieldStrInfo ").append(name).append("(NULL, NULL,&").append(typeName).append(");");
  3203. BuildCtx fieldctx(declarectx);
  3204. fieldctx.setNextPriority(TypeInfoPrio);
  3205. fieldctx.addQuoted(s);
  3206. instanceName.append("&").append(name);
  3207. return fieldType;
  3208. }
  3209. unsigned HqlCppTranslator::expandRtlRecordFields(StringBuffer & fieldListText, IHqlExpression * record, IHqlExpression * rowRecord)
  3210. {
  3211. unsigned fieldType = 0;
  3212. ForEachChild(i, record)
  3213. {
  3214. IHqlExpression * cur = record->queryChild(i);
  3215. StringBuffer next;
  3216. unsigned childType = 0;
  3217. switch (cur->getOperator())
  3218. {
  3219. case no_field:
  3220. case no_ifblock:
  3221. {
  3222. OwnedHqlExpr fieldKey = getRtlFieldKey(cur, rowRecord);
  3223. childType = buildRtlField(&fieldListText, fieldKey);
  3224. fieldListText.append(",");
  3225. break;
  3226. }
  3227. case no_record:
  3228. childType = expandRtlRecordFields(fieldListText, cur, rowRecord);
  3229. break;
  3230. }
  3231. fieldType |= (childType & (RFTMcontainsunknown|RFTMinvalidxml|RFTMhasxmlattr));
  3232. }
  3233. return fieldType;
  3234. }
  3235. unsigned HqlCppTranslator::buildRtlRecordFields(StringBuffer & instanceName, IHqlExpression * record, IHqlExpression * rowRecord)
  3236. {
  3237. StringBuffer fieldListText;
  3238. unsigned fieldFlags = expandRtlRecordFields(fieldListText, record, rowRecord);
  3239. StringBuffer name;
  3240. name.append("tl").append(++nextTypeId);
  3241. StringBuffer s;
  3242. s.append("const RtlFieldInfo * const ").append(name).append("[] = { ").append(fieldListText).append(" 0 };");
  3243. BuildCtx listctx(*code, declareAtom);
  3244. listctx.setNextPriority(TypeInfoPrio);
  3245. listctx.addQuoted(s);
  3246. instanceName.append(name);
  3247. return fieldFlags;
  3248. }
  3249. unsigned HqlCppTranslator::getRtlFieldInfo(StringBuffer & fieldInfoName, IHqlExpression * field, IHqlExpression * rowRecord)
  3250. {
  3251. OwnedHqlExpr fieldKey = getRtlFieldKey(field, rowRecord);
  3252. return buildRtlField(&fieldInfoName, fieldKey);
  3253. }
  3254. unsigned HqlCppTranslator::buildRtlType(StringBuffer & instanceName, ITypeInfo * type)
  3255. {
  3256. assertex(type);
  3257. type_t tc = type->getTypeCode();
  3258. if (tc == type_record)
  3259. type = queryUnqualifiedType(type);
  3260. OwnedHqlExpr search = createVariable("t", LINK(type));
  3261. BuildCtx declarectx(*code, declareAtom);
  3262. HqlExprAssociation * match = declarectx.queryMatchExpr(search);
  3263. if (match)
  3264. {
  3265. IHqlExpression * value = match->queryExpr();
  3266. value->queryChild(0)->toString(instanceName);
  3267. return (unsigned)getIntValue(value->queryChild(1));
  3268. }
  3269. StringBuffer name, className, arguments;
  3270. if (options.debugGeneratedCpp)
  3271. {
  3272. StringBuffer ecl;
  3273. type->getECLType(ecl);
  3274. name.append("ty_");
  3275. convertToValidLabel(name, ecl.str(), ecl.length());
  3276. name.append("_").append(++nextTypeId);
  3277. }
  3278. else
  3279. name.append("ty").append(++nextTypeId);
  3280. unsigned fieldType= 0;
  3281. if (tc == type_alien)
  3282. {
  3283. ITypeInfo * physicalType = queryAlienType(type)->queryPhysicalType();
  3284. if (physicalType->getSize() != UNKNOWN_LENGTH)
  3285. {
  3286. //Don't use the generated class for xml generation since it will generate physical rather than logical
  3287. fieldType |= (RFTMalien|RFTMinvalidxml);
  3288. type = physicalType;
  3289. tc = type->getTypeCode();
  3290. }
  3291. else
  3292. {
  3293. fieldType |= RFTMunknownsize;
  3294. //can't work out the size of the field - to keep it as unknown for the moment.
  3295. //until the alien field type is supported
  3296. }
  3297. }
  3298. fieldType |= tc;
  3299. unsigned length = type->getSize();
  3300. if (length == UNKNOWN_LENGTH)
  3301. {
  3302. fieldType |= RFTMunknownsize;
  3303. length = 0;
  3304. }
  3305. unsigned childType = 0;
  3306. switch (tc)
  3307. {
  3308. case type_boolean:
  3309. className.append("RtlBoolTypeInfo");
  3310. break;
  3311. case type_real:
  3312. className.append("RtlRealTypeInfo");
  3313. break;
  3314. case type_date:
  3315. case type_enumerated:
  3316. case type_int:
  3317. className.append("RtlIntTypeInfo");
  3318. if (!type->isSigned())
  3319. fieldType |= RFTMunsigned;
  3320. break;
  3321. case type_swapint:
  3322. className.append("RtlSwapIntTypeInfo");
  3323. if (!type->isSigned())
  3324. fieldType |= RFTMunsigned;
  3325. break;
  3326. case type_packedint:
  3327. className.append("RtlPackedIntTypeInfo");
  3328. if (!type->isSigned())
  3329. fieldType |= RFTMunsigned;
  3330. break;
  3331. case type_decimal:
  3332. className.append("RtlDecimalTypeInfo");
  3333. if (!type->isSigned())
  3334. fieldType |= RFTMunsigned;
  3335. length = type->getDigits() | (type->getPrecision() << 16);
  3336. break;
  3337. case type_char:
  3338. className.append("RtlCharTypeInfo");
  3339. break;
  3340. case type_data:
  3341. className.append("RtlDataTypeInfo");
  3342. break;
  3343. case type_qstring:
  3344. className.append("RtlQStringTypeInfo");
  3345. length = type->getStringLen();
  3346. break;
  3347. case type_varstring:
  3348. className.append("RtlVarStringTypeInfo");
  3349. if (type->queryCharset() && type->queryCharset()->queryName()==ebcdicAtom)
  3350. fieldType |= RFTMebcdic;
  3351. length = type->getStringLen();
  3352. break;
  3353. case type_string:
  3354. className.append("RtlStringTypeInfo");
  3355. if (type->queryCharset() && type->queryCharset()->queryName()==ebcdicAtom)
  3356. fieldType |= RFTMebcdic;
  3357. break;
  3358. case type_bitfield:
  3359. {
  3360. className.append("RtlBitfieldTypeInfo");
  3361. unsigned size = type->getSize();
  3362. unsigned bitsize = type->getBitSize();
  3363. unsigned offset = (unsigned)getIntValue(queryPropertyChild(type, bitfieldOffsetAtom, 0),-1);
  3364. bool isLastBitfield = (queryProperty(type, isLastBitfieldAtom) != NULL);
  3365. if (isLastBitfield)
  3366. fieldType |= RFTMislastbitfield;
  3367. if (!type->isSigned())
  3368. fieldType |= RFTMunsigned;
  3369. length = size | (bitsize << 8) | (offset << 16);
  3370. break;
  3371. }
  3372. case type_record:
  3373. {
  3374. IHqlExpression * record = ::queryRecord(type);
  3375. className.append("RtlRecordTypeInfo");
  3376. arguments.append(",");
  3377. childType = buildRtlRecordFields(arguments, record, record);
  3378. // fieldType |= (childType & RFTMcontainsifblock);
  3379. length = getMaxRecordSize(record);
  3380. if (!isFixedRecordSize(record))
  3381. fieldType |= RFTMunknownsize;
  3382. break;
  3383. }
  3384. case type_row:
  3385. {
  3386. className.clear().append("RtlRowTypeInfo");
  3387. arguments.append(",&");
  3388. childType = buildRtlType(arguments, ::queryRecordType(type));
  3389. // fieldType |= (childType & RFTMcontainsifblock);
  3390. if (hasLinkCountedModifier(type))
  3391. fieldType |= RFTMlinkcounted;
  3392. break;
  3393. }
  3394. case type_table:
  3395. case type_groupedtable:
  3396. {
  3397. className.clear().append("RtlDatasetTypeInfo");
  3398. arguments.append(",&");
  3399. childType = buildRtlType(arguments, ::queryRecordType(type));
  3400. if (hasLinkCountedModifier(type))
  3401. fieldType |= RFTMlinkcounted;
  3402. break;
  3403. }
  3404. case type_set:
  3405. className.clear().append("RtlSetTypeInfo");
  3406. arguments.append(",&");
  3407. childType = buildRtlType(arguments, type->queryChildType());
  3408. break;
  3409. case type_unicode:
  3410. className.clear().append("RtlUnicodeTypeInfo");
  3411. arguments.append(", \"").append(type->queryLocale()).append("\"").toLowerCase();
  3412. length = type->getStringLen();
  3413. break;
  3414. case type_varunicode:
  3415. className.clear().append("RtlVarUnicodeTypeInfo");
  3416. arguments.append(", \"").append(type->queryLocale()).append("\"").toLowerCase();
  3417. length = type->getStringLen();
  3418. break;
  3419. case type_utf8:
  3420. className.clear().append("RtlUtf8TypeInfo");
  3421. arguments.append(", \"").append(type->queryLocale()).append("\"").toLowerCase();
  3422. length = type->getStringLen();
  3423. break;
  3424. case type_blob:
  3425. case type_pointer:
  3426. case type_class:
  3427. case type_array:
  3428. case type_void:
  3429. case type_alien:
  3430. case type_none:
  3431. case type_any:
  3432. case type_pattern:
  3433. case type_rule:
  3434. case type_token:
  3435. case type_feature:
  3436. case type_event:
  3437. case type_null:
  3438. case type_scope:
  3439. case type_transform:
  3440. default:
  3441. className.append("RtlUnimplementedTypeInfo");
  3442. fieldType |= (RFTMcontainsunknown|RFTMinvalidxml);
  3443. break;
  3444. }
  3445. fieldType |= (childType & (RFTMcontainsunknown|RFTMinvalidxml|RFTMhasxmlattr));
  3446. StringBuffer definition;
  3447. definition.append("const ").append(className).append(" ").append(name).append("(0x").appendf("%x", fieldType).append(",").append(length).append(arguments).append(");");
  3448. BuildCtx typectx(declarectx);
  3449. typectx.setNextPriority(TypeInfoPrio);
  3450. typectx.addQuoted(definition);
  3451. OwnedHqlExpr nameExpr = createVariable(name.str(), makeVoidType());
  3452. OwnedHqlExpr mapped = createAttribute(fieldAtom, LINK(nameExpr), getSizetConstant(fieldType));
  3453. declarectx.associateExpr(search, mapped);
  3454. instanceName.append(name);
  3455. return fieldType;
  3456. }
  3457. void HqlCppTranslator::buildMetaInfo(MetaInstance & instance)
  3458. {
  3459. if (options.spanMultipleCpp)
  3460. {
  3461. StringBuffer queryFunctionName;
  3462. queryFunctionName.append("q").append(instance.instanceName).append("()");
  3463. instance.instanceObject.set(queryFunctionName);
  3464. }
  3465. BuildCtx declarectx(*code, declareAtom);
  3466. IHqlExpression * dataset = instance.dataset;
  3467. OwnedHqlExpr search = instance.getMetaUniqueKey();
  3468. // stop duplicate classes being generated.
  3469. // MORE: If this ever includes sorting/grouping, the dependence on a record will need to be revised
  3470. HqlExprAssociation * match = declarectx.queryMatchExpr(search);
  3471. if (match)
  3472. return;
  3473. bool savedContextAvailable = contextAvailable;
  3474. contextAvailable = false;
  3475. metas.append(*search.getLink());
  3476. StringBuffer s;
  3477. StringBuffer serializerName, deserializerName, prefetcherName;
  3478. StringBuffer endText;
  3479. endText.append(" ").append(instance.instanceName).append(";");
  3480. BuildCtx metactx(declarectx);
  3481. ITypeInfo * type = dataset->queryType();
  3482. IHqlExpression * record = dataset->queryRecord();
  3483. ColumnToOffsetMap * map = queryRecordOffsetMap(record);
  3484. unsigned flags = MDFhasserialize; // we always generate a serialize since
  3485. bool useTypeForXML = false;
  3486. if (type && type->getTypeCode() == type_groupedtable)
  3487. flags |= MDFgrouped;
  3488. if (map)
  3489. flags |= MDFhasxml;
  3490. if (record)
  3491. {
  3492. if (recordRequiresDestructor(record))
  3493. flags |= MDFneeddestruct;
  3494. if (recordRequiresSerialization(record))
  3495. flags |= MDFneedserialize;
  3496. if (maxRecordSizeUsesDefault(record))
  3497. flags |= MDFunknownmaxlength;
  3498. useTypeForXML = true;
  3499. }
  3500. if (type && type->getTypeCode() == type_groupedtable)
  3501. {
  3502. OwnedHqlExpr ungrouped = createDataset(no_group, LINK(dataset));
  3503. MetaInstance ungroupedMeta(*this, ungrouped);
  3504. buildMetaInfo(ungroupedMeta);
  3505. s.append("struct ").append(instance.metaName).append(" : public ").append(ungroupedMeta.metaName);
  3506. metactx.setNextPriority(RowMetaPrio);
  3507. metactx.addQuotedCompound(s, endText.str());
  3508. doBuildUnsignedFunction(metactx, "getMetaFlags", flags);
  3509. }
  3510. else
  3511. {
  3512. //Serialization classes need to be generated for all meta information - because they may be called by parent row classes
  3513. //however, the CFixedOutputMetaData base class contains a default implementation - reducing the required code.
  3514. if (map && (!map->isFixedWidth() || (flags & MDFneedserialize)))
  3515. {
  3516. //Base class provides a default variable width implementation
  3517. if (flags & MDFneedserialize)
  3518. {
  3519. serializerName.append("s").append(instance.metaName);
  3520. buildMetaSerializerClass(declarectx, record, serializerName.str());
  3521. }
  3522. //still generate a deserialize for the variable width case because it offers protection
  3523. //against accessing out of bounds data
  3524. deserializerName.append("d").append(instance.metaName);
  3525. buildMetaDeserializerClass(declarectx, record, deserializerName.str());
  3526. //The base class implements prefetch using the serialized meta so no need to generate...
  3527. if (!(flags & MDFneedserialize))
  3528. {
  3529. prefetcherName.append("p").append(instance.metaName);
  3530. if (!buildMetaPrefetcherClass(declarectx, record, prefetcherName))
  3531. prefetcherName.clear();
  3532. }
  3533. }
  3534. s.append("struct ").append(instance.metaName).append(" : public ");
  3535. if (!map)
  3536. s.append("CActionOutputMetaData");
  3537. else if (map->isFixedWidth())
  3538. s.append("CFixedOutputMetaData");
  3539. else
  3540. s.append("CVariableOutputMetaData");
  3541. metactx.setNextPriority(RowMetaPrio);
  3542. IHqlStmt * metaclass = metactx.addQuotedCompound(s, endText.str());
  3543. metaclass->setIncomplete(true);
  3544. if (map)
  3545. {
  3546. if (map->isFixedWidth())
  3547. {
  3548. unsigned fixedSize = map->getFixedRecordSize();
  3549. s.clear().append("inline ").append(instance.metaName).append("() : CFixedOutputMetaData(").append(fixedSize).append(") {}");
  3550. metactx.addQuoted(s);
  3551. }
  3552. else
  3553. {
  3554. unsigned minSize = getMinRecordSize(record);
  3555. unsigned maxLength = map->getMaxSize();
  3556. assertex(maxLength >= minSize);
  3557. #ifdef _DEBUG
  3558. //Paranoia check to ensure the two methods agree.
  3559. unsigned calcMinSize = map->getTotalMinimumSize();
  3560. assertex(minSize == calcMinSize);
  3561. #endif
  3562. //These use a CVariableOutputMetaData base class instead, and trade storage for number of virtuals
  3563. s.clear().append("inline ").append(instance.metaName).append("() : CVariableOutputMetaData(").append(minSize).append(") {}");
  3564. metactx.addQuoted(s);
  3565. if (options.testIgnoreMaxLength)
  3566. maxLength = minSize;
  3567. BuildCtx getctx(metactx);
  3568. s.clear().append("virtual size32_t getRecordSize(const void * data)");
  3569. getctx.addQuotedCompound(s);
  3570. s.clear().append("if (!data) return ").append(maxLength).append(";");
  3571. getctx.addQuoted(s.str());
  3572. getctx.addQuoted("const unsigned char * left = (const unsigned char *)data;");
  3573. LinkedHqlExpr selfDs = dataset;
  3574. if (!selfDs->isDataset() || !selfDs->isDatarow())
  3575. selfDs.setown(createDataset(no_null, LINK(dataset->queryRecord())));
  3576. BoundRow * selfRow = bindTableCursorOrRow(getctx, selfDs, "left");
  3577. OwnedHqlExpr size = getRecordSize(selfRow->querySelector());
  3578. buildReturn(getctx, size);
  3579. }
  3580. assertex(!(type && type->getTypeCode() == type_groupedtable));
  3581. StringBuffer typeName;
  3582. unsigned recordTypeFlags = buildRtlType(typeName, record->queryType());
  3583. s.clear().append("virtual const RtlTypeInfo * queryTypeInfo() const { return &").append(typeName).append("; }");
  3584. metactx.addQuoted(s);
  3585. if (record->numChildren() != 0)
  3586. {
  3587. OwnedHqlExpr anon = createDataset(no_anon, LINK(dataset->queryRecord()));
  3588. if (!useTypeForXML || (recordTypeFlags & (RFTMinvalidxml|RFTMhasxmlattr)))
  3589. buildXmlSerialize(metactx, anon, "toXML", true);
  3590. }
  3591. if (record)
  3592. generateMetaRecordSerialize(metactx, record, serializerName.str(), deserializerName.str(), prefetcherName.str());
  3593. if (flags != (MDFhasserialize|MDFhasxml))
  3594. doBuildUnsignedFunction(metactx, "getMetaFlags", flags);
  3595. if (flags & MDFneedserialize)
  3596. {
  3597. OwnedHqlExpr serializedRecord = getSerializedForm(record);
  3598. OwnedHqlExpr serializedDataset = createDataset(no_anon, LINK(serializedRecord));
  3599. MetaInstance serializedMeta(*this, serializedDataset);
  3600. buildMetaInfo(serializedMeta);
  3601. StringBuffer s;
  3602. s.append("virtual IOutputMetaData * querySerializedMeta() { return &").append(serializedMeta.queryInstanceObject()).append("; }");
  3603. metactx.addQuoted(s);
  3604. }
  3605. }
  3606. metaclass->setIncomplete(false);
  3607. }
  3608. s.clear().append("extern \"C\" ECL_API IOutputMetaData * ").append(instance.metaFactoryName).append("() { ");
  3609. s.append(instance.instanceName).append(".Link(); ");
  3610. s.append("return &").append(instance.instanceName).append("; ");
  3611. s.append("}");
  3612. declarectx.setNextPriority(RowMetaPrio);
  3613. declarectx.addQuoted(s);
  3614. if (options.spanMultipleCpp)
  3615. {
  3616. StringBuffer temp;
  3617. createAccessFunctions(temp, declarectx, RowMetaPrio, "IOutputMetaData", instance.instanceName);
  3618. }
  3619. OwnedHqlExpr temp = createVariable(instance.metaName, makeVoidType());
  3620. declarectx.associateExpr(search, temp);
  3621. contextAvailable = savedContextAvailable;
  3622. }
  3623. class MetaMemberCallback
  3624. {
  3625. public:
  3626. MetaMemberCallback(HqlCppTranslator & _translator) : translator(_translator) {}
  3627. void callChildFunction(BuildCtx & ctx, IHqlExpression * selected)
  3628. {
  3629. MetaInstance childMeta(translator, selected);
  3630. translator.buildMetaInfo(childMeta);
  3631. callChildFunction(ctx, selected, childMeta);
  3632. }
  3633. void walkRecord(BuildCtx & ctx, IHqlExpression * selector, IHqlExpression * record)
  3634. {
  3635. ForEachChild(i, record)
  3636. {
  3637. IHqlExpression * cur = record->queryChild(i);
  3638. switch (cur->getOperator())
  3639. {
  3640. case no_record:
  3641. walkRecord(ctx, selector, cur);
  3642. break;
  3643. case no_ifblock:
  3644. {
  3645. OwnedHqlExpr cond = replaceSelector(cur->queryChild(0), querySelfReference(), selector);
  3646. BuildCtx condctx(ctx);
  3647. translator.buildFilter(condctx, cond);
  3648. walkRecord(condctx, selector, cur->queryChild(1));
  3649. break;
  3650. }
  3651. case no_field:
  3652. {
  3653. ITypeInfo * type = cur->queryType();
  3654. switch (type->getTypeCode())
  3655. {
  3656. case type_alien:
  3657. //MORE: Allow for alien data types to have destructors.
  3658. break;
  3659. case type_row:
  3660. {
  3661. IHqlExpression * record = cur->queryRecord();
  3662. if (recordRequiresDestructor(record))
  3663. {
  3664. OwnedHqlExpr selected = createSelectExpr(LINK(selector), LINK(cur));
  3665. callChildFunction(ctx, selected);
  3666. }
  3667. break;
  3668. }
  3669. case type_table:
  3670. case type_groupedtable:
  3671. {
  3672. OwnedHqlExpr selected = createSelectExpr(LINK(selector), LINK(cur));
  3673. IHqlExpression * record = cur->queryRecord();
  3674. if (cur->hasProperty(_linkCounted_Atom))
  3675. {
  3676. //releaseRowset(ctx, count, rowset)
  3677. MetaInstance childMeta(translator, selected);
  3678. translator.buildMetaInfo(childMeta);
  3679. processRowset(ctx, selected, childMeta);
  3680. }
  3681. else if (recordRequiresDestructor(record))
  3682. {
  3683. BuildCtx iterctx(ctx);
  3684. translator.buildDatasetIterate(iterctx, selected, false);
  3685. OwnedHqlExpr active = createRow(no_activerow, LINK(selected));
  3686. callChildFunction(iterctx, active);
  3687. }
  3688. break;
  3689. }
  3690. }
  3691. break;
  3692. }
  3693. }
  3694. }
  3695. }
  3696. protected:
  3697. virtual void callChildFunction(BuildCtx & ctx, IHqlExpression * selected, MetaInstance & childMeta) = 0;
  3698. virtual void processRowset(BuildCtx & ctx, IHqlExpression * selected, MetaInstance & childMeta) = 0;
  3699. protected:
  3700. HqlCppTranslator & translator;
  3701. };
  3702. class MetaDestructCallback : public MetaMemberCallback
  3703. {
  3704. public:
  3705. MetaDestructCallback(HqlCppTranslator & _translator) : MetaMemberCallback(_translator) {}
  3706. protected:
  3707. virtual void callChildFunction(BuildCtx & ctx, IHqlExpression * selected, MetaInstance & childMeta)
  3708. {
  3709. HqlExprArray args;
  3710. args.append(*createQuoted(childMeta.queryInstanceObject(), makeBoolType()));
  3711. args.append(*LINK(selected));
  3712. translator.buildFunctionCall(ctx, destructMetaMemberAtom, args);
  3713. }
  3714. virtual void processRowset(BuildCtx & ctx, IHqlExpression * selected, MetaInstance & childMeta)
  3715. {
  3716. HqlExprArray args;
  3717. args.append(*LINK(selected));
  3718. translator.buildFunctionCall(ctx, releaseRowsetAtom, args);
  3719. }
  3720. };
  3721. void HqlCppTranslator::doGenerateMetaDestruct(BuildCtx & ctx, IHqlExpression * selector, IHqlExpression * record)
  3722. {
  3723. MetaDestructCallback builder(*this);
  3724. builder.walkRecord(ctx, selector, record);
  3725. }
  3726. class MetaWalkIndirectCallback : public MetaMemberCallback
  3727. {
  3728. public:
  3729. MetaWalkIndirectCallback(HqlCppTranslator & _translator, IHqlExpression * _visitor)
  3730. : MetaMemberCallback(_translator), visitor(_visitor) {}
  3731. protected:
  3732. virtual void callChildFunction(BuildCtx & ctx, IHqlExpression * selected, MetaInstance & childMeta)
  3733. {
  3734. HqlExprArray args;
  3735. args.append(*createQuoted(childMeta.queryInstanceObject(), makeBoolType()));
  3736. args.append(*LINK(selected));
  3737. args.append(*LINK(visitor));
  3738. translator.buildFunctionCall(ctx, walkIndirectMetaMemberAtom, args);
  3739. }
  3740. virtual void processRowset(BuildCtx & ctx, IHqlExpression * selected, MetaInstance & childMeta)
  3741. {
  3742. HqlExprArray args;
  3743. args.append(*LINK(visitor));
  3744. args.append(*LINK(selected));
  3745. translator.buildFunctionCall(ctx, IIndirectMemberVisitor_visitRowsetAtom, args);
  3746. }
  3747. protected:
  3748. LinkedHqlExpr visitor;
  3749. };
  3750. void HqlCppTranslator::generateMetaRecordSerialize(BuildCtx & ctx, IHqlExpression * record, const char * serializerName, const char * deserializerName, const char * prefetcherName)
  3751. {
  3752. OwnedHqlExpr dataset = createDataset(no_null, LINK(record));
  3753. if (recordRequiresDestructor(record))
  3754. {
  3755. BuildCtx destructctx(ctx);
  3756. destructctx.addQuotedCompound("virtual void destruct(byte * self)");
  3757. bindTableCursor(destructctx, dataset, "self");
  3758. MetaDestructCallback builder(*this);
  3759. builder.walkRecord(destructctx, dataset, record);
  3760. }
  3761. if (recordRequiresDestructor(record))
  3762. {
  3763. BuildCtx walkctx(ctx);
  3764. OwnedHqlExpr visitor = createVariable("visitor", makeBoolType()); // makeClassType("IIndirectMemberVisitor");
  3765. walkctx.addQuotedCompound("virtual void walkIndirectMembers(const byte * self, IIndirectMemberVisitor & visitor)");
  3766. bindTableCursor(walkctx, dataset, "self");
  3767. MetaWalkIndirectCallback builder(*this, visitor);
  3768. builder.walkRecord(walkctx, dataset, record);
  3769. }
  3770. if (serializerName && *serializerName)
  3771. {
  3772. BuildCtx serializectx(ctx);
  3773. serializectx.addQuotedCompound("virtual IOutputRowSerializer * createRowSerializer(ICodeContext * ctx, unsigned activityId)");
  3774. StringBuffer s;
  3775. s.append("return cr").append(serializerName).append("(ctx, activityId);");
  3776. serializectx.addQuoted(s);
  3777. }
  3778. if (deserializerName && *deserializerName)
  3779. {
  3780. BuildCtx deserializectx(ctx);
  3781. deserializectx.addQuotedCompound("virtual IOutputRowDeserializer * createRowDeserializer(ICodeContext * ctx, unsigned activityId)");
  3782. StringBuffer s;
  3783. s.append("return cr").append(deserializerName).append("(ctx, activityId);");
  3784. deserializectx.addQuoted(s);
  3785. }
  3786. if (prefetcherName && *prefetcherName)
  3787. {
  3788. BuildCtx deserializectx(ctx);
  3789. deserializectx.addQuotedCompound("virtual CSourceRowPrefetcher * createRawRowPrefetcher(unsigned activityId)");
  3790. StringBuffer s;
  3791. s.append("return new ").append(prefetcherName).append("(activityId);");
  3792. deserializectx.addQuoted(s);
  3793. }
  3794. }
  3795. IHqlExpression * HqlCppTranslator::buildMetaParameter(IHqlExpression * arg)
  3796. {
  3797. OwnedHqlExpr dataset = createDataset(no_anon, LINK(arg->queryRecord()));
  3798. MetaInstance meta(*this, dataset);
  3799. buildMetaInfo(meta);
  3800. return createQuoted(meta.queryInstanceObject(), makeBoolType());
  3801. }
  3802. void HqlCppTranslator::buildMetaMember(BuildCtx & ctx, IHqlExpression * datasetOrRecord, const char * name)
  3803. {
  3804. LinkedHqlExpr dataset = datasetOrRecord;
  3805. if (datasetOrRecord->getOperator() == no_record)
  3806. dataset.setown(createDataset(no_anon, LINK(datasetOrRecord)));
  3807. MetaInstance meta(*this, dataset);
  3808. StringBuffer s;
  3809. buildMetaInfo(meta);
  3810. s.append("virtual IOutputMetaData * ").append(name).append("() { return &").append(meta.queryInstanceObject()).append("; }");
  3811. ctx.addQuoted(s);
  3812. }
  3813. void HqlCppTranslator::buildMetaForRecord(StringBuffer & name, IHqlExpression * record)
  3814. {
  3815. OwnedHqlExpr dataset = createDataset(no_anon, LINK(record));
  3816. MetaInstance meta(*this, dataset);
  3817. buildMetaInfo(meta);
  3818. name.append(meta.queryInstanceObject());
  3819. }
  3820. void HqlCppTranslator::buildMetaForSerializedRecord(StringBuffer & name, IHqlExpression * record, bool isGrouped)
  3821. {
  3822. if (isGrouped)
  3823. {
  3824. HqlExprArray args;
  3825. unwindChildren(args, record);
  3826. args.append(*createField(__eogAtom, makeBoolType(), NULL, NULL));
  3827. OwnedHqlExpr groupedRecord = record->clone(args);
  3828. buildMetaForRecord(name, groupedRecord);
  3829. }
  3830. else
  3831. buildMetaForRecord(name, record);
  3832. }
  3833. void HqlCppTranslator::ensureRowSerializer(StringBuffer & serializerName, BuildCtx & ctx, IHqlExpression * record, _ATOM kind)
  3834. {
  3835. OwnedHqlExpr marker = createAttribute(serializerInstanceMarkerAtom, LINK(record->queryBody()), createAttribute(kind));
  3836. HqlExprAssociation * match = ctx.queryMatchExpr(marker);
  3837. if (match)
  3838. {
  3839. generateExprCpp(serializerName, match->queryExpr());
  3840. return;
  3841. }
  3842. StringBuffer uid;
  3843. getUniqueId(uid.append("ser"));
  3844. BuildCtx * declarectx = &ctx;
  3845. BuildCtx * callctx = &ctx;
  3846. getInvariantMemberContext(ctx, &declarectx, &callctx, true, false);
  3847. StringBuffer s;
  3848. if (kind == serializerAtom)
  3849. s.append("Owned<IOutputRowSerializer> ").append(uid).append(";");
  3850. else
  3851. s.append("Owned<IOutputRowDeserializer> ").append(uid).append(";");
  3852. declarectx->addQuoted(s);
  3853. OwnedHqlExpr ds = createDataset(no_anon, LINK(record));
  3854. MetaInstance meta(*this, ds);
  3855. buildMetaInfo(meta);
  3856. s.clear().append(uid).append(".setown(").append(meta.queryInstanceObject());
  3857. if (kind == serializerAtom)
  3858. s.append(".createRowSerializer");
  3859. else
  3860. s.append(".createRowDeserializer");
  3861. s.append("(ctx, ");
  3862. OwnedHqlExpr activityId = getCurrentActivityId(ctx);
  3863. generateExprCpp(s, activityId);
  3864. s.append("));");
  3865. callctx->addQuoted(s);
  3866. OwnedHqlExpr value = createVariable(uid.str(), makeBoolType());
  3867. declarectx->associateExpr(marker, value);
  3868. serializerName.append(uid);
  3869. }
  3870. void HqlCppTranslator::ensureRowPrefetcher(StringBuffer & prefetcherName, BuildCtx & ctx, IHqlExpression * record)
  3871. {
  3872. OwnedHqlExpr marker = createAttribute(prefetcherInstanceMarkerAtom, LINK(record->queryBody()));
  3873. HqlExprAssociation * match = ctx.queryMatchExpr(marker);
  3874. if (match)
  3875. {
  3876. generateExprCpp(prefetcherName, match->queryExpr());
  3877. return;
  3878. }
  3879. StringBuffer uid;
  3880. getUniqueId(uid.append("pf"));
  3881. BuildCtx * declarectx = &ctx;
  3882. BuildCtx * callctx = &ctx;
  3883. getInvariantMemberContext(ctx, &declarectx, &callctx, true, false);
  3884. StringBuffer s;
  3885. s.append("Owned<ISourceRowPrefetcher> ").append(uid).append(";");
  3886. declarectx->addQuoted(s);
  3887. OwnedHqlExpr ds = createDataset(no_anon, LINK(record));
  3888. MetaInstance meta(*this, ds);
  3889. buildMetaInfo(meta);
  3890. s.clear().append(uid).append(".setown(").append(meta.queryInstanceObject());
  3891. s.append(".createRowPrefetcher(ctx, ");
  3892. OwnedHqlExpr activityId = getCurrentActivityId(ctx);
  3893. generateExprCpp(s, activityId);
  3894. s.append("));");
  3895. callctx->addQuoted(s);
  3896. OwnedHqlExpr value = createVariable(uid.str(), makeBoolType());
  3897. declarectx->associateExpr(marker, value);
  3898. prefetcherName.append(uid);
  3899. }
  3900. IHqlExpression * HqlCppTranslator::createRowSerializer(BuildCtx & ctx, IHqlExpression * record, _ATOM kind)
  3901. {
  3902. StringBuffer serializerName;
  3903. ensureRowSerializer(serializerName, ctx, record, kind);
  3904. return createQuoted(serializerName.str(), makeBoolType());
  3905. }
  3906. IHqlExpression * HqlCppTranslator::createResultName(IHqlExpression * name, bool expandLogicalName)
  3907. {
  3908. IHqlExpression * resultName = ::createResultName(name);
  3909. if (!expandLogicalName)
  3910. return resultName;
  3911. HqlExprArray args;
  3912. args.append(*resultName);
  3913. return bindFunctionCall(getExpandLogicalNameAtom, args);
  3914. }
  3915. bool HqlCppTranslator::registerGlobalUsage(IHqlExpression * filename)
  3916. {
  3917. bool matched = false;
  3918. ForEachItemIn(i, globalFiles)
  3919. {
  3920. if (globalFiles.item(i).checkMatch(filename))
  3921. matched = true;
  3922. }
  3923. return matched;
  3924. }
  3925. //---------------------------------------------------------------------------
  3926. IHqlExpression * HqlCppTranslator::convertBetweenCountAndSize(const CHqlBoundExpr & bound, bool getCount)
  3927. {
  3928. ITypeInfo * type = bound.expr->queryType();
  3929. if (getCount)
  3930. {
  3931. if (getIntValue(bound.length, 1) == 0)
  3932. return getSizetConstant(0);
  3933. }
  3934. else
  3935. {
  3936. if (getIntValue(bound.count, 1) == 0)
  3937. return getSizetConstant(0);
  3938. }
  3939. OwnedHqlExpr record;
  3940. switch (type->getTypeCode())
  3941. {
  3942. case type_table:
  3943. case type_groupedtable:
  3944. record.set(bound.expr->queryRecord());
  3945. break;
  3946. case type_set:
  3947. case type_array:
  3948. {
  3949. ITypeInfo * elementType = type->queryChildType();
  3950. HqlExprArray fields;
  3951. fields.append(*createField(valueAtom, LINK(elementType), NULL));
  3952. record.setown(createRecord(fields));
  3953. break;
  3954. }
  3955. default:
  3956. UNIMPLEMENTED;
  3957. }
  3958. ColumnToOffsetMap * map = queryRecordOffsetMap(record);
  3959. if (map->isFixedWidth())
  3960. {
  3961. unsigned fixedSize = map->getFixedRecordSize();
  3962. if (fixedSize == 0)
  3963. throwError(HQLERR_ZeroLengthIllegal);
  3964. if (type->getTypeCode() == type_groupedtable)
  3965. fixedSize++;
  3966. if (getCount)
  3967. {
  3968. if (fixedSize == 1)
  3969. return LINK(bound.length);
  3970. IValue * value = bound.length->queryValue();
  3971. if (value)
  3972. return getSizetConstant((unsigned)value->getIntValue()/fixedSize);
  3973. return createValue(no_div, LINK(sizetType), LINK(bound.length), getSizetConstant(fixedSize));
  3974. }
  3975. else
  3976. {
  3977. if (fixedSize == 1)
  3978. return LINK(bound.count);
  3979. IValue * value = bound.count->queryValue();
  3980. if (value)
  3981. return getSizetConstant((unsigned)value->getIntValue() * fixedSize);
  3982. return createValue(no_mul, LINK(sizetType), LINK(bound.count), getSizetConstant(fixedSize));
  3983. }
  3984. }
  3985. StringBuffer metaInstanceName, s;
  3986. buildMetaForSerializedRecord(metaInstanceName, record, (type->getTypeCode() == type_groupedtable));
  3987. HqlExprArray args;
  3988. _ATOM func;
  3989. if (getCount)
  3990. {
  3991. args.append(*getBoundSize(bound));
  3992. args.append(*LINK(bound.expr));
  3993. func = countRowsAtom;
  3994. }
  3995. else
  3996. {
  3997. args.append(*LINK(bound.count));
  3998. args.append(*LINK(bound.expr));
  3999. func = countToSizeAtom;
  4000. }
  4001. args.append(*createQuoted(s.clear().append("&").append(metaInstanceName), makeBoolType()));
  4002. return bindTranslatedFunctionCall(func, args);
  4003. }
  4004. //---------------------------------------------------------------------------
  4005. void HqlCppTranslator::noteResultDefined(BuildCtx & ctx, ActivityInstance * activityInstance, IHqlExpression * seq, IHqlExpression * name, bool alwaysExecuted)
  4006. {
  4007. unsigned graph = curGraphSequence();
  4008. assertex(graph);
  4009. SubGraphInfo * activeSubgraph = queryActiveSubGraph(ctx);
  4010. assertex(activeSubgraph);
  4011. if (isInternalSeq(seq))
  4012. {
  4013. internalResults.append(* new InternalResultTracker(name, activeSubgraph->tree, graph, activityInstance));
  4014. }
  4015. else if (alwaysExecuted)
  4016. {
  4017. assertex(activeSubgraph->tree->hasProp("att[@name=\"rootGraph\"]"));
  4018. }
  4019. }
  4020. void HqlCppTranslator::noteResultAccessed(BuildCtx & ctx, IHqlExpression * seq, IHqlExpression * name)
  4021. {
  4022. if (isInternalSeq(seq))
  4023. {
  4024. unsigned graph = curGraphSequence();
  4025. ForEachItemIn(i, internalResults)
  4026. {
  4027. if (internalResults.item(i).noteUse(name, graph))
  4028. break;
  4029. }
  4030. }
  4031. }
  4032. void HqlCppTranslator::buildGetResultInfo(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr * boundTarget, const CHqlBoundTarget * targetAssign)
  4033. {
  4034. IHqlExpression * seq = queryPropertyChild(expr, sequenceAtom, 0);
  4035. IHqlExpression * name = queryPropertyChild(expr, namedAtom, 0);
  4036. if (!name)
  4037. name = queryPropertyChild(expr, nameAtom, 0);
  4038. noteResultAccessed(ctx, seq, name);
  4039. if (insideLibrary())
  4040. {
  4041. SCMStringBuffer libraryName;
  4042. getOutputLibraryName(libraryName, wu());
  4043. StringBuffer storedName;
  4044. getStoredDescription(storedName, seq, name, true);
  4045. throwError2(HQLERR_LibraryNoWorkunitRead, libraryName.str(), storedName.str());
  4046. }
  4047. __int64 seqValue = seq->queryValue()->getIntValue();
  4048. assertex(!expr->hasProperty(internalAtom) && !expr->hasProperty(_internal_Atom));
  4049. bool expandLogical = (seqValue == ResultSequencePersist) && !expr->hasProperty(_internal_Atom);
  4050. HqlExprArray args;
  4051. args.append(*createResultName(name, expandLogical));
  4052. args.append(*LINK(seq));
  4053. _ATOM func;
  4054. ITypeInfo * type = expr->queryType();
  4055. type_t ttc = type->getTypeCode();
  4056. OwnedITypeInfo overrideType;
  4057. switch(ttc)
  4058. {
  4059. case type_int: func = getResultIntAtom; break;
  4060. case type_swapint: func = getResultIntAtom; break;
  4061. case type_boolean: func = getResultBoolAtom; break;
  4062. case type_data: func = getResultDataAtom; break;
  4063. case type_table:
  4064. case type_groupedtable:
  4065. case type_set:
  4066. //MORE: type_row...
  4067. {
  4068. OwnedHqlExpr record;
  4069. bool ensureSerialized = true;
  4070. if (ttc == type_table || ttc == type_groupedtable)
  4071. {
  4072. overrideType.set(type);
  4073. record.set(::queryRecord(type));
  4074. //NB: The result type (including grouping) will be overridden then this function is bound
  4075. func = getResultDatasetAtom;
  4076. bool defaultLCR = targetAssign ? hasLinkedRow(targetAssign->queryType()) : options.tempDatasetsUseLinkedRows;
  4077. if (hasLinkCountedModifier(type) || defaultLCR)
  4078. {
  4079. ensureSerialized = false;
  4080. args.append(*createRowAllocator(ctx, record));
  4081. args.append(*createRowSerializer(ctx, record, deserializerAtom));
  4082. args.append(*createConstant(isGrouped(expr)));
  4083. overrideType.setown(setLinkCountedAttr(overrideType, true));
  4084. func = getResultRowsetAtom;
  4085. }
  4086. }
  4087. else
  4088. {
  4089. overrideType.set(type);
  4090. ITypeInfo * elementType = type->queryChildType();
  4091. OwnedHqlExpr field = createField(valueAtom, LINK(elementType), NULL);
  4092. record.setown(createRecord(field));
  4093. func = getResultSetAtom;
  4094. }
  4095. if (ensureSerialized && record)
  4096. record.setown(getSerializedForm(record));
  4097. if (record && (seqValue == ResultSequenceStored))
  4098. {
  4099. StringBuffer s;
  4100. OwnedHqlExpr ds = createDataset(no_anon, LINK(record));
  4101. StringBuffer xmlInstanceName, xmlFactoryName;
  4102. bool usesContents = false;
  4103. getUniqueId(xmlInstanceName.append("xml"));
  4104. buildXmlReadTransform(ds, xmlFactoryName, usesContents);
  4105. OwnedHqlExpr curActivityId = getCurrentActivityId(ctx);
  4106. s.append("Owned<IXmlToRowTransformer> ").append(xmlInstanceName).append(" = ").append(xmlFactoryName).append("(ctx,");
  4107. generateExprCpp(s, curActivityId).append(");");
  4108. ctx.addQuoted(s);
  4109. args.append(*createQuoted(xmlInstanceName, makeBoolType()));
  4110. StringBuffer csvInstanceName;
  4111. if (canReadFromCsv(record))
  4112. {
  4113. buildCsvReadTransformer(ds, csvInstanceName, NULL);
  4114. csvInstanceName.insert(0, "&");
  4115. }
  4116. else
  4117. {
  4118. csvInstanceName.clear().append("0");
  4119. }
  4120. args.append(*createQuoted(csvInstanceName, makeBoolType()));
  4121. }
  4122. else
  4123. {
  4124. args.append(*createQuoted("0", makeBoolType()));
  4125. args.append(*createQuoted("0", makeBoolType()));
  4126. }
  4127. break;
  4128. }
  4129. case type_string:
  4130. {
  4131. func = getResultStringAtom;
  4132. if ((type->queryCharset()->queryName() != asciiAtom) || !targetAssign)
  4133. break;
  4134. ITypeInfo * targetType = targetAssign->queryType();
  4135. if ((targetType->getTypeCode() != type_string) || (targetType->getSize() == UNKNOWN_LENGTH) ||
  4136. (targetType->queryCharset() != type->queryCharset()))
  4137. break;
  4138. //more: if (options.checkOverflow && queryUnqualifiedType(targetType) != queryUnqualifiedType(type)
  4139. args.add(*targetAssign->getTranslatedExpr(), 0);
  4140. buildFunctionCall(ctx, getResultStringFAtom, args);
  4141. return;
  4142. }
  4143. case type_qstring: func = getResultStringAtom; break;
  4144. case type_varstring:func = getResultVarStringAtom; break;
  4145. case type_unicode: func = getResultUnicodeAtom; break;
  4146. case type_varunicode:func = getResultVarUnicodeAtom; break;
  4147. case type_utf8: func = getResultUnicodeAtom; break;
  4148. case type_real: func = getResultRealAtom; break;
  4149. case type_decimal:
  4150. {
  4151. //Special case - need to bind the first parameter..., since not calling buildExpr on the call.
  4152. //sequence is always a constant, so no need to bind.
  4153. CHqlBoundExpr boundName;
  4154. buildExpr(ctx, &args.item(0), boundName);
  4155. args.replace(*LINK(boundName.expr), 0);
  4156. const CHqlBoundTarget * getTarget = targetAssign;
  4157. CHqlBoundTarget tempTarget;
  4158. if (!getTarget)
  4159. {
  4160. getTarget = &tempTarget;
  4161. createTempFor(ctx, expr, tempTarget);
  4162. }
  4163. args.add(*createConstant((int)type->getSize()), 0);
  4164. args.add(*getSizetConstant(type->getPrecision()),1);
  4165. args.add(*createConstant(type->isSigned()),2);
  4166. args.add(*getPointer(getTarget->expr), 3);
  4167. callProcedure(ctx, getResultDecimalAtom, args);
  4168. if (boundTarget)
  4169. boundTarget->setFromTarget(*getTarget);
  4170. return;
  4171. }
  4172. case type_row: UNIMPLEMENTED; break; //should be translated to rawData.
  4173. default:
  4174. PrintLog("%d", ttc);
  4175. assertex(!"No getResult defined for this type");
  4176. break;
  4177. }
  4178. OwnedHqlExpr function = bindFunctionCall(func, args, overrideType);
  4179. switch (ttc)
  4180. {
  4181. case type_qstring:
  4182. {
  4183. Owned<ITypeInfo> qstrType = makeQStringType(UNKNOWN_LENGTH);
  4184. function.setown(ensureExprType(function, type));
  4185. break;
  4186. }
  4187. case type_string:
  4188. case type_varstring:
  4189. {
  4190. if (type->queryCollation()->queryName() != asciiAtom)
  4191. function.setown(ensureExprType(function, type));
  4192. break;
  4193. }
  4194. }
  4195. if (boundTarget)
  4196. buildExpr(ctx, function, *boundTarget);
  4197. else
  4198. buildExprAssign(ctx, *targetAssign, function);
  4199. }
  4200. void HqlCppTranslator::buildSetXmlSerializer(StringBuffer & helper, ITypeInfo * valueType)
  4201. {
  4202. BuildCtx declarectx(*code, declareAtom);
  4203. OwnedHqlExpr search = createQuoted("setXmlHelper", LINK(valueType));
  4204. // stop duplicate classes being generated.
  4205. // MORE: If this ever includes sorting/grouping, the dependence on a record will need to be revised
  4206. HqlExprAssociation * match = declarectx.queryMatchExpr(search);
  4207. if (match)
  4208. {
  4209. match->queryExpr()->toString(helper);
  4210. return;
  4211. }
  4212. StringBuffer helperclass;
  4213. unique_id_t id = getUniqueId();
  4214. appendUniqueId(helper.append("r2x"), id);
  4215. appendUniqueId(helperclass.append("cr2x"), id);
  4216. BuildCtx r2xctx(declarectx);
  4217. r2xctx.setNextPriority(XmlTransformerPrio);
  4218. StringBuffer s, endText;
  4219. s.append("struct ").append(helperclass).append(" : public ISetToXmlTransformer");
  4220. endText.append(" ").append(helper).append(";");
  4221. r2xctx.addQuotedCompound(s, endText.str());
  4222. CHqlBoundExpr boundValue;
  4223. boundValue.isAll.setown(createVariable("isAll", makeBoolType()));
  4224. boundValue.length.setown(createVariable("length", LINK(sizetType)));
  4225. boundValue.expr.setown(createVariable("self", makeReferenceModifier(LINK(valueType))));
  4226. r2xctx.addQuotedCompound("virtual void toXML(bool isAll, size32_t length, const byte * self, IXmlWriter & out)");
  4227. OwnedHqlExpr itemName = createConstant("Item");
  4228. OwnedHqlExpr value = boundValue.getTranslatedExpr();
  4229. buildXmlSerializeSetValues(r2xctx, value, itemName, true);
  4230. if (options.spanMultipleCpp)
  4231. {
  4232. StringBuffer helperFunc;
  4233. createAccessFunctions(helperFunc, declarectx, XmlTransformerPrio, "ISetToXmlTransformer", helper);
  4234. helper.clear().append(helperFunc).append("()");
  4235. }
  4236. OwnedHqlExpr name = createVariable(helper, makeVoidType());
  4237. declarectx.associateExpr(search, name);
  4238. }
  4239. IWUResult * HqlCppTranslator::createWorkunitResult(int sequence, IHqlExpression * nameExpr)
  4240. {
  4241. switch(sequence)
  4242. {
  4243. case ResultSequenceStored:
  4244. {
  4245. assertex(nameExpr);
  4246. StringBuffer storedName;
  4247. getStringValue(storedName, nameExpr);
  4248. return wu()->updateVariableByName(storedName.str());
  4249. }
  4250. case ResultSequencePersist:
  4251. case ResultSequenceInternal:
  4252. case ResultSequenceOnce:
  4253. return NULL;
  4254. }
  4255. assertex(sequence >= 0);
  4256. StringBuffer resultName;
  4257. getStringValue(resultName, nameExpr);
  4258. if (resultName.length() == 0)
  4259. resultName.append("Result ").append(sequence+1);
  4260. Owned<IWUResult> result = wu()->updateResultBySequence(sequence);
  4261. result->setResultName(resultName);
  4262. return result.getClear();
  4263. }
  4264. void HqlCppTranslator::buildSetResultInfo(BuildCtx & ctx, IHqlExpression * originalExpr, IHqlExpression * value, ITypeInfo * type, bool isPersist, bool associateResult)
  4265. {
  4266. IHqlExpression * seq = queryPropertyChild(originalExpr, sequenceAtom, 0);
  4267. IHqlExpression * name = queryPropertyChild(originalExpr, namedAtom, 0);
  4268. if (insideLibrary())
  4269. {
  4270. SCMStringBuffer libraryName;
  4271. getOutputLibraryName(libraryName, wu());
  4272. StringBuffer storedName;
  4273. getStoredDescription(storedName, seq, name, true);
  4274. throwError2(HQLERR_LibraryNoWorkunitWrite, libraryName.str(), storedName.str());
  4275. }
  4276. ITypeInfo * resultType = type ? type->queryPromotedType() : value->queryType()->queryPromotedType();
  4277. Linked<ITypeInfo> schemaType = resultType;
  4278. type_t retType = schemaType->getTypeCode();
  4279. _ATOM func;
  4280. CHqlBoundExpr valueToSave;
  4281. LinkedHqlExpr castValue = value;
  4282. switch(retType)
  4283. {
  4284. case type_int:
  4285. case type_swapint:
  4286. {
  4287. bool isSigned = schemaType->isSigned();
  4288. func = isSigned ? setResultIntAtom : setResultUIntAtom;
  4289. schemaType.setown(makeIntType(8, isSigned));
  4290. break;
  4291. }
  4292. case type_boolean: func = setResultBoolAtom; break;
  4293. case type_string: func = setResultStringAtom; schemaType.setown(makeStringType(UNKNOWN_LENGTH, NULL, NULL)); break;
  4294. case type_unicode: func = setResultUnicodeAtom; schemaType.setown(makeUnicodeType(UNKNOWN_LENGTH, 0)); break;
  4295. case type_utf8: func = setResultUnicodeAtom; schemaType.setown(makeUnicodeType(UNKNOWN_LENGTH, 0)); castValue.setown(ensureExprType(value, schemaType)); associateResult = false; break;
  4296. case type_qstring: func = setResultStringAtom; schemaType.setown(makeStringType(UNKNOWN_LENGTH, NULL, NULL)); break;
  4297. case type_data: func = setResultDataAtom; schemaType.setown(makeDataType(UNKNOWN_LENGTH)); break;
  4298. case type_varstring:func = setResultVarStringAtom; schemaType.setown(makeStringType(UNKNOWN_LENGTH, NULL, NULL)); break;
  4299. case type_varunicode:func = setResultVarUnicodeAtom; schemaType.setown(makeUnicodeType(UNKNOWN_LENGTH, 0)); break;
  4300. case type_real: func = setResultRealAtom; schemaType.setown(makeRealType(8)); break;
  4301. case type_decimal: func = setResultDecimalAtom; break;
  4302. case type_row:
  4303. {
  4304. CHqlBoundExpr boundLength;
  4305. OwnedHqlExpr serialized = ::ensureSerialized(value);
  4306. func = setResultRawAtom;
  4307. Owned<IReferenceSelector> ds = buildNewRow(ctx, serialized);
  4308. OwnedHqlExpr size = createSizeof(ds->queryExpr());
  4309. buildExpr(ctx, size, boundLength);
  4310. ds->buildAddress(ctx, valueToSave);
  4311. valueToSave.length.set(boundLength.expr);
  4312. valueToSave.expr.setown(createValue(no_typetransfer, makeDataType(UNKNOWN_LENGTH), LINK(valueToSave.expr)));
  4313. schemaType.set(schemaType->queryChildType());
  4314. break;
  4315. }
  4316. case type_set:
  4317. {
  4318. func = setResultSetAtom;
  4319. ITypeInfo * elementType = LINK(schemaType->queryChildType());
  4320. if (!elementType)
  4321. elementType = makeStringType(UNKNOWN_LENGTH, NULL, NULL);
  4322. schemaType.setown(makeSetType(elementType));
  4323. }
  4324. break;
  4325. case type_table:
  4326. case type_groupedtable:
  4327. {
  4328. throwUnexpected();
  4329. #if 0
  4330. HqlExprArray args;
  4331. args.append(*LINK(value));
  4332. args.append(*createAttribute(sequenceAtom, LINK(seq)));
  4333. if (name)
  4334. args.append(*createAttribute(nameAtom, LINK(name)));
  4335. OwnedHqlExpr createFile = createValue(no_output, makeVoidType(), args);
  4336. buildStmt(ctx, createFile);
  4337. // MORE - the file name should be a unique temporary...
  4338. // MORE - associate a logical name with it
  4339. // MORE - save the logical name in the workunit
  4340. #endif
  4341. return;
  4342. }
  4343. case type_any:
  4344. //Someone has used error instead of fail. Don't do anything....
  4345. if (value->getOperator() == no_fail)
  4346. {
  4347. buildStmt(ctx, value);
  4348. return;
  4349. }
  4350. //fall through
  4351. default:
  4352. PrintLog("%d", retType);
  4353. throwError(HQLERR_InvalidSetResultType);
  4354. }
  4355. HqlExprArray args;
  4356. if (!valueToSave.expr)
  4357. {
  4358. LinkedHqlExpr cseValue = castValue;
  4359. if (options.spotCSE)
  4360. cseValue.setown(spotScalarCSE(cseValue));
  4361. if ((retType == type_set) && isComplexSet(resultType) && castValue->getOperator() == no_list && !isNullList(castValue))
  4362. {
  4363. CHqlBoundTarget tempTarget;
  4364. createTempFor(ctx, resultType, tempTarget, typemod_none, FormatBlockedDataset);
  4365. Owned<IHqlCppSetBuilder> builder = createTempSetBuilder(tempTarget.queryType()->queryChildType(), tempTarget.isAll);
  4366. builder->buildDeclare(ctx);
  4367. buildSetAssign(ctx, builder, castValue);
  4368. builder->buildFinish(ctx, tempTarget);
  4369. valueToSave.setFromTarget(tempTarget);
  4370. }
  4371. else
  4372. buildSimpleExpr(ctx, cseValue, valueToSave);
  4373. if (associateResult)
  4374. {
  4375. OwnedHqlExpr getResult = createGetResultFromSetResult(originalExpr);
  4376. ctx.associateExpr(getResult, valueToSave);
  4377. }
  4378. }
  4379. OwnedHqlExpr nameText = createResultName(name, isPersist);
  4380. if (retType == type_decimal)
  4381. {
  4382. assertex(schemaType->getSize() != UNKNOWN_LENGTH);
  4383. //An ugly exception because it takes an arbitrary length decimal.
  4384. //This should be handled by having a decimal(unknown length) parameter to a function which passes size and precision
  4385. CHqlBoundExpr boundName;
  4386. buildExpr(ctx, nameText, boundName);
  4387. args.append(*LINK(boundName.expr));
  4388. args.append(*LINK(seq));
  4389. args.append(*getBoundSize(valueToSave));
  4390. args.append(*getSizetConstant(schemaType->getPrecision()));
  4391. args.append(*createConstant(schemaType->isSigned()));
  4392. args.append(*getPointer(valueToSave.expr));
  4393. buildTranslatedFunctionCall(ctx, func, args);
  4394. }
  4395. else
  4396. {
  4397. args.append(*nameText.getLink());
  4398. args.append(*LINK(seq));
  4399. args.append(*valueToSave.getTranslatedExpr());
  4400. if (func == setResultSetAtom)
  4401. {
  4402. StringBuffer helper, s;
  4403. buildSetXmlSerializer(helper, resultType);
  4404. s.clear().append("&").append(helper);
  4405. args.append(*createQuoted(s, makeBoolType()));
  4406. }
  4407. buildFunctionCall(ctx, func, args);
  4408. }
  4409. if(wu())
  4410. {
  4411. if (retType == type_row)
  4412. {
  4413. Owned<IWUResult> result = createDatasetResultSchema(seq, name, ::queryRecord(schemaType), false, false);
  4414. if (result)
  4415. result->setResultTotalRowCount(1);
  4416. }
  4417. else
  4418. {
  4419. // Bit of a mess - should split into two procedures
  4420. int sequence = (int) seq->queryValue()->getIntValue();
  4421. Owned<IWUResult> result = createWorkunitResult(sequence, name);
  4422. if(result)
  4423. {
  4424. StringBuffer fieldName;
  4425. SCMStringBuffer resultName;
  4426. result->getResultName(resultName);
  4427. const char * cur = resultName.str();
  4428. while (*cur)
  4429. {
  4430. unsigned char c = *cur++;
  4431. if (isalnum(c) || (c == '_'))
  4432. fieldName.append(c);
  4433. else if (isspace(c))
  4434. fieldName.append('_');
  4435. }
  4436. MemoryBuffer schema;
  4437. schema.append(fieldName.str());
  4438. schemaType->serialize(schema);
  4439. schema.append("").append((unsigned char) type_void);
  4440. schema.append((unsigned)0);
  4441. result->setResultSchemaRaw(schema.length(), schema.toByteArray());
  4442. StringBuffer xml;
  4443. {
  4444. XmlSchemaBuilder xmlbuilder(false);
  4445. xmlbuilder.addField(fieldName, *schemaType);
  4446. xmlbuilder.getXml(xml);
  4447. }
  4448. addSchemaResource(sequence, resultName.str(), xml.length()+1, xml.str());
  4449. }
  4450. }
  4451. }
  4452. }
  4453. void HqlCppTranslator::buildCompareClass(BuildCtx & ctx, const char * name, IHqlExpression * orderExpr, IHqlExpression * datasetLeft, IHqlExpression * datasetRight, IHqlExpression * selSeq)
  4454. {
  4455. BuildCtx classctx(ctx);
  4456. beginNestedClass(classctx, name, "ICompare");
  4457. BuildCtx funcctx(classctx);
  4458. funcctx.addQuotedCompound("virtual int docompare(const void * _left, const void * _right) const");
  4459. funcctx.addQuoted("const unsigned char * left = (const unsigned char *) _left;");
  4460. funcctx.addQuoted("const unsigned char * right = (const unsigned char *) _right;");
  4461. funcctx.associateExpr(constantMemberMarkerExpr, constantMemberMarkerExpr);
  4462. bindTableCursor(funcctx, datasetLeft, "left", no_left, selSeq);
  4463. bindTableCursor(funcctx, datasetRight, "right", no_right, selSeq);
  4464. if (orderExpr->getOperator() == no_order)
  4465. doBuildReturnCompare(funcctx, orderExpr, no_order, false);
  4466. else
  4467. buildReturn(funcctx, orderExpr);
  4468. endNestedClass();
  4469. }
  4470. void HqlCppTranslator::buildCompareMemberLR(BuildCtx & ctx, const char * name, IHqlExpression * orderExpr, IHqlExpression * datasetLeft, IHqlExpression * datasetRight, IHqlExpression * selSeq)
  4471. {
  4472. StringBuffer s;
  4473. s.clear().append("virtual ICompare * query").append(name).append("() { return &").append(name).append("; }");
  4474. ctx.addQuoted(s);
  4475. buildCompareClass(ctx, name, orderExpr, datasetLeft, datasetRight, selSeq);
  4476. }
  4477. void HqlCppTranslator::buildCompareMember(BuildCtx & ctx, const char * name, IHqlExpression * cond, const DatasetReference & dataset)
  4478. {
  4479. //MORE:Support multiple comparison fields.
  4480. IHqlExpression * datasetExpr = dataset.queryDataset();
  4481. OwnedHqlExpr seq = createDummySelectorSequence();
  4482. OwnedHqlExpr leftSelect = createSelector(no_left, datasetExpr, seq);
  4483. OwnedHqlExpr rightSelect = createSelector(no_right, datasetExpr, seq);
  4484. IHqlExpression * left = dataset.mapCompound(cond, leftSelect);
  4485. IHqlExpression * right = dataset.mapCompound(cond, rightSelect);
  4486. OwnedHqlExpr compare = createValue(no_order, LINK(signedType), left, right);
  4487. buildCompareMemberLR(ctx, name, compare, datasetExpr, datasetExpr, seq);
  4488. }
  4489. void HqlCppTranslator::buildCompareEqClass(BuildCtx & ctx, const char * name, IHqlExpression * orderExpr, IHqlExpression * datasetLeft, IHqlExpression * datasetRight, IHqlExpression * selSeq)
  4490. {
  4491. BuildCtx classctx(ctx);
  4492. beginNestedClass(classctx, name, "ICompareEq");
  4493. BuildCtx funcctx(classctx);
  4494. funcctx.addQuotedCompound("virtual bool match(const void * _left, const void * _right) const");
  4495. funcctx.addQuoted("const unsigned char * left = (const unsigned char *) _left;");
  4496. funcctx.addQuoted("const unsigned char * right = (const unsigned char *) _right;");
  4497. funcctx.associateExpr(constantMemberMarkerExpr, constantMemberMarkerExpr);
  4498. bindTableCursor(funcctx, datasetLeft, "left", no_left, selSeq);
  4499. bindTableCursor(funcctx, datasetRight, "right", no_right, selSeq);
  4500. if (orderExpr->getOperator() == no_order)
  4501. doBuildReturnCompare(funcctx, orderExpr, no_eq, true);
  4502. else
  4503. buildReturn(funcctx, orderExpr);
  4504. endNestedClass();
  4505. }
  4506. void HqlCppTranslator::buildCompareEqMemberLR(BuildCtx & ctx, const char * name, IHqlExpression * orderExpr, IHqlExpression * datasetLeft, IHqlExpression * datasetRight, IHqlExpression * selSeq)
  4507. {
  4508. StringBuffer s;
  4509. s.clear().append("virtual ICompareEq * query").append(name).append("() { return &").append(name).append("; }");
  4510. ctx.addQuoted(s);
  4511. buildCompareEqClass(ctx, name, orderExpr, datasetLeft, datasetRight, selSeq);
  4512. }
  4513. void HqlCppTranslator::buildNaryCompareClass(BuildCtx & ctx, const char * name, IHqlExpression * expr, IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid)
  4514. {
  4515. BuildCtx classctx(ctx);
  4516. beginNestedClass(classctx, name, "INaryCompareEq");
  4517. BuildCtx funcctx(classctx);
  4518. funcctx.addQuotedCompound("virtual bool match(unsigned numRows, const void * * _rows) const");
  4519. funcctx.addQuoted("const unsigned char * left = (const unsigned char *) _rows[0];");
  4520. funcctx.addQuoted("unsigned char * * rows = (unsigned char * *) _rows;");
  4521. funcctx.associateExpr(constantMemberMarkerExpr, constantMemberMarkerExpr);
  4522. bindTableCursor(funcctx, dataset, "left", no_left, selSeq);
  4523. bindRows(funcctx, no_left, selSeq, rowsid, dataset, "numRows", "rows", false);
  4524. buildReturn(funcctx, expr);
  4525. endNestedClass();
  4526. }
  4527. void HqlCppTranslator::buildNaryCompareMember(BuildCtx & ctx, const char * name, IHqlExpression * expr, IHqlExpression * datasetLeft, IHqlExpression * selSeq, IHqlExpression * rowsid)
  4528. {
  4529. StringBuffer s;
  4530. s.clear().append("virtual INaryCompareEq * query").append(name).append("() { return &").append(name).append("; }");
  4531. ctx.addQuoted(s);
  4532. buildNaryCompareClass(ctx, name, expr, datasetLeft, selSeq, rowsid);
  4533. }
  4534. void HqlCppTranslator::buildCompareEqMember(BuildCtx & ctx, const char * name, IHqlExpression * cond, const DatasetReference & dataset)
  4535. {
  4536. //MORE:Support multiple comparison fields.
  4537. IHqlExpression * datasetExpr = dataset.queryDataset();
  4538. OwnedHqlExpr seq = createDummySelectorSequence();
  4539. OwnedHqlExpr leftSelect = createSelector(no_left, datasetExpr, seq);
  4540. OwnedHqlExpr rightSelect = createSelector(no_right, datasetExpr, seq);
  4541. IHqlExpression * left = dataset.mapCompound(cond, leftSelect);
  4542. IHqlExpression * right = dataset.mapCompound(cond, rightSelect);
  4543. OwnedHqlExpr compare = createValue(no_order, LINK(signedType), left, right);
  4544. buildCompareEqMemberLR(ctx, name, compare, datasetExpr, datasetExpr, seq);
  4545. }
  4546. void HqlCppTranslator::buildOrderedCompare(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * sorts, CHqlBoundExpr & bound, IHqlExpression * leftDataset, IHqlExpression * rightDataset)
  4547. {
  4548. //MORE: This needs to be more intelligent to deal with related datasets (child datasets will work ok).
  4549. //MORE: Should create a member function if a member of a class.
  4550. //Otherwise a function + generate an error if it uses fields outside of the context?
  4551. //Would also need to pass in parent cursors for a global function since parent will not be available.
  4552. //MORE:Support multiple comparison fields.
  4553. OwnedHqlExpr compare = createOrderFromSortList(DatasetReference(dataset), sorts, leftDataset, rightDataset);
  4554. buildTempExpr(ctx, compare, bound);
  4555. }
  4556. void HqlCppTranslator::buildHashClass(BuildCtx & ctx, const char * name, IHqlExpression * orderExpr, const DatasetReference & dataset)
  4557. {
  4558. StringBuffer s;
  4559. s.clear().append("virtual IHash * query").append(name).append("() { return &").append(name).append("; }");
  4560. ctx.addQuoted(s);
  4561. BuildCtx classctx(ctx);
  4562. beginNestedClass(classctx, name, "IHash");
  4563. BuildCtx funcctx(classctx);
  4564. s.clear().append("virtual unsigned hash(const void * _self)");
  4565. funcctx.addQuotedCompound(s);
  4566. s.clear().append("const unsigned char * self = (const unsigned char *) _self;");
  4567. funcctx.addQuoted(s);
  4568. bindTableCursor(funcctx, dataset.queryDataset(), "self", dataset.querySide(), dataset.querySelSeq());
  4569. buildReturn(funcctx, orderExpr);
  4570. endNestedClass();
  4571. }
  4572. void HqlCppTranslator::buildCompareClass(BuildCtx & ctx, const char * name, IHqlExpression * sortList, const DatasetReference & dataset)
  4573. {
  4574. BuildCtx comparectx(ctx);
  4575. beginNestedClass(comparectx, name, "ICompare");
  4576. BuildCtx funcctx(comparectx);
  4577. funcctx.addQuotedCompound("virtual int docompare(const void * _left, const void * _right) const");
  4578. funcctx.addQuoted("const unsigned char * left = (const unsigned char *) _left;");
  4579. funcctx.addQuoted("const unsigned char * right = (const unsigned char *) _right;");
  4580. funcctx.associateExpr(constantMemberMarkerExpr, constantMemberMarkerExpr);
  4581. buildReturnOrder(funcctx, sortList, dataset);
  4582. endNestedClass();
  4583. }
  4584. void HqlCppTranslator::buildHashOfExprsClass(BuildCtx & ctx, const char * name, IHqlExpression * cond, const DatasetReference & dataset, bool compareToSelf)
  4585. {
  4586. IHqlExpression * attr = compareToSelf ? createAttribute(internalAtom) : NULL;
  4587. OwnedHqlExpr hash = createValue(no_hash32, LINK(unsignedType), LINK(cond), attr);
  4588. buildHashClass(ctx, name, hash, dataset);
  4589. }
  4590. //---------------------------------------------------------------------------
  4591. IHqlExpression * queryImplementationInterface(IHqlExpression * moduleFunc)
  4592. {
  4593. IHqlExpression * module = moduleFunc->queryChild(0);
  4594. IHqlExpression * library = module->queryProperty(libraryAtom);
  4595. if (library)
  4596. return library->queryChild(0);
  4597. return moduleFunc;
  4598. }
  4599. bool isLibraryScope(IHqlExpression * expr)
  4600. {
  4601. while (expr->getOperator() == no_comma)
  4602. expr = expr->queryChild(1);
  4603. return expr->isScope();
  4604. }
  4605. bool HqlCppTranslator::prepareToGenerate(IHqlExpression * exprlist, WorkflowArray & actions, bool isEmbeddedLibrary)
  4606. {
  4607. bool createLibrary = isLibraryScope(exprlist);
  4608. OwnedHqlExpr query = LINK(exprlist);
  4609. if (createLibrary)
  4610. {
  4611. if (query->getOperator() != no_funcdef)
  4612. throwError(HQLERR_LibraryMustBeFunctional);
  4613. ::Release(outputLibrary);
  4614. outputLibrary = NULL;
  4615. outputLibraryId.setown(createAttribute(graphAtom, getSizetConstant(nextActivityId())));
  4616. outputLibrary = new HqlCppLibraryImplementation(*this, queryImplementationInterface(query), outputLibraryId, targetClusterType);
  4617. if (!isEmbeddedLibrary)
  4618. {
  4619. SCMStringBuffer libraryName;
  4620. wu()->getJobName(libraryName);
  4621. wu()->setLibraryInformation(libraryName.str(), outputLibrary->getInterfaceHash(), getLibraryCRC(query));
  4622. }
  4623. }
  4624. else
  4625. {
  4626. if (options.applyInstantEclTransformations)
  4627. query.setown(doInstantEclTransformations(query, options.applyInstantEclTransformationsLimit));
  4628. }
  4629. if (!transformGraphForGeneration(query, actions))
  4630. return false;
  4631. return true;
  4632. }
  4633. void HqlCppTranslator::updateClusterType()
  4634. {
  4635. const char * clusterTypeText="?";
  4636. switch (targetClusterType)
  4637. {
  4638. case ThorCluster:
  4639. clusterTypeText = "thor";
  4640. break;
  4641. case ThorLCRCluster:
  4642. clusterTypeText = "thorlcr";
  4643. break;
  4644. case HThorCluster:
  4645. clusterTypeText = "hthor";
  4646. break;
  4647. case RoxieCluster:
  4648. clusterTypeText = "roxie";
  4649. break;
  4650. }
  4651. //ensure targetClusterType is consistently set in the work unit - so library code can use it.
  4652. wu()->setDebugValue("targetClusterType", clusterTypeText, true);
  4653. }
  4654. struct CountEntry : public CInterface
  4655. {
  4656. ThorActivityKind from;
  4657. ThorActivityKind to;
  4658. unsigned cnt;
  4659. };
  4660. int compareItems(CInterface * * _l, CInterface * * _r)
  4661. {
  4662. CountEntry * l = (CountEntry *)*_l;
  4663. CountEntry * r = (CountEntry *)*_r;
  4664. return (int)(r->cnt - l->cnt);
  4665. }
  4666. void dumpActivityCounts()
  4667. {
  4668. #ifdef _GATHER_USAGE_STATS
  4669. CIArray items;
  4670. for (unsigned i = TAKnone; i < TAKlast; i++)
  4671. {
  4672. for (unsigned j = TAKnone; j < TAKlast; j++)
  4673. {
  4674. if (activityCounts[i][j])
  4675. {
  4676. CountEntry & next = * new CountEntry;
  4677. next.from = (ThorActivityKind)i;
  4678. next.to = (ThorActivityKind)j;
  4679. next.cnt = activityCounts[i][j];
  4680. items.append(next);
  4681. }
  4682. }
  4683. }
  4684. items.sort(compareItems);
  4685. FILE * out = fopen("stats.txt", "w");
  4686. if (!out)
  4687. return;
  4688. fprintf(out, "-Count- Source -> Sink\n");
  4689. ForEachItemIn(k, items)
  4690. {
  4691. CountEntry & cur = (CountEntry &)items.item(k);
  4692. fprintf(out, "%8d %s -> %s\n", cur.cnt, getActivityText(cur.from), getActivityText(cur.to));
  4693. }
  4694. fclose(out);
  4695. #endif
  4696. }
  4697. bool HqlCppTranslator::buildCode(IHqlExpression * exprlist, const char * embeddedLibraryName, bool isEmbeddedLibrary)
  4698. {
  4699. unsigned time = msTick();
  4700. WorkflowArray workflow;
  4701. bool ok = prepareToGenerate(exprlist, workflow, isEmbeddedLibrary);
  4702. if (ok)
  4703. {
  4704. //This is done late so that pickBestEngine has decided which engine we are definitely targeting.
  4705. if (!isEmbeddedLibrary)
  4706. updateClusterType();
  4707. if (options.addTimingToWorkunit)
  4708. wu()->setTimerInfo("EclServer: tree transform", NULL, msTick()-time, 1, 0);
  4709. if (insideLibrary())
  4710. {
  4711. //always do these checks for consistency
  4712. assertex(workflow.ordinality() == 1);
  4713. HqlExprArray & exprs = workflow.item(0).queryExprs();
  4714. assertex(exprs.ordinality() == 1);
  4715. IHqlExpression & expr = exprs.item(0);
  4716. assertex(expr.getOperator() == no_thor);
  4717. if (embeddedLibraryName)
  4718. {
  4719. BuildCtx ctx(*code, goAtom);
  4720. buildLibraryGraph(ctx, &expr, embeddedLibraryName);
  4721. }
  4722. else
  4723. buildWorkflow(workflow);
  4724. }
  4725. else
  4726. buildWorkflow(workflow);
  4727. if (options.calculateComplexity)
  4728. {
  4729. unsigned time = msTick();
  4730. StringBuffer complexityText;
  4731. complexityText.append(getComplexity(workflow));
  4732. wu()->setDebugValue("__Calculated__Complexity__", complexityText, true);
  4733. if (options.addTimingToWorkunit)
  4734. wu()->setTimerInfo("EclServer: calculate complexity", NULL, msTick()-time, 1, 0);
  4735. }
  4736. }
  4737. ::Release(outputLibrary);
  4738. outputLibrary = NULL;
  4739. outputLibraryId.clear();
  4740. return ok;
  4741. }
  4742. bool HqlCppTranslator::buildCpp(IHqlCppInstance & _code, IHqlExpression * exprlist)
  4743. {
  4744. if (!internalScope)
  4745. return false;
  4746. try
  4747. {
  4748. unsigned time = msTick();
  4749. wu()->setCodeVersion(ACTIVITY_INTERFACE_VERSION,BUILD_TAG,LANGUAGE_VERSION);
  4750. StringAttrAdaptor adaptor(defaultCluster);
  4751. wu()->getClusterName(adaptor);
  4752. curCluster.set(defaultCluster);
  4753. cacheOptions();
  4754. useLibrary(ECLRTL_LIB);
  4755. useInclude("eclrtl.hpp");
  4756. HqlExprArray internalLibraries;
  4757. OwnedHqlExpr query = separateLibraries(exprlist, internalLibraries);
  4758. //General internal libraries first, in dependency order
  4759. ForEachItemIn(i, internalLibraries)
  4760. {
  4761. IHqlExpression & cur = internalLibraries.item(i);
  4762. assertex(cur.getOperator() == no_funcdef);
  4763. IHqlExpression * moduleExpr = cur.queryChild(0);
  4764. IHqlExpression * definition = queryPropertyChild(moduleExpr, internalAtom, 0);
  4765. IHqlExpression * name = queryPropertyChild(moduleExpr, nameAtom, 0);
  4766. StringBuffer internalLibraryName;
  4767. name->queryValue()->getStringValue(internalLibraryName);
  4768. overrideOptionsForLibrary();
  4769. if (!buildCode(definition, internalLibraryName.str(), true))
  4770. return false;
  4771. }
  4772. if (isLibraryScope(exprlist))
  4773. overrideOptionsForLibrary();
  4774. else
  4775. overrideOptionsForQuery();
  4776. if (!buildCode(query, NULL, false))
  4777. return false;
  4778. #ifdef _GATHER_USAGE_STATS
  4779. if (getDebugFlag("dumpActivityCounts", false))
  4780. dumpActivityCounts();
  4781. #endif
  4782. ForEachItemIn(i1, globalFiles)
  4783. globalFiles.item(i1).writeToGraph();
  4784. //Have to submit graphs to work unit right at the end so that globalUsage counts etc. can be updated in them.
  4785. ForEachItemIn(i2, graphs)
  4786. {
  4787. GeneratedGraphInfo & cur = graphs.item(i2);
  4788. Owned<IWUGraph> wug = wu()->updateGraph(cur.name);
  4789. wug->setXGMMLTree(cur.graph.getClear());
  4790. wug->setType(GraphTypeActivities);
  4791. }
  4792. code->processIncludes();
  4793. if (options.peephole)
  4794. {
  4795. cycle_t time = msTick();
  4796. peepholeOptimize(*code, *this);
  4797. DEBUG_TIMER("EclServer: peephole optimize", msTick()-time);
  4798. }
  4799. }
  4800. catch (IException *)
  4801. {
  4802. ensureWorkUnitUpdated();
  4803. throw;
  4804. }
  4805. catch (...)
  4806. {
  4807. ensureWorkUnitUpdated();
  4808. throw;
  4809. }
  4810. ensureWorkUnitUpdated();
  4811. return true;
  4812. }
  4813. class WuTimingUpdater : implements ITimeReportInfo
  4814. {
  4815. public:
  4816. WuTimingUpdater(IWorkUnit * _wu) { wu = _wu; }
  4817. virtual void report(const char *name, const __int64 totaltime, const __int64 maxtime, const unsigned count)
  4818. {
  4819. wu->setTimerInfo(name, NULL, (unsigned)totaltime, count, (unsigned)maxtime);
  4820. }
  4821. protected:
  4822. IWorkUnit * wu;
  4823. };
  4824. void HqlCppTranslator::ensureWorkUnitUpdated()
  4825. {
  4826. if (timeReporter && options.addTimingToWorkunit)
  4827. {
  4828. WuTimingUpdater updater(wu());
  4829. timeReporter->report(updater);
  4830. }
  4831. }
  4832. double HqlCppTranslator::getComplexity(IHqlExpression * expr, ClusterType cluster)
  4833. {
  4834. double complexity = 0;
  4835. switch (expr->getOperator())
  4836. {
  4837. case no_sequential:
  4838. case no_comma:
  4839. case no_compound:
  4840. case no_parallel:
  4841. case no_actionlist:
  4842. break;
  4843. case no_thor:
  4844. {
  4845. OwnedHqlExpr resourced = getResourcedGraph(expr->queryChild(0), NULL);
  4846. lockTransformMutex();
  4847. complexity = getComplexity(resourced, targetClusterType);
  4848. unlockTransformMutex();
  4849. return complexity;
  4850. }
  4851. case no_subgraph:
  4852. return getComplexity(expr->queryChild(0), cluster);
  4853. case no_selfjoin:
  4854. case no_join:
  4855. if (isLocalActivity(expr) || !isThorCluster(cluster))
  4856. complexity = 1;
  4857. else
  4858. complexity = 5;
  4859. break;
  4860. case no_distribute:
  4861. if (isThorCluster(cluster))
  4862. complexity = 5;
  4863. break;
  4864. case no_sort:
  4865. if (isLocalActivity(expr) || !isThorCluster(cluster))
  4866. complexity = 2;
  4867. else
  4868. complexity = 5;
  4869. break;
  4870. case no_ensureresult:
  4871. if (options.freezePersists)
  4872. return 0;
  4873. if (expr->queryChild(0)->queryType()->getTypeCode() == type_void)
  4874. return getComplexity(expr->queryChild(0), cluster);
  4875. //fall through..
  4876. case no_setresult:
  4877. {
  4878. if (cluster == NoCluster)
  4879. return 0.05;
  4880. return 1;
  4881. }
  4882. case no_split:
  4883. //Only count the parents of a splitter once
  4884. if (expr->queryTransformExtra())
  4885. return 0;
  4886. expr->setTransformExtraUnlinked(expr);
  4887. complexity = 1;
  4888. break;
  4889. case no_forcelocal:
  4890. return getComplexity(expr->queryChild(0), cluster);
  4891. default:
  4892. complexity = 1;
  4893. break;
  4894. }
  4895. ForEachChildActivity(i, expr)
  4896. complexity += getComplexity(queryChildActivity(expr, i), cluster);
  4897. return complexity;
  4898. }
  4899. double HqlCppTranslator::getComplexity(IHqlCppInstance & _code, IHqlExpression * exprlist)
  4900. {
  4901. WorkflowArray workflow;
  4902. if (!prepareToGenerate(exprlist, workflow, false))
  4903. return 0;
  4904. return getComplexity(workflow);
  4905. }
  4906. double HqlCppTranslator::getComplexity(WorkflowArray & workflow)
  4907. {
  4908. double complexity = 0;
  4909. ForEachItemIn(i, workflow)
  4910. complexity += getComplexity(workflow.item(i).queryExprs());
  4911. return complexity;
  4912. }
  4913. double HqlCppTranslator::getComplexity(HqlExprArray & exprs)
  4914. {
  4915. double complexity = 0;
  4916. ForEachItemIn(idx, exprs)
  4917. complexity += getComplexity(&exprs.item(idx), NoCluster);
  4918. return complexity;
  4919. }
  4920. BoundRow * HqlCppTranslator::resolveDatasetRequired(BuildCtx & ctx, IHqlExpression * expr)
  4921. {
  4922. BoundRow * cursor = resolveSelectorDataset(ctx, expr);
  4923. if (!cursor)
  4924. {
  4925. StringBuffer tablename;
  4926. tablename.append(expr->queryName());
  4927. if (tablename.length() == 0)
  4928. getExprECL(expr, tablename);
  4929. if (ctx.queryFirstAssociation(AssocCursor))
  4930. throwError1(HQLERR_CouldNotFindDataset, tablename.str());
  4931. throwError1(HQLERR_CouldNotAnyDatasetX, tablename.str());
  4932. }
  4933. return cursor;
  4934. }
  4935. IReferenceSelector * HqlCppTranslator::buildActiveReference(BuildCtx & ctx, IHqlExpression * expr)
  4936. {
  4937. ITypeInfo * type = expr->queryType();
  4938. assertex(type);
  4939. switch (type->getTypeCode())
  4940. {
  4941. case type_row:
  4942. return buildActiveRow(ctx, expr);
  4943. }
  4944. return buildReference(ctx, expr);
  4945. }
  4946. IReferenceSelector * HqlCppTranslator::createReferenceSelector(BoundRow * cursor, IHqlExpression * path)
  4947. {
  4948. return new DatasetSelector(*this, cursor, path);
  4949. }
  4950. IReferenceSelector * HqlCppTranslator::createReferenceSelector(BoundRow * cursor)
  4951. {
  4952. return new DatasetSelector(*this, cursor, cursor->querySelector());
  4953. }
  4954. IReferenceSelector * HqlCppTranslator::buildReference(BuildCtx & ctx, IHqlExpression * expr)
  4955. {
  4956. ITypeInfo * type = expr->queryType();
  4957. assertex(type);
  4958. switch (type->getTypeCode())
  4959. {
  4960. case type_row:
  4961. return buildNewRow(ctx, expr);
  4962. }
  4963. const node_operator op = expr->getOperator();
  4964. switch (op)
  4965. {
  4966. case no_variable:
  4967. case no_field:
  4968. case no_ifblock:
  4969. throwUnexpectedOp(op);
  4970. case no_select:
  4971. {
  4972. IHqlExpression * ds = expr->queryChild(0);
  4973. IHqlExpression * field = expr->queryChild(1);
  4974. Owned<IReferenceSelector> selector;
  4975. if (expr->hasProperty(newAtom))
  4976. {
  4977. //could optimize selection from a project of an in-scope dataset.
  4978. if (((ds->getOperator() == no_newusertable) || (ds->getOperator() == no_hqlproject)) &&
  4979. isAlwaysActiveRow(ds->queryChild(0)))
  4980. {
  4981. //MORE: could optimize selection from a project of an in-scope dataset.
  4982. }
  4983. selector.setown(buildNewRow(ctx, ds));
  4984. }
  4985. else
  4986. selector.setown(buildActiveRow(ctx, ds));
  4987. return selector->select(ctx, expr);
  4988. }
  4989. }
  4990. BoundRow * cursor = resolveDatasetRequired(ctx, expr);
  4991. IReferenceSelector * alias = cursor->queryAlias();
  4992. if (alias)
  4993. return LINK(alias);
  4994. IHqlExpression * dataset = cursor->queryDataset();
  4995. return createReferenceSelector(cursor, dataset);
  4996. }
  4997. ABoundActivity * HqlCppTranslator::buildActivity(BuildCtx & ctx, IHqlExpression * expr, bool isRoot)
  4998. {
  4999. checkAbort();
  5000. WarningProcessor::OnWarningStateBlock saved(warningProcessor);
  5001. //Process any annotations first - but still pass the original expr to the doBuildActivtyXXX functions.
  5002. IHqlExpression * cur = expr;
  5003. loop
  5004. {
  5005. IHqlExpression * body = cur->queryBody(true);
  5006. if (cur == body)
  5007. break;
  5008. switch (cur->getAnnotationKind())
  5009. {
  5010. case annotate_meta:
  5011. warningProcessor.processMetaAnnotation(cur);
  5012. break;
  5013. case annotate_symbol:
  5014. {
  5015. #ifdef SPOT_POTENTIAL_COMMON_ACTIVITIES
  5016. if (cur->getStartLine() && cur->queryName()->str()[0] != '_')
  5017. {
  5018. unsigned match = savedActivityLocations.findLocation(cur);
  5019. if (match == NotFound)
  5020. {
  5021. savedActivityLocations.append(*LINK(cur));
  5022. savedActivities.append(*LINK(expr));
  5023. }
  5024. else
  5025. {
  5026. debugFindFirstDifference(cur, &savedActivities.item(match));
  5027. //MORE: Could add some kind of comment.
  5028. }
  5029. }
  5030. #endif
  5031. warningProcessor.setSymbol(cur);
  5032. break;
  5033. }
  5034. }
  5035. cur = body;
  5036. }
  5037. ABoundActivity * result;
  5038. try
  5039. {
  5040. switch (expr->getOperator())
  5041. {
  5042. case no_merge:
  5043. result = doBuildActivityMerge(ctx, expr);
  5044. break;
  5045. case no_nwaymerge:
  5046. result = doBuildActivityNWayMerge(ctx, expr);
  5047. break;
  5048. case no_mergejoin:
  5049. case no_nwayjoin:
  5050. result = doBuildActivityNWayMergeJoin(ctx, expr);
  5051. break;
  5052. case no_rowsetrange:
  5053. result = doBuildActivityRowsetRange(ctx, expr);
  5054. break;
  5055. case no_rowsetindex:
  5056. result = doBuildActivityRowsetIndex(ctx, expr);
  5057. break;
  5058. case no_datasetlist:
  5059. {
  5060. OwnedHqlExpr allExpr = createValue(no_all, makeSetType(LINK(sizetType)));
  5061. result = doBuildActivityRowsetRange(ctx, expr, expr, allExpr);
  5062. break;
  5063. }
  5064. case no_addfiles:
  5065. // result = doBuildActivityChildDataset(ctx, expr);
  5066. // break;
  5067. result = doBuildActivityConcat(ctx, expr);
  5068. break;
  5069. case no_regroup:
  5070. result = doBuildActivityRegroup(ctx, expr);
  5071. break;
  5072. case no_nonempty:
  5073. result = doBuildActivityNonEmpty(ctx, expr);
  5074. break;
  5075. case no_combine:
  5076. result = doBuildActivityCombine(ctx, expr);
  5077. break;
  5078. case no_combinegroup:
  5079. result = doBuildActivityCombineGroup(ctx, expr);
  5080. break;
  5081. case no_rollupgroup:
  5082. result = doBuildActivityRollupGroup(ctx, expr);
  5083. break;
  5084. case no_filtergroup:
  5085. result = doBuildActivityFilterGroup(ctx, expr);
  5086. break;
  5087. case no_cachealias:
  5088. result = doBuildActivityCacheAlias(ctx, expr);
  5089. break;
  5090. case no_cloned:
  5091. result = doBuildActivityCloned(ctx, expr);
  5092. break;
  5093. case no_distribute:
  5094. case no_assertdistributed:
  5095. result = doBuildActivityDistribute(ctx, expr);
  5096. break;
  5097. case no_keyeddistribute:
  5098. result = doBuildActivityKeyedDistribute(ctx, expr);
  5099. break;
  5100. case no_select:
  5101. if (isNewSelector(expr))
  5102. result = doBuildActivitySelectNew(ctx, expr);
  5103. else
  5104. result = doBuildActivityChildDataset(ctx, expr);
  5105. break;
  5106. //Items in this list need to also be in the list inside doBuildActivityChildDataset
  5107. case no_call:
  5108. case no_externalcall:
  5109. if (expr->isAction())
  5110. result = doBuildActivityAction(ctx, expr, isRoot, false);
  5111. else if (hasStreamedModifier(expr->queryType()))
  5112. {
  5113. result = doBuildActivityStreamedCall(ctx, expr);
  5114. }
  5115. else if (hasLinkCountedModifier(expr->queryType()))
  5116. {
  5117. result = doBuildActivityLinkedRawChildDataset(ctx, expr);
  5118. }
  5119. else
  5120. result = doBuildActivityChildDataset(ctx, expr);
  5121. break;
  5122. case no_left:
  5123. case no_right:
  5124. case no_top:
  5125. case no_id2blob:
  5126. case no_typetransfer:
  5127. case no_rows:
  5128. case no_xmlproject:
  5129. case no_libraryinput:
  5130. case no_activetable:
  5131. case no_translated:
  5132. result = doBuildActivityChildDataset(ctx, expr);
  5133. break;
  5134. case no_compound_inline:
  5135. result = doBuildActivityChildDataset(ctx, expr->queryChild(0));
  5136. break;
  5137. case no_table:
  5138. result = doBuildActivityTable(ctx, expr);
  5139. break;
  5140. case no_compound_diskread:
  5141. result = doBuildActivityDiskRead(ctx, expr);
  5142. break;
  5143. case no_compound_diskaggregate:
  5144. case no_compound_diskcount:
  5145. result = doBuildActivityDiskAggregate(ctx, expr);
  5146. break;
  5147. case no_compound_diskgroupaggregate:
  5148. result = doBuildActivityDiskGroupAggregate(ctx, expr);
  5149. break;
  5150. case no_compound_disknormalize:
  5151. result = doBuildActivityDiskNormalize(ctx, expr);
  5152. break;
  5153. case no_compound_childread:
  5154. case no_compound_childnormalize:
  5155. result = doBuildActivityChildNormalize(ctx, expr);
  5156. break;
  5157. case no_compound_childaggregate:
  5158. case no_compound_childcount:
  5159. result = doBuildActivityChildAggregate(ctx, expr);
  5160. break;
  5161. case no_compound_childgroupaggregate:
  5162. result = doBuildActivityChildGroupAggregate(ctx, expr);
  5163. break;
  5164. case no_compound_selectnew:
  5165. result = doBuildActivityCompoundSelectNew(ctx, expr);
  5166. break;
  5167. case no_keyed:
  5168. result = buildCachedActivity(ctx, expr->queryChild(0));
  5169. break;
  5170. case no_denormalize:
  5171. case no_denormalizegroup:
  5172. result = doBuildActivityDenormalize(ctx, expr);
  5173. break;
  5174. case no_datasetfromrow:
  5175. {
  5176. OwnedHqlExpr row = expr->cloneAllAnnotations(expr->queryChild(0)); // preserve any position information....
  5177. if ((getNumActivityArguments(expr) == 0) && canProcessInline(&ctx, row))
  5178. result = doBuildActivityCreateRow(ctx, row, false);
  5179. else
  5180. result = buildCachedActivity(ctx, row);
  5181. break;
  5182. }
  5183. case no_temptable:
  5184. result = doBuildActivityTempTable(ctx, expr);
  5185. break;
  5186. case no_inlinetable:
  5187. result = doBuildActivityInlineTable(ctx, expr);
  5188. break;
  5189. case no_temprow:
  5190. throwUnexpected();
  5191. break;
  5192. case no_workunit_dataset:
  5193. if (targetRoxie() && queryCurrentActivity(ctx))
  5194. result = doBuildActivityChildDataset(ctx, expr);
  5195. else
  5196. result = doBuildActivityWorkunitRead(ctx, expr);
  5197. break;
  5198. case no_fail:
  5199. result = doBuildActivityAction(ctx, expr, isRoot, !expr->isAction());
  5200. break;
  5201. case no_null:
  5202. if (expr->isDatarow())
  5203. result = doBuildActivityCreateRow(ctx, expr, false);
  5204. else
  5205. result = doBuildActivityNull(ctx, expr, isRoot);
  5206. break;
  5207. case no_split:
  5208. result = doBuildActivitySplit(ctx, expr);
  5209. break;
  5210. case no_apply:
  5211. result = doBuildActivityApply(ctx, expr, isRoot);
  5212. break;
  5213. case no_output:
  5214. result = doBuildActivityOutput(ctx, expr, isRoot);
  5215. break;
  5216. case no_extractresult:
  5217. case no_setresult:
  5218. result = doBuildActivitySetResult(ctx, expr, isRoot);
  5219. break;
  5220. case no_returnresult:
  5221. result = doBuildActivityReturnResult(ctx, expr, isRoot);
  5222. break;
  5223. case no_getgraphresult:
  5224. //Use the get graph result activity if we are generating the correct level graph.
  5225. //otherwise it needs to be serialized from the parent activity
  5226. {
  5227. if (isCurrentActiveGraph(ctx, expr->queryChild(1)))
  5228. result = doBuildActivityGetGraphResult(ctx, expr);
  5229. else
  5230. result = doBuildActivityChildDataset(ctx, expr);
  5231. break;
  5232. }
  5233. case no_getgraphloopresult:
  5234. //Use the get graph result activity if we are generating the correct level graph.
  5235. //otherwise it needs to be serialized from the parent activity
  5236. {
  5237. if (isCurrentActiveGraph(ctx, expr->queryChild(1)))
  5238. result = doBuildActivityGetGraphLoopResult(ctx, expr);
  5239. else
  5240. throwError(HQLERR_GraphInputAccessedChild);
  5241. break;
  5242. }
  5243. case no_setgraphresult:
  5244. case no_spillgraphresult:
  5245. result = doBuildActivitySetGraphResult(ctx, expr, isRoot);
  5246. break;
  5247. case no_setgraphloopresult:
  5248. result = doBuildActivitySetGraphLoopResult(ctx, expr);
  5249. break;
  5250. case no_spill:
  5251. result = doBuildActivitySpill(ctx, expr);
  5252. break;
  5253. case no_buildindex:
  5254. result = doBuildActivityOutputIndex(ctx, expr, isRoot);
  5255. break;
  5256. case no_distribution:
  5257. result = doBuildActivityDistribution(ctx, expr, isRoot);
  5258. break;
  5259. case no_keydiff:
  5260. result = doBuildActivityKeyDiff(ctx, expr, isRoot);
  5261. break;
  5262. case no_keypatch:
  5263. result = doBuildActivityKeyPatch(ctx, expr, isRoot);
  5264. break;
  5265. case no_if:
  5266. result = doBuildActivityIf(ctx, expr, isRoot);
  5267. break;
  5268. case no_case:
  5269. case no_map:
  5270. result = doBuildActivityCase(ctx, expr, isRoot);
  5271. break;
  5272. case no_iterate:
  5273. result = doBuildActivityIterate(ctx, expr);
  5274. break;
  5275. case no_process:
  5276. result = doBuildActivityProcess(ctx, expr);
  5277. break;
  5278. case no_group:
  5279. case no_cogroup:
  5280. case no_assertgrouped:
  5281. result = doBuildActivityGroup(ctx, expr);
  5282. break;
  5283. case no_normalize:
  5284. result = doBuildActivityNormalize(ctx, expr);
  5285. break;
  5286. case no_normalizegroup:
  5287. result = doBuildActivityNormalizeGroup(ctx, expr);
  5288. break;
  5289. case no_sorted:
  5290. case no_distributed:
  5291. case no_preservemeta:
  5292. case no_grouped:
  5293. case no_nofold:
  5294. case no_nohoist:
  5295. case no_globalscope:
  5296. case no_thisnode:
  5297. case no_forcegraph:
  5298. result = buildCachedActivity(ctx, expr->queryChild(0));
  5299. break;
  5300. case no_alias_scope:
  5301. case no_alias:
  5302. result = buildCachedActivity(ctx, expr->queryChild(0));
  5303. break;
  5304. case no_section:
  5305. result = doBuildActivitySection(ctx, expr);
  5306. break;
  5307. case no_sectioninput:
  5308. result = doBuildActivitySectionInput(ctx, expr);
  5309. break;
  5310. case no_forcelocal:
  5311. result = doBuildActivityForceLocal(ctx, expr);
  5312. break;
  5313. case no_preload:
  5314. {
  5315. StringBuffer s;
  5316. DBGLOG("%s", getExprECL(expr, s).str());
  5317. throwError(HQLERR_TooComplicatedToPreload);
  5318. }
  5319. case no_sub:
  5320. result = doBuildActivitySub(ctx, expr);
  5321. break;
  5322. case no_sort:
  5323. case no_cosort:
  5324. case no_topn:
  5325. case no_assertsorted:
  5326. result = doBuildActivitySort(ctx, expr);
  5327. break;
  5328. case no_dedup:
  5329. result = doBuildActivityDedup(ctx, expr);
  5330. break;
  5331. case no_enth:
  5332. result = doBuildActivityEnth(ctx, expr);
  5333. break;
  5334. case no_pipe:
  5335. result = doBuildActivityPipeThrough(ctx, expr);
  5336. break;
  5337. case no_sample:
  5338. result = doBuildActivitySample(ctx, expr);
  5339. break;
  5340. case no_filter:
  5341. result = doBuildActivityFilter(ctx, expr);
  5342. break;
  5343. case no_limit:
  5344. result = doBuildActivityLimit(ctx, expr);
  5345. break;
  5346. case no_catchds:
  5347. result = doBuildActivityCatch(ctx, expr);
  5348. break;
  5349. case no_keyedlimit:
  5350. {
  5351. traceExpression("keyed fail", expr);
  5352. StringBuffer s;
  5353. getExprECL(expr->queryChild(1), s);
  5354. throwError1(HQLERR_KeyedLimitNotKeyed, s.str());
  5355. }
  5356. case no_index:
  5357. case no_selectnth:
  5358. result = doBuildActivitySelectNth(ctx, expr);
  5359. break;
  5360. case no_join:
  5361. case no_selfjoin:
  5362. result = doBuildActivityJoin(ctx, expr);
  5363. break;
  5364. case no_fetch:
  5365. case no_compound_fetch:
  5366. result = doBuildActivityFetch(ctx, expr);
  5367. break;
  5368. case no_rollup:
  5369. result = doBuildActivityRollup(ctx, expr);
  5370. break;
  5371. case no_hqlproject:
  5372. case no_transformascii:
  5373. case no_transformebcdic:
  5374. case no_projectrow:
  5375. result = doBuildActivityProject(ctx, expr);
  5376. break;
  5377. case no_createrow:
  5378. result = doBuildActivityCreateRow(ctx, expr, false);
  5379. break;
  5380. case no_newusertable:
  5381. result = doBuildActivityProject(ctx, expr);
  5382. break;
  5383. case no_aggregate:
  5384. case no_newaggregate:
  5385. case no_throughaggregate:
  5386. result = doBuildActivityAggregate(ctx, expr);
  5387. break;
  5388. case no_metaactivity:
  5389. result = doBuildActivityMetaActivity(ctx, expr);
  5390. break;
  5391. case no_choosen:
  5392. result = doBuildActivityFirstN(ctx, expr);
  5393. break;
  5394. case no_choosesets:
  5395. result = doBuildActivityChooseSets(ctx, expr);
  5396. break;
  5397. case no_newkeyindex:
  5398. case no_compound_indexread:
  5399. result = doBuildActivityIndexRead(ctx, expr);
  5400. break;
  5401. case no_compound_indexcount:
  5402. case no_compound_indexaggregate:
  5403. result = doBuildActivityIndexAggregate(ctx, expr);
  5404. break;
  5405. case no_compound_indexgroupaggregate:
  5406. result = doBuildActivityIndexGroupAggregate(ctx, expr);
  5407. break;
  5408. case no_compound_indexnormalize:
  5409. result = doBuildActivityIndexNormalize(ctx, expr);
  5410. break;
  5411. case no_compound:
  5412. buildStmt(ctx, expr->queryChild(0));
  5413. result = buildCachedActivity(ctx, expr->queryChild(1));
  5414. break;
  5415. case no_newparse:
  5416. result = doBuildActivityParse(ctx, expr);
  5417. break;
  5418. case no_newxmlparse:
  5419. result = doBuildActivityXmlParse(ctx, expr);
  5420. break;
  5421. case no_httpcall:
  5422. result = doBuildActivityHTTP(ctx, expr, (expr->isAction()), isRoot);
  5423. break;
  5424. case no_newsoapcall:
  5425. case no_newsoapcall_ds:
  5426. case no_newsoapaction_ds:
  5427. result = doBuildActivitySOAP(ctx, expr, (expr->isAction()), isRoot);
  5428. break;
  5429. case no_parallel:
  5430. case no_sequential:
  5431. case no_actionlist:
  5432. result = doBuildActivitySequentialParallel(ctx, expr, isRoot);
  5433. break;
  5434. case no_activerow:
  5435. {
  5436. OwnedHqlExpr row = createDatasetFromRow(LINK(expr));
  5437. return buildCachedActivity(ctx, row);
  5438. }
  5439. case no_assert_ds:
  5440. result = doBuildActivityAssert(ctx, expr);
  5441. break;
  5442. case no_loop:
  5443. case no_loop2:
  5444. result = doBuildActivityLoop(ctx, expr);
  5445. break;
  5446. case no_graphloop:
  5447. result = doBuildActivityGraphLoop(ctx, expr);
  5448. break;
  5449. case no_allnodes:
  5450. result = doBuildActivityRemote(ctx, expr, isRoot);
  5451. break;
  5452. case no_libraryselect:
  5453. result = doBuildActivityLibrarySelect(ctx, expr);
  5454. break;
  5455. case no_libraryscopeinstance:
  5456. result = doBuildActivityLibraryInstance(ctx, expr);
  5457. break;
  5458. case no_serialize:
  5459. case no_deserialize:
  5460. result = doBuildActivitySerialize(ctx, expr);
  5461. break;
  5462. case no_definesideeffect:
  5463. result = doBuildActivityDefineSideEffect(ctx, expr);
  5464. break;
  5465. case no_callsideeffect:
  5466. result = doBuildActivityCallSideEffect(ctx, expr);
  5467. break;
  5468. case no_executewhen:
  5469. result = doBuildActivityExecuteWhen(ctx, expr);
  5470. break;
  5471. case no_thor:
  5472. UNIMPLEMENTED;
  5473. break;
  5474. default:
  5475. if (expr->isAction())
  5476. return doBuildActivityAction(ctx, expr, isRoot, false);
  5477. if (expr->isDatarow())
  5478. {
  5479. OwnedHqlExpr row = createDatasetFromRow(LINK(expr));
  5480. return buildCachedActivity(ctx, row);
  5481. }
  5482. else
  5483. {
  5484. UNIMPLEMENTED_XY("Activity", getOpString(expr->getOperator()));
  5485. }
  5486. }
  5487. }
  5488. catch (IException * e)
  5489. {
  5490. if (dynamic_cast<IECLError *>(e))
  5491. throw;
  5492. IHqlExpression * location = queryActiveActivityLocation();
  5493. if (location)
  5494. {
  5495. IECLError * error = annotateExceptionWithLocation(e, location);
  5496. e->Release();
  5497. throw error;
  5498. }
  5499. throw;
  5500. }
  5501. return result;
  5502. }
  5503. ABoundActivity * HqlCppTranslator::buildCachedActivity(BuildCtx & ctx, IHqlExpression * expr, bool isRoot)
  5504. {
  5505. // if (isRoot && curActivityId > 400)
  5506. // return NULL;
  5507. switch (expr->getOperator())
  5508. {
  5509. case no_split:
  5510. case no_libraryscopeinstance:
  5511. {
  5512. ActivityAssociation * match = static_cast<ActivityAssociation *>(ctx.queryAssociation(expr, AssocActivity, NULL));
  5513. if (match)
  5514. return LINK(match->activity);
  5515. break;
  5516. }
  5517. case no_if:
  5518. {
  5519. if (options.recreateMapFromIf && !expr->isAction())
  5520. {
  5521. OwnedHqlExpr converted = combineIfsToMap(expr);
  5522. if (converted)
  5523. return buildCachedActivity(ctx, converted);
  5524. }
  5525. break;
  5526. }
  5527. }
  5528. //NB: ActivityExprStack is purely used for improving the error reporting
  5529. activityExprStack.append(*LINK(expr));
  5530. try
  5531. {
  5532. //Don't modify aliases in child queries - otherwise they can fail to match the aliases generated in the parent
  5533. OwnedHqlExpr optimized = insideChildGraph(ctx) ? LINK(expr) : optimizeActivityAliasReferences(expr);
  5534. ThorBoundActivity * bound = (ThorBoundActivity *)buildActivity(ctx, optimized, isRoot);
  5535. activityExprStack.pop();
  5536. ActivityAssociation * table = new ActivityAssociation(expr->queryBody(), bound);
  5537. ctx.associateOwn(*table);
  5538. return bound;
  5539. }
  5540. catch (IException *)
  5541. {
  5542. activityExprStack.pop();
  5543. throw;
  5544. }
  5545. }
  5546. void HqlCppTranslator::buildRootActivity(BuildCtx & ctx, IHqlExpression * expr)
  5547. {
  5548. switch (expr->getOperator())
  5549. {
  5550. case no_compound:
  5551. case no_parallel:
  5552. case no_actionlist:
  5553. {
  5554. ForEachChild(idx, expr)
  5555. buildRootActivity(ctx, expr->queryChild(idx));
  5556. break;
  5557. }
  5558. case no_null:
  5559. if (expr->isAction())
  5560. return;
  5561. //fall through
  5562. default:
  5563. {
  5564. WarningProcessor::OnWarningStateBlock saved(warningProcessor);
  5565. ::Release(buildCachedActivity(ctx, expr, true));
  5566. break;
  5567. }
  5568. }
  5569. }
  5570. void HqlCppTranslator::buildRecordSerializeExtract(BuildCtx & ctx, IHqlExpression * memoryRecord)
  5571. {
  5572. OwnedHqlExpr serializedRecord = getSerializedForm(memoryRecord);
  5573. OwnedHqlExpr serializedDataset = createDataset(no_null, LINK(serializedRecord));
  5574. OwnedHqlExpr memoryDataset = createDataset(no_anon, LINK(memoryRecord));
  5575. MetaInstance meta(*this, memoryDataset);
  5576. buildMetaInfo(meta);
  5577. if (recordTypesMatch(memoryRecord, serializedRecord))
  5578. {
  5579. StringBuffer s;
  5580. if (isFixedRecordSize(memoryRecord))
  5581. {
  5582. ctx.addQuoted(s.append("size32_t size = ").append(getFixedRecordSize(memoryRecord)).append(";"));
  5583. }
  5584. else
  5585. {
  5586. ctx.addQuoted(s.append("size32_t size = ").append(meta.queryInstanceObject()).append(".getRecordSize(_left);"));
  5587. }
  5588. ctx.addQuoted("byte * self = crSelf.ensureCapacity(size, NULL);");
  5589. ctx.addQuoted("memcpy(crSelf.row(), _left, size);");
  5590. ctx.addQuoted("return size;");
  5591. }
  5592. else
  5593. {
  5594. ctx.addQuoted("const byte * left = (const byte *)_left;");
  5595. BoundRow * self = bindSelf(ctx, serializedDataset, "crSelf");
  5596. BoundRow * left = bindTableCursor(ctx, memoryDataset, "left");
  5597. OwnedHqlExpr rhs = ensureActiveRow(left->querySelector());
  5598. OwnedHqlExpr serializedRow = ::ensureSerialized(rhs);
  5599. buildAssign(ctx, self->querySelector(), serializedRow);
  5600. buildReturnRecordSize(ctx, self);
  5601. }
  5602. }
  5603. //---------------------------------------------------------------------------
  5604. BoundRow * HqlCppTranslator::bindTableCursor(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * bound, node_operator side, IHqlExpression * selSeq)
  5605. {
  5606. BoundRow * cursor = createTableCursor(dataset, bound, side, selSeq);
  5607. ctx.associateOwn(*cursor);
  5608. return cursor;
  5609. }
  5610. BoundRow * HqlCppTranslator::bindTableCursor(BuildCtx & ctx, IHqlExpression * dataset, const char * name, bool isLinkCounted, node_operator side, IHqlExpression * selSeq)
  5611. {
  5612. Owned<ITypeInfo> type = makeRowReferenceType(NULL);
  5613. if (isLinkCounted)
  5614. type.setown(makeAttributeModifier(type.getClear(), getLinkCountedAttr()));
  5615. Owned<IHqlExpression> bound = createVariable(name, type.getClear());
  5616. // Owned<IHqlExpression> bound = createVariable(name, makeRowReferenceType(dataset));
  5617. return bindTableCursor(ctx, dataset, bound, side, selSeq);
  5618. }
  5619. BoundRow * HqlCppTranslator::rebindTableCursor(BuildCtx & ctx, IHqlExpression * dataset, BoundRow * row, node_operator side, IHqlExpression * selSeq)
  5620. {
  5621. BoundRow * cursor = recreateTableCursor(dataset, row, side, selSeq);
  5622. ctx.associateOwn(*cursor);
  5623. return cursor;
  5624. }
  5625. BoundRow * HqlCppTranslator::createTableCursor(IHqlExpression * dataset, IHqlExpression * bound, node_operator side, IHqlExpression * selSeq)
  5626. {
  5627. return new BoundRow(dataset, bound, queryRecordOffsetMap(dataset->queryRecord()), side, selSeq);
  5628. }
  5629. BoundRow * HqlCppTranslator::recreateTableCursor(IHqlExpression * dataset, BoundRow * row, node_operator side, IHqlExpression * selSeq)
  5630. {
  5631. return new BoundRow(row, dataset, side, selSeq);
  5632. }
  5633. BoundRow * HqlCppTranslator::createTableCursor(IHqlExpression * dataset, const char * name, bool isLinkCounted, node_operator side, IHqlExpression * selSeq)
  5634. {
  5635. Owned<ITypeInfo> type = makeRowReferenceType(NULL);
  5636. if (isLinkCounted)
  5637. type.setown(makeAttributeModifier(type.getClear(), getLinkCountedAttr()));
  5638. Owned<IHqlExpression> bound = createVariable(name, type.getClear());
  5639. return createTableCursor(dataset, bound, side, selSeq);
  5640. }
  5641. BoundRow * HqlCppTranslator::bindXmlTableCursor(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * bound, node_operator side, IHqlExpression * selSeq, bool translateVirtuals)
  5642. {
  5643. Owned<ColumnToOffsetMap> xmlMap = new XmlColumnToOffsetMap(dataset->queryRecord(), getDefaultMaxRecordSize(), translateVirtuals);
  5644. xmlMap->init(recordMap);
  5645. BoundRow * cursor = new BoundRow(dataset, bound, xmlMap, side, selSeq);
  5646. ctx.associateOwn(*cursor);
  5647. return cursor;
  5648. }
  5649. BoundRow * HqlCppTranslator::bindXmlTableCursor(BuildCtx & ctx, IHqlExpression * dataset, const char * name, node_operator side, IHqlExpression * selSeq, bool translateVirtuals)
  5650. {
  5651. OwnedHqlExpr bound = createVariable(name, makeRowReferenceType(NULL));
  5652. // OwnedHqlExpr bound = createVariable(name, makeRowReferenceType(dataset));
  5653. return bindXmlTableCursor(ctx, dataset, bound, side, selSeq, translateVirtuals);
  5654. }
  5655. BoundRow * HqlCppTranslator::bindCsvTableCursor(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * bound, node_operator side, IHqlExpression * selSeq, bool translateVirtuals, _ATOM encoding)
  5656. {
  5657. Owned<ColumnToOffsetMap> csvMap = new CsvColumnToOffsetMap(dataset->queryRecord(), getDefaultMaxRecordSize(), translateVirtuals, encoding);
  5658. csvMap->init(recordMap);
  5659. BoundRow * cursor = new BoundRow(dataset, bound, csvMap, side, selSeq);
  5660. ctx.associateOwn(*cursor);
  5661. return cursor;
  5662. }
  5663. BoundRow * HqlCppTranslator::bindCsvTableCursor(BuildCtx & ctx, IHqlExpression * dataset, const char * name, node_operator side, IHqlExpression * selSeq, bool translateVirtuals, _ATOM encoding)
  5664. {
  5665. OwnedHqlExpr bound = createVariable(name, makeRowReferenceType(NULL));
  5666. // OwnedHqlExpr bound = createVariable(name, makeRowReferenceType(dataset));
  5667. return bindCsvTableCursor(ctx, dataset, bound, side, selSeq, translateVirtuals, encoding);
  5668. }
  5669. BoundRow * HqlCppTranslator::bindSelf(BuildCtx & ctx, IHqlExpression * dataset, const char * builderName)
  5670. {
  5671. StringBuffer bound;
  5672. bound.append(builderName).append(".row()");
  5673. BoundRow * row = bindTableCursor(ctx, dataset, bound, false, no_self, NULL);
  5674. OwnedHqlExpr builder = createVariable(builderName, makeBoolType());
  5675. row->setBuilder(builder);
  5676. return row;
  5677. }
  5678. BoundRow * HqlCppTranslator::bindSelf(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * expr, IHqlExpression * builder)
  5679. {
  5680. BoundRow * row = bindTableCursor(ctx, dataset, expr, no_self, NULL);
  5681. row->setBuilder(builder);
  5682. return row;
  5683. }
  5684. BoundRow * HqlCppTranslator::bindRow(BuildCtx & ctx, IHqlExpression * expr, IHqlExpression * bound)
  5685. {
  5686. BoundRow * row = createBoundRow(expr, bound);
  5687. ctx.associateOwn(*row);
  5688. return row;
  5689. }
  5690. BoundRow * HqlCppTranslator::bindRow(BuildCtx & ctx, IHqlExpression * expr, const char * name)
  5691. {
  5692. Owned<IHqlExpression> bound = createVariable(name, makeRowReferenceType(NULL));
  5693. return bindRow(ctx, expr, bound);
  5694. }
  5695. BoundRow * HqlCppTranslator::bindTableCursorOrRow(BuildCtx & ctx, IHqlExpression * expr, const char * name)
  5696. {
  5697. if (expr->getOperator() == no_activerow)
  5698. expr = expr->queryChild(0);
  5699. if (expr->isDatarow())
  5700. return bindRow(ctx, expr, name);
  5701. else
  5702. return bindTableCursor(ctx, expr, name);
  5703. }
  5704. BoundRow * HqlCppTranslator::createBoundRow(IHqlExpression * dataset, IHqlExpression * bound)
  5705. {
  5706. return new BoundRow(dataset->queryBody(), bound, queryRecordOffsetMap(dataset->queryRecord()));
  5707. }
  5708. BoundRow * HqlCppTranslator::bindSelectorAsSelf(BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * expr)
  5709. {
  5710. BoundRow * rootRow = selector->queryRootRow();
  5711. if (!rootRow->queryBuilder())
  5712. {
  5713. if (options.alwaysCreateRowBuilder || (options.supportDynamicRows && !isFixedWidthDataset(rootRow->queryRecord())))
  5714. UNIMPLEMENTED_X("expected a row builder");
  5715. }
  5716. if (selector->isRoot())
  5717. {
  5718. if (rootRow->querySide() == no_self)
  5719. {
  5720. ctx.associate(*rootRow);
  5721. return rootRow;
  5722. }
  5723. return bindSelf(ctx, expr, rootRow->queryBound(), rootRow->queryBuilder());
  5724. }
  5725. //Need to bind a delta address to a new variable.
  5726. // throwUnexpected(); // check this is actually called
  5727. CHqlBoundExpr offset;
  5728. selector->getOffset(ctx, offset);
  5729. CHqlBoundExpr address;
  5730. selector->buildAddress(ctx, address);
  5731. OwnedHqlExpr row = createValue(no_typetransfer, makeReferenceModifier(LINK(selector->queryType())), LINK(address.expr));
  5732. unsigned trailingFixed = selector->getContainerTrailingFixed();
  5733. StringBuffer builderName;
  5734. {
  5735. getUniqueId(builderName.append("b"));
  5736. StringBuffer s;
  5737. s.append("RtlNestedRowBuilder ").append(builderName).append("(");
  5738. generateExprCpp(s, rootRow->queryBuilder()).append(",");
  5739. generateExprCpp(s, offset.expr).append(",").append(trailingFixed).append(");");
  5740. ctx.addQuoted(s);
  5741. }
  5742. BoundRow * selfRow = bindSelf(ctx, expr, builderName);
  5743. selfRow->setAlias(selector);
  5744. return selfRow;
  5745. }
  5746. void HqlCppTranslator::finishSelf(BuildCtx & ctx, BoundRow * self, BoundRow * target)
  5747. {
  5748. if (target)
  5749. {
  5750. OwnedHqlExpr sizeofSelf = createSizeof(self->querySelector());
  5751. CHqlBoundExpr bound;
  5752. buildExpr(ctx, sizeofSelf, bound);
  5753. OwnedHqlExpr sizeofTarget = createSizeof(target->querySelector());
  5754. ctx.associateExpr(sizeofTarget, bound);
  5755. }
  5756. ctx.removeAssociation(self);
  5757. }
  5758. void HqlCppTranslator::ensureRowAllocated(BuildCtx & ctx, const char * builder)
  5759. {
  5760. StringBuffer s;
  5761. s.append(builder).append(".getSelf();");
  5762. ctx.addQuoted(s);
  5763. }
  5764. void HqlCppTranslator::ensureRowAllocated(BuildCtx & ctx, BoundRow * row)
  5765. {
  5766. assertex(row->queryBuilder());
  5767. StringBuffer s;
  5768. generateExprCpp(s, row->queryBuilder()).append(".getSelf();");
  5769. ctx.addQuoted(s);
  5770. }
  5771. //---------------------------------------------------------------------------
  5772. BoundRow * HqlCppTranslator::bindSelectorAsRootRow(BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * expr)
  5773. {
  5774. BoundRow * rootRow = selector->queryRootRow();
  5775. assertex(!rootRow->queryBuilder());
  5776. if (selector->isRoot())
  5777. {
  5778. ctx.associate(*rootRow);
  5779. return rootRow;
  5780. }
  5781. //Need to bind a delta address to a new variable.
  5782. CHqlBoundExpr address;
  5783. selector->buildAddress(ctx, address);
  5784. OwnedHqlExpr row = createValue(no_typetransfer, makeReferenceModifier(LINK(selector->queryType())), LINK(address.expr));
  5785. BoundRow * childRow = bindTableCursor(ctx, expr, row);
  5786. childRow->setAlias(selector);
  5787. return childRow;
  5788. }
  5789. //---------------------------------------------------------------------------
  5790. BoundRow * HqlCppTranslator::resolveSelectorDataset(BuildCtx & ctx, IHqlExpression * dataset)
  5791. {
  5792. return static_cast<BoundRow *>(ctx.queryAssociation(dataset->queryNormalizedSelector(), AssocCursor, NULL));
  5793. }
  5794. //---------------------------------------------------------------------------
  5795. void HqlCppTranslator::addDependency(BuildCtx & ctx, ABoundActivity * element, ABoundActivity * dependent, _ATOM kind, const char * label, int whenId)
  5796. {
  5797. ABoundActivity * sourceActivity = element;
  5798. ABoundActivity * sinkActivity = dependent;
  5799. unsigned outputIndex = 0;
  5800. if (kind != childAtom)
  5801. outputIndex = sourceActivity->nextOutputCount();
  5802. StringBuffer idText;
  5803. idText.append(sourceActivity->queryActivityId()).append('_').append(sinkActivity->queryActivityId());
  5804. #if 0
  5805. StringBuffer edgeText;
  5806. edgeText.append("edge[@id=\").append(idText).append("\"]");
  5807. if (graph->hasProp(edgePath))
  5808. return;
  5809. #endif
  5810. // if (outputIndex)
  5811. // idText.append("_").append(outputIndex);
  5812. IPropertyTree *edge = createPTree();
  5813. edge->setProp("@id", idText.str());
  5814. edge->setPropInt64("@target", sinkActivity->queryGraphId());
  5815. edge->setPropInt64("@source", sourceActivity->queryGraphId());
  5816. if (targetHThor())
  5817. {
  5818. if (sinkActivity->queryGraphId() == sourceActivity->queryGraphId())
  5819. throwError1(HQLERR_DependencyWithinGraph, sinkActivity->queryGraphId());
  5820. }
  5821. if (label)
  5822. edge->setProp("@label", label);
  5823. if (targetRoxie())
  5824. {
  5825. if (outputIndex)
  5826. addGraphAttributeInt(edge, "_sourceIndex", outputIndex);
  5827. }
  5828. if (kind == dependencyAtom)
  5829. addGraphAttributeBool(edge, "_dependsOn", true);
  5830. else if (kind == childAtom)
  5831. addGraphAttributeBool(edge, "_childGraph", true);
  5832. if (whenId)
  5833. addGraphAttributeInt(edge, "_when", whenId);
  5834. addGraphAttributeInt(edge, "_sourceActivity", sourceActivity->queryActivityId());
  5835. addGraphAttributeInt(edge, "_targetActivity", sinkActivity->queryActivityId());
  5836. graph->addPropTree("edge", edge);
  5837. }
  5838. void HqlCppTranslator::buildClearRecord(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * record, int direction)
  5839. {
  5840. Owned<IReferenceSelector> selector = buildActiveRow(ctx, dataset);
  5841. selector->buildClear(ctx, direction);
  5842. }
  5843. IHqlExpression * HqlCppTranslator::getClearRecordFunction(IHqlExpression * record, int direction)
  5844. {
  5845. IHqlExpression * dirExpr = getSizetConstant((size32_t)direction);
  5846. OwnedHqlExpr search = createAttribute(__clearHelperAtom, LINK(record->queryBody()), dirExpr);
  5847. BuildCtx declarectx(*code, declareAtom);
  5848. HqlExprAssociation * match = declarectx.queryMatchExpr(search);
  5849. if (match)
  5850. return LINK(match->queryExpr());
  5851. StringBuffer functionName;
  5852. getUniqueId(functionName.append("clearRow"));
  5853. BuildCtx clearctx(declarectx);
  5854. StringBuffer s;
  5855. s.append("size32_t ").append(functionName).append("(ARowBuilder & crSelf, IResourceContext * ctx)");
  5856. clearctx.setNextPriority(RowMetaPrio);
  5857. if (record)
  5858. {
  5859. IHqlStmt * func = clearctx.addQuotedCompound(s);
  5860. func->setIncomplete(true);
  5861. OwnedHqlExpr dataset = createDataset(no_anon, LINK(record));
  5862. BoundRow * cursor = bindSelf(clearctx, dataset, "crSelf");
  5863. ensureRowAllocated(clearctx, "crSelf");
  5864. buildClearRecord(clearctx, cursor->querySelector(), record, direction);
  5865. buildReturnRecordSize(clearctx, cursor);
  5866. func->setIncomplete(false);
  5867. }
  5868. else
  5869. clearctx.addQuotedCompound(s.append(" {}"));
  5870. if (options.spanMultipleCpp)
  5871. {
  5872. s.clear().append("extern size32_t ").append(functionName).append("(ARowBuilder & crSelf, IResourceContext * ctx);");
  5873. BuildCtx protoctx(*code, mainprototypesAtom);
  5874. protoctx.addQuoted(s);
  5875. }
  5876. OwnedHqlExpr temp = createVariable(functionName, makeVoidType());
  5877. declarectx.associateExpr(search, temp);
  5878. return temp.getLink();
  5879. }
  5880. void HqlCppTranslator::buildClearRecordMember(BuildCtx & ctx, const char * name, IHqlExpression * dataset)
  5881. {
  5882. BuildCtx clearCtx(ctx);
  5883. StringBuffer s;
  5884. s.append("virtual size32_t createDefault").append(name).append("(ARowBuilder & crSelf)");
  5885. if (dataset && dataset->queryRecord()->numChildren())
  5886. {
  5887. OwnedHqlExpr func = getClearRecordFunction(dataset->queryRecord());
  5888. generateExprCpp(s.append(" { return "), func).append("(crSelf, ctx); }");
  5889. clearCtx.addQuoted(s);
  5890. }
  5891. else
  5892. clearCtx.addQuoted(s.append(" { return 0; }"));
  5893. }
  5894. void HqlCppTranslator::doBuildExprEvaluate(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt)
  5895. {
  5896. IHqlExpression * dataset = expr->queryChild(0);
  5897. IHqlExpression * field = expr->queryChild(1);
  5898. BoundRow * boundRow = resolveSelectorDataset(ctx, dataset);
  5899. if (boundRow)
  5900. {
  5901. //E.g. EVALUATE(LEFT, attribute). Need to make sure field refs are unambiguous.
  5902. BuildCtx subctx(ctx);
  5903. subctx.addGroup();
  5904. bindTableCursor(subctx, boundRow->queryDataset(), boundRow->queryBound());
  5905. buildExpr(subctx, field, tgt);
  5906. }
  5907. else
  5908. {
  5909. switch (dataset->queryType()->getTypeCode())
  5910. {
  5911. case type_row:
  5912. BuildCtx subctx(ctx);
  5913. subctx.addGroup();
  5914. Owned<IReferenceSelector> selector = buildNewRow(subctx, dataset);
  5915. Owned<BoundRow> boundRow = selector->getRow(subctx);
  5916. //subctx.associateOwn???
  5917. bindTableCursor(subctx, boundRow->queryDataset(), boundRow->queryBound());
  5918. buildExpr(subctx, field, tgt);
  5919. return;
  5920. }
  5921. throwError(HQLERR_EvaluateTableNotInScope);
  5922. }
  5923. }
  5924. void HqlCppTranslator::doBuildExprCounter(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt)
  5925. {
  5926. if (buildExprInCorrectContext(ctx, expr, tgt, false))
  5927. return;
  5928. throwError(HQLERR_CounterNotValid);
  5929. }
  5930. //---------------------------------------------------------------------------
  5931. void HqlCppTranslator::doBuildSerialize(BuildCtx & ctx, _ATOM name, IHqlExpression * length, CHqlBoundExpr & bound, const char * bufferName)
  5932. {
  5933. HqlExprArray args;
  5934. if (length)
  5935. args.append(*LINK(length));
  5936. else
  5937. {
  5938. args.append(*getBoundSize(bound));
  5939. }
  5940. args.append(*getPointer(bound.expr));
  5941. args.append(*createVariable(bufferName, makeBoolType()));
  5942. callProcedure(ctx, name, args);
  5943. }
  5944. void HqlCppTranslator::ensureSerialized(const CHqlBoundTarget & variable, BuildCtx & serializectx, BuildCtx & deserializectx, const char * inBufferName, const char * outBufferName)
  5945. {
  5946. CHqlBoundExpr value;
  5947. value.setFromTarget(variable);
  5948. while (isCast(value.expr))
  5949. value.expr.set(value.expr->queryChild(0));
  5950. ITypeInfo * type = value.expr->queryType();
  5951. if ((type->getSize() == UNKNOWN_LENGTH) || hasLinkCountedModifier(type) || hasWrapperModifier(type))
  5952. {
  5953. HqlExprArray serializeArgs;
  5954. serializeArgs.append(*value.getTranslatedExpr());
  5955. HqlExprArray deserializeArgs;
  5956. _ATOM serializeName, deserializeName;
  5957. OwnedITypeInfo serializedType;
  5958. type_t tc = type->getTypeCode();
  5959. switch (tc)
  5960. {
  5961. case type_varstring:
  5962. serializeName = serializeCStringXAtom;
  5963. deserializeName = deserializeCStringXAtom;
  5964. break;
  5965. case type_string:
  5966. serializeName = serializeStringXAtom;
  5967. deserializeName = deserializeStringXAtom;
  5968. break;
  5969. case type_data:
  5970. serializeName = serializeDataXAtom;
  5971. deserializeName = deserializeDataXAtom;
  5972. break;
  5973. case type_set:
  5974. serializeName = serializeSetAtom;
  5975. deserializeName = deserializeSetAtom;
  5976. break;
  5977. case type_qstring:
  5978. serializeName = serializeQStrXAtom;
  5979. deserializeName = deserializeQStrXAtom;
  5980. break;
  5981. case type_unicode:
  5982. serializeName = serializeUnicodeXAtom;
  5983. deserializeName = deserializeUnicodeXAtom;
  5984. break;
  5985. case type_varunicode:
  5986. serializeName = serializeUnicodeXAtom;
  5987. deserializeName = deserializeVUnicodeXAtom;
  5988. break;
  5989. case type_utf8:
  5990. serializeName = serializeUtf8XAtom;
  5991. deserializeName = deserializeUtf8XAtom;
  5992. break;
  5993. case type_table:
  5994. case type_groupedtable:
  5995. {
  5996. IHqlExpression * record = ::queryRecord(type);
  5997. if (hasLinkCountedModifier(type))
  5998. {
  5999. deserializeArgs.append(*createRowSerializer(deserializectx, record, deserializerAtom));
  6000. serializeArgs.append(*createRowSerializer(serializectx, record, serializerAtom));
  6001. if (tc == type_table)
  6002. {
  6003. serializeName = serializeRowsetXAtom;
  6004. deserializeName = deserializeRowsetXAtom;
  6005. }
  6006. else
  6007. {
  6008. serializeName = serializeGroupedRowsetXAtom;
  6009. deserializeName = deserializeGroupedRowsetXAtom;
  6010. }
  6011. }
  6012. else
  6013. {
  6014. assertex(!recordRequiresSerialization(record));
  6015. if (tc == type_table)
  6016. {
  6017. serializeName = serializeDatasetXAtom;
  6018. deserializeName = deserializeDatasetXAtom;
  6019. }
  6020. else
  6021. {
  6022. serializeName = serializeGroupedDatasetXAtom;
  6023. deserializeName = deserializeGroupedDatasetXAtom;
  6024. }
  6025. }
  6026. serializedType.set(type);
  6027. break;
  6028. }
  6029. case type_row:
  6030. {
  6031. IHqlExpression * record = ::queryRecord(type);
  6032. assertex(hasWrapperModifier(type));
  6033. serializeArgs.append(*createRowSerializer(serializectx, record, serializerAtom));
  6034. serializeArgs.append(*createVariable(outBufferName, makeBoolType()));
  6035. buildFunctionCall(serializectx, serializeRowAtom, serializeArgs);
  6036. deserializeArgs.append(*createRowAllocator(deserializectx, record));
  6037. deserializeArgs.append(*createRowSerializer(deserializectx, record, deserializerAtom));
  6038. deserializeArgs.append(*createVariable(inBufferName, makeBoolType()));
  6039. Owned<ITypeInfo> resultType = makeReferenceModifier(makeAttributeModifier(makeRowType(record->getType()), getLinkCountedAttr()));
  6040. OwnedHqlExpr call = bindFunctionCall(deserializeRowAtom, deserializeArgs, resultType);
  6041. buildExprAssign(deserializectx, variable, call);
  6042. return;
  6043. }
  6044. default:
  6045. UNIMPLEMENTED;
  6046. }
  6047. serializeArgs.append(*createVariable(outBufferName, makeBoolType()));
  6048. deserializeArgs.append(*createVariable(inBufferName, makeBoolType()));
  6049. buildFunctionCall(serializectx, serializeName, serializeArgs);
  6050. OwnedHqlExpr deserializeCall = bindFunctionCall(deserializeName, deserializeArgs, serializedType);
  6051. buildExprAssign(deserializectx, variable, deserializeCall);
  6052. }
  6053. else
  6054. {
  6055. OwnedHqlExpr length;
  6056. switch (type->getTypeCode())
  6057. {
  6058. case type_int:
  6059. case type_swapint:
  6060. case type_packedint:
  6061. switch (type->getSize())
  6062. {
  6063. case 3: case 5: case 6: case 7:
  6064. if (isLittleEndian(type))
  6065. buildClear(deserializectx, variable);
  6066. else
  6067. {
  6068. unsigned newSize = (type->getSize() == 3) ? 4 : 8;
  6069. length.setown(getSizetConstant(newSize));
  6070. }
  6071. break;
  6072. }
  6073. break;
  6074. case type_row:
  6075. {
  6076. //MORE: This will cause problems if rows are dynamically allocated
  6077. if (hasReferenceModifier(type))
  6078. length.setown(getSizetConstant(sizeof(void*)));
  6079. else
  6080. {
  6081. IHqlExpression * record = ::queryRecord(type);
  6082. ColumnToOffsetMap * map = queryRecordOffsetMap(record);
  6083. length.setown(getSizetConstant(map->getMaxSize()));
  6084. }
  6085. break;
  6086. }
  6087. case type_bitfield:
  6088. UNIMPLEMENTED;
  6089. }
  6090. doBuildSerialize(serializectx, serializeRawAtom, length, value, outBufferName);
  6091. doBuildSerialize(deserializectx, deserializeRawAtom, length, value, inBufferName);
  6092. }
  6093. }
  6094. void HqlCppTranslator::ensureSerialized(BuildCtx & ctx, const CHqlBoundTarget & variable)
  6095. {
  6096. EvalContext * instance = queryEvalContext(ctx);
  6097. assertex(instance);
  6098. instance->tempCompatiablityEnsureSerialized(variable);
  6099. }
  6100. bool HqlCppTranslator::checkGetResultContext(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt)
  6101. {
  6102. IHqlExpression * seq = queryPropertyChild(expr, sequenceAtom, 0);
  6103. IHqlExpression * name = queryPropertyChild(expr, namedAtom, 0);
  6104. if (!name)
  6105. name = queryPropertyChild(expr, nameAtom, 0);
  6106. if (!contextAvailable)
  6107. {
  6108. StringBuffer s;
  6109. getStoredDescription(s, seq, name, true);
  6110. ::throwError1(HQLERR_InvalidAcessStoredVariable, s.str());
  6111. }
  6112. if (!insideOnCreate(ctx) && !ctx.queryMatchExpr(globalContextMarkerExpr) && !matchesConstantValue(seq, ResultSequenceOnce))
  6113. {
  6114. if (queryEvalContext(ctx))
  6115. {
  6116. doBuildAliasValue(ctx, expr, tgt);
  6117. return true;
  6118. }
  6119. StringBuffer s;
  6120. getStoredDescription(s, seq, name, true);
  6121. ::throwError1(HQLERR_CannotAccessStoredVariable, s.str());
  6122. }
  6123. if (ctx.getMatchExpr(expr, tgt))
  6124. return true;
  6125. //Use top activity, rather than queryCurrentActivity() - since we want the dependency to the child (for graph display)
  6126. if (activeActivities.ordinality())
  6127. queryAddResultDependancy(activeActivities.tos(), seq, name);
  6128. else if (name && targetRoxie())
  6129. {
  6130. OwnedHqlExpr attr = createResultAttribute(seq, name);
  6131. registerGlobalUsage(attr);
  6132. }
  6133. return false;
  6134. }
  6135. void HqlCppTranslator::doBuildExprGetResult(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt)
  6136. {
  6137. if (checkGetResultContext(ctx, expr, tgt))
  6138. return;
  6139. ITypeInfo * exprType = expr->queryType();
  6140. ExpressionFormat format = (hasLinkCountedModifier(exprType) || options.tempDatasetsUseLinkedRows) ? FormatLinkedDataset : FormatBlockedDataset;
  6141. CHqlBoundTarget tempTarget;
  6142. createTempFor(ctx, exprType, tempTarget, typemod_none, format);
  6143. buildGetResultInfo(ctx, expr, NULL, &tempTarget);
  6144. tgt.setFromTarget(tempTarget);
  6145. ctx.associateExpr(expr, tgt);
  6146. }
  6147. void HqlCppTranslator::doBuildAssignGetResult(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr)
  6148. {
  6149. CHqlBoundExpr bound;
  6150. if (checkGetResultContext(ctx, expr, bound))
  6151. {
  6152. assign(ctx, target, bound);
  6153. return;
  6154. }
  6155. buildGetResultInfo(ctx, expr, NULL, &target);
  6156. }
  6157. void HqlCppTranslator::pushCluster(BuildCtx & ctx, IHqlExpression * cluster, StringAttr & savedCluster)
  6158. {
  6159. savedCluster.set(curCluster);
  6160. HqlExprArray args;
  6161. args.append(*LINK(cluster));
  6162. callProcedure(ctx, selectClusterAtom, args);
  6163. StringBuffer clusterText;
  6164. cluster->queryValue()->getStringValue(clusterText);
  6165. ctxCallback->noteCluster(clusterText.str());
  6166. curCluster.set(clusterText.str());
  6167. }
  6168. void HqlCppTranslator::popCluster(BuildCtx & ctx, const char * savedCluster)
  6169. {
  6170. HqlExprArray args;
  6171. callProcedure(ctx, restoreClusterAtom, args);
  6172. curCluster.set(savedCluster);
  6173. }
  6174. void HqlCppTranslator::doBuildStmtSetResult(BuildCtx & ctx, IHqlExpression * expr)
  6175. {
  6176. IHqlExpression * seq = queryPropertyChild(expr, sequenceAtom, 0);
  6177. IHqlExpression * name = queryPropertyChild(expr, namedAtom, 0);
  6178. IHqlExpression * persist = expr->queryProperty(_workflowPersist_Atom);
  6179. IHqlExpression * cluster = expr->queryProperty(clusterAtom);
  6180. BuildCtx subctx(ctx);
  6181. LinkedHqlExpr value;
  6182. if (expr->getOperator() == no_extractresult)
  6183. {
  6184. IHqlExpression * ds = expr->queryChild(0);
  6185. OwnedHqlExpr row = removeDatasetWrapper(ds);
  6186. OwnedHqlExpr newDs = (row->getOperator() == no_activerow) ? LINK(row) : createRow(no_newrow, LINK(row));
  6187. value.setown(replaceSelector(expr->queryChild(1), ds, newDs));
  6188. }
  6189. else
  6190. value.set(expr->queryChild(0));
  6191. if (matchesConstantValue(seq, ResultSequenceStored) || matchesConstantValue(seq, ResultSequencePersist))
  6192. {
  6193. StringBuffer text;
  6194. text.append("Create ");
  6195. getStoredDescription(text, seq, name, true);
  6196. graphLabel.set(text.str());
  6197. }
  6198. StringAttr prevClusterName;
  6199. if (cluster)
  6200. pushCluster(subctx, cluster->queryChild(0), prevClusterName);
  6201. switch (value->queryType()->getTypeCode())
  6202. {
  6203. case type_void:
  6204. {
  6205. buildStmt(subctx, value);
  6206. IHqlExpression * result = queryBoolExpr(true);
  6207. if (expr->queryProperty(checkpointAtom))
  6208. {
  6209. IHqlExpression * search = value;
  6210. if (search->getOperator() == no_thor)
  6211. search = search->queryChild(0);
  6212. if ((search->getOperator() == no_output) && search->queryChild(1))
  6213. {
  6214. BuildCtx atendctx(*code, goAtom);
  6215. atendctx.setNextDestructor();
  6216. HqlExprArray args;
  6217. args.append(*LINK(search->queryChild(1)));
  6218. callProcedure(atendctx, deleteFileAtom, args);
  6219. }
  6220. }
  6221. if (!expr->hasProperty(noSetAtom))
  6222. buildSetResultInfo(subctx, expr, result, NULL, (persist != NULL), false);
  6223. }
  6224. break;
  6225. case type_set:
  6226. {
  6227. ITypeInfo * setType = NULL;
  6228. IHqlExpression * original = queryPropertyChild(expr, _original_Atom, 0);
  6229. if (original)
  6230. setType = original->queryType();
  6231. OwnedHqlExpr normalized = normalizeListCasts(value);
  6232. buildSetResultInfo(subctx, expr, normalized, setType, (persist != NULL), true);
  6233. break;
  6234. }
  6235. case type_table:
  6236. case type_groupedtable:
  6237. switch (value->getOperator())
  6238. {
  6239. case no_null:
  6240. {
  6241. HqlExprArray args;
  6242. args.append(*createResultName(name, false));
  6243. args.append(*LINK(seq));
  6244. args.append(*createValue(no_translated, makeSetType(NULL), createValue(no_nullptr, makeSetType(NULL)), getSizetConstant(0)));
  6245. args.append(*createTranslatedOwned(createValue(no_nullptr, makeBoolType())));
  6246. buildFunctionCall(subctx, setResultSetAtom, args);
  6247. Owned<IWUResult> result = createDatasetResultSchema(seq, name, value->queryRecord(), true, false);
  6248. break;
  6249. }
  6250. default:
  6251. assertex(!"Should never occur - should have been transformed to an OUTPUT()");
  6252. }
  6253. break;
  6254. default:
  6255. buildSetResultInfo(subctx, expr, value, NULL, (persist != NULL), true);
  6256. break;
  6257. }
  6258. if (cluster)
  6259. popCluster(subctx, prevClusterName);
  6260. if (matchesConstantValue(seq, ResultSequenceStored) || matchesConstantValue(seq, ResultSequencePersist))
  6261. graphLabel.clear();
  6262. }
  6263. static bool isFilePersist(IHqlExpression * expr)
  6264. {
  6265. loop
  6266. {
  6267. switch (expr->getOperator())
  6268. {
  6269. case no_thor:
  6270. expr = expr->queryChild(0);
  6271. break;
  6272. case no_compound:
  6273. expr = expr->queryChild(1);
  6274. break;
  6275. case no_output:
  6276. return (queryRealChild(expr, 1) != NULL);
  6277. case no_actionlist:
  6278. expr = expr->queryChild(expr->numChildren()-1);
  6279. break;
  6280. default:
  6281. return false;
  6282. }
  6283. }
  6284. }
  6285. IHqlExpression * HqlCppTranslator::calculatePersistInputCrc(BuildCtx & ctx, IHqlExpression * expr)
  6286. {
  6287. DependenciesUsed dependencies(true);
  6288. gatherDependencies(expr, dependencies, GatherAll);
  6289. dependencies.removeInternalReads();
  6290. return calculatePersistInputCrc(ctx, dependencies);
  6291. }
  6292. IHqlExpression * HqlCppTranslator::calculatePersistInputCrc(BuildCtx & ctx, DependenciesUsed & dependencies)
  6293. {
  6294. Owned<ITypeInfo> crcType = makeIntType(8, false);
  6295. OwnedHqlExpr zero = createNullExpr(crcType);
  6296. if ((dependencies.tablesRead.ordinality() == 0) && (dependencies.resultsRead.ordinality() == 0))
  6297. return zero.getClear();
  6298. OwnedHqlExpr crcExpr = ctx.getTempDeclare(crcType, zero);
  6299. ForEachItemIn(idx1, dependencies.tablesRead)
  6300. {
  6301. IHqlExpression & cur = dependencies.tablesRead.item(idx1);
  6302. HqlExprArray args;
  6303. args.append(OLINK(cur));
  6304. args.append(*LINK(crcExpr));
  6305. OwnedHqlExpr function = bindFunctionCall(getDatasetHashAtom, args);
  6306. buildAssignToTemp(ctx, crcExpr, function);
  6307. }
  6308. ForEachItemIn(idx2, dependencies.resultsRead)
  6309. {
  6310. IHqlExpression & cur = dependencies.resultsRead.item(idx2);
  6311. IHqlExpression * seq = cur.queryChild(0);
  6312. IHqlExpression * name = cur.queryChild(1);
  6313. //Not sure if we need to do this if the result is internal. Leave on for the moment.
  6314. //if (seq->queryValue()->getIntValue() != ResultSequenceInternal)
  6315. bool expandLogical = matchesConstantValue(seq, ResultSequencePersist) && !cur.hasProperty(_internal_Atom);
  6316. HqlExprArray args;
  6317. args.append(*createResultName(name, expandLogical));
  6318. args.append(*LINK(seq));
  6319. OwnedHqlExpr call = bindFunctionCall(getResultHashAtom, args);
  6320. OwnedHqlExpr value = createValue(no_bxor, crcExpr->getType(), LINK(crcExpr), ensureExprType(call, crcExpr->queryType()));
  6321. buildAssignToTemp(ctx, crcExpr, value);
  6322. }
  6323. return crcExpr.getClear();
  6324. }
  6325. void HqlCppTranslator::doBuildStmtEnsureResult(BuildCtx & ctx, IHqlExpression * expr)
  6326. {
  6327. IHqlExpression * value = expr->queryChild(0);
  6328. IHqlExpression * seq = queryPropertyChild(expr, sequenceAtom, 0);
  6329. IHqlExpression * name = queryPropertyChild(expr, namedAtom, 0);
  6330. OwnedHqlExpr resultName = ::createResultName(name);
  6331. resultName.setown(ensureExprType(resultName, unknownVarStringType));
  6332. HqlExprArray args;
  6333. args.append(*LINK(resultName));
  6334. args.append(*LINK(seq));
  6335. OwnedHqlExpr checkExists = createValue(no_not, makeBoolType(), bindFunctionCall(isResultAtom, args));
  6336. if ((value->getOperator() == no_thor) && (value->queryChild(0)->getOperator() == no_output))
  6337. {
  6338. IHqlExpression * filename = queryRealChild(value->queryChild(0), 1);
  6339. if (filename)
  6340. {
  6341. args.append(*LINK(filename));
  6342. IHqlExpression * fileExists = createValue(no_not, makeBoolType(), bindFunctionCall(fileExistsAtom, args));
  6343. checkExists.setown(createBoolExpr(no_or, checkExists.getClear(), fileExists));
  6344. }
  6345. }
  6346. BuildCtx subctx(ctx);
  6347. buildFilter(subctx, checkExists);
  6348. doBuildStmtSetResult(subctx, expr);
  6349. }
  6350. //---------------------------------------------------------------------------
  6351. void HqlCppTranslator::doBuildEvalOnce(BuildCtx & ctx, const CHqlBoundTarget * target, IHqlExpression * expr, CHqlBoundExpr * bound)
  6352. {
  6353. IHqlExpression * value = expr->queryChild(0);
  6354. CHqlBoundExpr result;
  6355. doBuildAliasValue(ctx, value, result);
  6356. if (target)
  6357. {
  6358. OwnedHqlExpr translated = result.getTranslatedExpr();
  6359. buildExprAssign(ctx, *target, translated);
  6360. }
  6361. else if (bound)
  6362. bound->set(result);
  6363. }
  6364. //---------------------------------------------------------------------------
  6365. class CHqlBoundTargetItem : public CInterface
  6366. {
  6367. public:
  6368. CHqlBoundTarget value;
  6369. };
  6370. static HqlTransformerInfo filterExtractorInfo("FilterExtractor");
  6371. class FilterExtractor : public NewHqlTransformer
  6372. {
  6373. public:
  6374. FilterExtractor(HqlCppTranslator & _translator, BuildCtx & _classctx, BuildCtx & _ctx, IHqlExpression * _rootTable)
  6375. : NewHqlTransformer(filterExtractorInfo), translator(_translator), classctx(_classctx), ctx(_ctx) { rootTable.set(_rootTable->queryNormalizedSelector()); }
  6376. void assignCursors(IHqlExpression * helper);
  6377. IHqlExpression * buildExtractLookupFields(IHqlExpression * expr);
  6378. IHqlExpression * createTransformed(IHqlExpression * expr);
  6379. protected:
  6380. struct AssignPairs
  6381. {
  6382. HqlExprArray from;
  6383. CIArrayOf<CHqlBoundTargetItem> to;
  6384. };
  6385. void addTable(IHqlExpression * expr);
  6386. IHqlExpression * extractExpr(IHqlExpression * expr, AssignPairs & assigns);
  6387. IHqlExpression * querySelectorTable(IHqlExpression * expr);
  6388. public:
  6389. HqlCppTranslator & translator;
  6390. BuildCtx & classctx;
  6391. BuildCtx & ctx;
  6392. CIArray tables;
  6393. AssignPairs ctxAssign;
  6394. HqlExprArray ctxAssignDirectFrom;
  6395. HqlExprArray ctxAssignDirectTo;
  6396. AssignPairs extractAssign;
  6397. HqlExprAttr rootTable;
  6398. };
  6399. void FilterExtractor::addTable(IHqlExpression * table)
  6400. {
  6401. if (table == rootTable)
  6402. return;
  6403. BoundRow * cursor;
  6404. if (table)
  6405. cursor = static_cast<BoundRow *>(ctx.queryAssociation(table, AssocCursor, NULL));
  6406. else
  6407. cursor = (BoundRow *)ctx.queryFirstAssociation(AssocCursor);
  6408. assertex(cursor);
  6409. if (tables.find(*cursor) == NotFound)
  6410. {
  6411. StringBuffer newCursorName;
  6412. translator.getUniqueId(newCursorName.append("cur"));
  6413. BoundRow * newCursor = translator.bindTableCursor(classctx, cursor->queryDataset(), newCursorName, cursor->querySide(), cursor->querySelSeq());
  6414. IHqlExpression * oldVar = cursor->queryBound();
  6415. IHqlExpression * newVar = newCursor->queryBound();
  6416. ctxAssignDirectFrom.append(*LINK(oldVar));
  6417. ctxAssignDirectTo.append(*LINK(newVar));
  6418. tables.append(*LINK(cursor));
  6419. StringBuffer s;
  6420. s.append("const unsigned char * ").append(newCursorName).append(";");
  6421. classctx.addQuoted(s);
  6422. }
  6423. }
  6424. IHqlExpression * FilterExtractor::extractExpr(IHqlExpression * expr, AssignPairs & assigns)
  6425. {
  6426. unsigned match = assigns.from.find(*expr);
  6427. if (match == NotFound)
  6428. {
  6429. CHqlBoundTargetItem & target = * new CHqlBoundTargetItem;
  6430. translator.createTempFor(classctx, expr, target.value);
  6431. assigns.from.append(*LINK(expr));
  6432. assigns.to.append(target);
  6433. translator.ensureSerialized(classctx, target.value);
  6434. match = assigns.to.ordinality()-1;
  6435. }
  6436. return assigns.to.item(match).value.getTranslatedExpr();
  6437. }
  6438. IHqlExpression * FilterExtractor::querySelectorTable(IHqlExpression * expr)
  6439. {
  6440. loop
  6441. {
  6442. switch (expr->getOperator())
  6443. {
  6444. case no_select:
  6445. expr = expr->queryChild(0);
  6446. break;
  6447. case no_left:
  6448. case no_right:
  6449. case no_self:
  6450. case no_top:
  6451. case no_activetable:
  6452. return expr;
  6453. default:
  6454. if (expr->isDataset() || expr->isDatarow())
  6455. return expr->queryNormalizedSelector();
  6456. return NULL;
  6457. }
  6458. }
  6459. }
  6460. IHqlExpression * FilterExtractor::createTransformed(IHqlExpression * expr)
  6461. {
  6462. switch (expr->getOperator())
  6463. {
  6464. case no_field:
  6465. return LINK(expr);
  6466. case no_select:
  6467. {
  6468. assertex(!expr->hasProperty(newAtom));
  6469. if (querySelectorTable(expr) == rootTable)
  6470. return LINK(expr);
  6471. addTable(querySelectorTable(expr));
  6472. return extractExpr(expr, extractAssign);
  6473. }
  6474. case no_counter:
  6475. return extractExpr(expr, ctxAssign);
  6476. }
  6477. return NewHqlTransformer::createTransformed(expr);
  6478. }
  6479. IHqlExpression * FilterExtractor::buildExtractLookupFields(IHqlExpression * expr)
  6480. {
  6481. if (expr->getOperator() != no_filter)
  6482. return LINK(expr);
  6483. HqlExprArray args;
  6484. ForEachChild(idx, expr)
  6485. {
  6486. IHqlExpression * cur = expr->queryChild(idx);
  6487. if (idx == 0)
  6488. args.append(*LINK(cur));
  6489. else
  6490. args.append(*createTransformed(cur));
  6491. }
  6492. if ((ctxAssign.from.ordinality() == 0) && (ctxAssignDirectFrom.ordinality() == 0))
  6493. return LINK(expr);
  6494. //virtual void extractLookupFields() {}
  6495. BuildCtx funcctx(classctx);
  6496. funcctx.addQuotedCompound("virtual void extractLookupFields()");
  6497. ForEachItemIn(idx2, extractAssign.from)
  6498. translator.buildExprAssign(funcctx, extractAssign.to.item(idx2).value, &extractAssign.from.item(idx2));
  6499. return expr->clone(args);
  6500. }
  6501. void FilterExtractor::assignCursors(IHqlExpression * helper)
  6502. {
  6503. ForEachItemIn(idx1, ctxAssignDirectFrom)
  6504. {
  6505. OwnedHqlExpr target = createSelectExpr(LINK(helper), LINK(&ctxAssignDirectTo.item(idx1)), createAttribute(internalAtom));
  6506. ctx.addAssign(target, &ctxAssignDirectFrom.item(idx1));
  6507. }
  6508. ForEachItemIn(idx2, ctxAssign.from)
  6509. {
  6510. CHqlBoundTarget target;
  6511. target.set(ctxAssign.to.item(idx2).value);
  6512. if (target.length) target.length.setown(createSelectExpr(LINK(helper), LINK(target.length), createAttribute(internalAtom)));
  6513. if (target.expr) target.expr.setown(createSelectExpr(LINK(helper), LINK(target.expr), createAttribute(internalAtom)));
  6514. translator.buildExprAssign(ctx, target, &ctxAssign.from.item(idx2));
  6515. }
  6516. }
  6517. void HqlCppTranslator::doBuildExprCountIndex(BuildCtx & ctx, IHqlExpression * _expr, CHqlBoundExpr & tgt)
  6518. {
  6519. //NB: Parameter has already be converted to physical - i.e.,
  6520. //filepos has been removed, optimized and folded... I'm not completely sure that
  6521. //is correct....
  6522. LinkedHqlExpr expr = _expr;
  6523. bool needToCreateGraph = !graph;
  6524. if (needToCreateGraph)
  6525. {
  6526. beginGraph(ctx);
  6527. expr.setown(spotTableInvariantChildren(expr));
  6528. }
  6529. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKcountindex, expr, "CountIndex");
  6530. buildActivityFramework(instance);
  6531. buildInstancePrefix(instance);
  6532. IHqlExpression * filter = expr->queryChild(0);
  6533. loop
  6534. {
  6535. node_operator op = filter->getOperator();
  6536. if ((op != no_hqlproject) && (op != no_newusertable) && (op != no_compound_indexread))
  6537. break;
  6538. filter = filter->queryChild(0);
  6539. }
  6540. IHqlExpression * index = queryPhysicalRootTable(filter);
  6541. assertex(index == filter || index == filter->queryChild(0));
  6542. //virtual const char * getIndexFileName() = 0;
  6543. buildFilenameFunction(*instance, instance->startctx, "getIndexFileName", index->queryChild(3), hasDynamicFilename(index));
  6544. //virtual IOutputMetaData * queryIndexRecordSize() = 0; //Excluding fpos and sequence
  6545. buildMetaMember(instance->classctx, index, "queryIndexRecordSize");
  6546. //Now extract the context dependent information from the filter
  6547. FilterExtractor extractor(*this, instance->startctx, ctx, index);
  6548. OwnedHqlExpr newFilter = extractor.buildExtractLookupFields(filter);
  6549. MonitorExtractor monitors(index, *this, -(int)numPayloadFields(index), false);
  6550. monitors.extractAllFilters(newFilter);
  6551. // virtual void createSegmentMonitors(IIndexReadContext *irc)
  6552. BuildCtx createSegmentCtx(instance->startctx);
  6553. createSegmentCtx.addQuotedCompound("virtual void createSegmentMonitors(IIndexReadContext *irc)");
  6554. monitors.buildSegments(createSegmentCtx, "irc", false);
  6555. //virtual size32_t isValid(const void * src)
  6556. if (monitors.queryExtraFilter())
  6557. {
  6558. instance->classctx.addQuoted("virtual bool hasPostFilter() { return true; }");
  6559. BuildCtx validctx(instance->startctx);
  6560. validctx.addQuotedCompound("virtual size32_t isValid(const void * _key, unsigned __int64 _fpos, IBlobProvider * blobs)");
  6561. validctx.addQuoted("const unsigned char * key = (const unsigned char *) _key;");
  6562. OwnedHqlExpr fposField = createVariable("_fpos", makeIntType(8,false));
  6563. OwnedHqlExpr fposExpr = getFilepos(index, false);
  6564. IHqlExpression * key = index->queryNormalizedSelector();
  6565. OwnedHqlExpr boundSrc = createVariable("key", makeRowReferenceType(key));
  6566. HqlExprAttr srcRecord = index->queryChild(1);
  6567. bindTableCursor(validctx, key, boundSrc);
  6568. validctx.associateExpr(fposExpr, fposField);
  6569. buildReturn(validctx, monitors.queryExtraFilter());
  6570. }
  6571. if (monitors.queryGlobalGuard())
  6572. doBuildBoolFunction(instance->classctx, "canMatchAny", monitors.queryGlobalGuard());
  6573. if (targetRoxie())
  6574. instance->addAttributeBool("_isIndexOpt", index->hasProperty(optAtom));
  6575. buildInstanceSuffix(instance);
  6576. Owned<IWUActivity> activity = wu()->updateActivity(instance->activityId);
  6577. activity->setKind(TAKcountindex);
  6578. activity->setHelper(instance->factoryName);
  6579. // Create the helper, assign any cursors, and then call the count activity.
  6580. StringBuffer s,name;
  6581. getUniqueId(name.append("help"));
  6582. ctx.addQuoted(s.append(instance->className).append(" & ").append(name).append(" = * new ").append(instance->className).append(";"));
  6583. OwnedHqlExpr helper = createQuoted(name.str(), makeBoolType());
  6584. extractor.assignCursors(helper);
  6585. // Now call the count function - takes ownership of the helper.
  6586. HqlExprArray args;
  6587. args.append(*createConstant((__int64)instance->activityId));
  6588. args.append(*LINK(helper));
  6589. OwnedHqlExpr call = bindFunctionCall(countIndexAtom, args);
  6590. buildExpr(ctx, call, tgt);
  6591. if (needToCreateGraph)
  6592. endGraph();
  6593. }
  6594. void HqlCppTranslator::doBuildExprCountFile(BuildCtx & ctx, IHqlExpression * _expr, CHqlBoundExpr & tgt)
  6595. {
  6596. //NB: This is currently only applied to fixed width unfiltered files
  6597. LinkedHqlExpr expr = _expr;
  6598. bool needToCreateGraph = !graph;
  6599. if (needToCreateGraph)
  6600. {
  6601. beginGraph(ctx);
  6602. expr.setown(spotTableInvariantChildren(expr));
  6603. }
  6604. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKcountdisk, expr, "CountFile");
  6605. buildActivityFramework(instance);
  6606. buildInstancePrefix(instance);
  6607. IHqlExpression * dataset = expr->queryChild(0);
  6608. IHqlExpression * filename = dataset->queryChild(0);
  6609. noteResultAccessed(ctx, queryPropertyChild(dataset, sequenceAtom, 0), filename);
  6610. VirtualFieldsInfo info;
  6611. info.gatherVirtualFields(dataset->queryRecord(), dataset->hasProperty(_noVirtual_Atom), true);
  6612. OwnedHqlExpr physicalRecord = info.createPhysicalRecord();
  6613. assertex(isFixedRecordSize(physicalRecord));
  6614. OwnedHqlExpr physicalDataset = createDataset(no_anon, LINK(physicalRecord));
  6615. //virtual const char * getIndexFileName() = 0;
  6616. buildFilenameFunction(*instance, instance->startctx, "getFileName", filename, hasDynamicFilename(dataset));
  6617. //virtual IOutputMetaData * queryIndexRecordSize() = 0; //Excluding fpos and sequence
  6618. buildMetaMember(instance->classctx, physicalDataset, "queryRecordSize");
  6619. StringBuffer flags;
  6620. if (dataset->hasProperty(optAtom)) flags.append("|TDRoptional");
  6621. if (!filename->isConstant()) flags.append("|TDXvarfilename");
  6622. if (hasDynamicFilename(dataset)) flags.append("|TDXdynamicfilename");
  6623. if (dataset->hasProperty(_spill_Atom)) flags.append("|TDXtemporary");
  6624. if (dataset->hasProperty(jobTempAtom)) flags.append("|TDXjobtemp");
  6625. if (dataset->hasProperty(groupedAtom)) flags.append("|TDXgrouped");
  6626. if (dataset->hasProperty(__compressed__Atom)) flags.append("|TDXcompress");
  6627. if (dataset->hasProperty(_workflowPersist_Atom)) flags.append("|TDXupdateaccessed");
  6628. if (flags.length())
  6629. doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);
  6630. Owned<ABoundActivity> bound = instance->getBoundActivity();
  6631. addFileDependency(filename, bound);
  6632. buildInstanceSuffix(instance);
  6633. if (targetRoxie())
  6634. instance->addAttributeBool("_isOpt", dataset->hasProperty(optAtom));
  6635. //**** The following code should be commoned up. with the code above. Probably not
  6636. //**** worth doing until generalized remote child datasets come along
  6637. Owned<IWUActivity> activity = wu()->updateActivity(instance->activityId);
  6638. activity->setKind(TAKcountdisk);
  6639. activity->setHelper(instance->factoryName);
  6640. // Create the helper, assign any cursors, and then call the count activity.
  6641. StringBuffer s,name;
  6642. getUniqueId(name.append("help"));
  6643. ctx.addQuoted(s.append(instance->className).append(" & ").append(name).append(" = * new ").append(instance->className).append(";"));
  6644. OwnedHqlExpr helper = createQuoted(name.str(), makeBoolType());
  6645. // Now call the count function - takes ownership of the helper.
  6646. HqlExprArray args;
  6647. args.append(*createConstant((__int64)instance->activityId));
  6648. args.append(*LINK(helper));
  6649. OwnedHqlExpr call = bindFunctionCall(countRoxieDiskFileAtom, args);
  6650. buildExpr(ctx, call, tgt);
  6651. if (needToCreateGraph)
  6652. endGraph();
  6653. }
  6654. //---------------------------------------------------------------------------
  6655. void HqlCppTranslator::doBuildExprSizeof(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt)
  6656. {
  6657. if (ctx.getMatchExpr(expr, tgt))
  6658. return;
  6659. IHqlExpression * child = expr->queryChild(0);
  6660. IHqlExpression * limitExpr = expr->queryChild(1);
  6661. ITypeInfo * type = child->queryType();
  6662. if (expr->hasProperty(maxAtom))
  6663. {
  6664. if (type)
  6665. {
  6666. unsigned size = UNKNOWN_LENGTH;
  6667. switch (type->getTypeCode())
  6668. {
  6669. case type_table:
  6670. case type_groupedtable:
  6671. case type_record:
  6672. case type_row:
  6673. {
  6674. OwnedHqlExpr record = getSerializedForm(child->queryRecord());
  6675. ColumnToOffsetMap * map = queryRecordOffsetMap(record);
  6676. if (map->isFixedWidth())
  6677. size = map->getFixedRecordSize();
  6678. else
  6679. size = map->getMaxSize();
  6680. }
  6681. break;
  6682. case type_alien:
  6683. {
  6684. IHqlAlienTypeInfo * alien = queryAlienType(type);
  6685. size = alien->getMaxSize();
  6686. break;
  6687. }
  6688. default:
  6689. size = type->getSize();
  6690. break;
  6691. }
  6692. if (size == UNKNOWN_LENGTH)
  6693. throwError(HQLERR_CouldNotDetermineMaxSize);
  6694. tgt.expr.setown(getSizetConstant(size));
  6695. return;
  6696. }
  6697. }
  6698. if (expr->hasProperty(minAtom))
  6699. {
  6700. if (type)
  6701. {
  6702. unsigned size = UNKNOWN_LENGTH;
  6703. switch (type->getTypeCode())
  6704. {
  6705. case type_table:
  6706. case type_groupedtable:
  6707. case type_record:
  6708. case type_row:
  6709. {
  6710. OwnedHqlExpr record = getSerializedForm(child->queryRecord());
  6711. size = getMinRecordSize(record);
  6712. }
  6713. break;
  6714. default:
  6715. size = type->getSize();
  6716. break;
  6717. }
  6718. if (size == UNKNOWN_LENGTH)
  6719. throwError(HQLERR_CouldNotDetermineMinSize);
  6720. tgt.expr.setown(getSizetConstant(size));
  6721. return;
  6722. }
  6723. }
  6724. #if 0
  6725. if (limitExpr)
  6726. {
  6727. OwnedHqlExpr other = createValue(no_sizeof, expr->getType(), LINK(child));
  6728. HqlExprAssociation * match = ctx.getMatchExpr(other);
  6729. if (match)
  6730. {
  6731. tgt.expr.set(match->expr);
  6732. return;
  6733. }
  6734. }
  6735. #endif
  6736. // Size calculation needs to only come in to play if the field/record can't be found in scope
  6737. // otherwise sizeof(field) is wrong if it is inside an ifblock.
  6738. Owned<IReferenceSelector> selector;
  6739. try
  6740. {
  6741. selector.setown(buildReference(ctx, child));
  6742. selector->getSize(ctx, tgt);
  6743. //cache non-constant values in a temporary variable...
  6744. if (!tgt.expr->queryValue())
  6745. {
  6746. if (!isSimpleLength(tgt.expr))
  6747. {
  6748. IHqlExpression * temp = ctx.getTempDeclare(expr->queryType(), tgt.expr);
  6749. tgt.expr.setown(temp);
  6750. }
  6751. ctx.associateExpr(expr, tgt);
  6752. }
  6753. }
  6754. catch (IException * e)
  6755. {
  6756. switch (child->getOperator())
  6757. {
  6758. case no_translated:
  6759. {
  6760. CHqlBoundExpr bound;
  6761. buildExpr(ctx, child, bound);
  6762. tgt.expr.setown(getBoundSize(bound));
  6763. return;
  6764. }
  6765. }
  6766. // Size calculation needs to only come in to play if the field/record can't be found in scope
  6767. // otherwise sizeof(field) is wrong if it is inside an ifblock.
  6768. if (type)
  6769. {
  6770. if (type->getTypeCode() == type_alien)
  6771. {
  6772. IHqlAlienTypeInfo * alien = queryAlienType(type);
  6773. type = alien->queryPhysicalType();
  6774. }
  6775. switch (type->getTypeCode())
  6776. {
  6777. case type_table:
  6778. case type_groupedtable:
  6779. case type_record:
  6780. case type_row:
  6781. {
  6782. e->Release();
  6783. OwnedHqlExpr record = getSerializedForm(child->queryRecord());
  6784. ColumnToOffsetMap * map = queryRecordOffsetMap(record);
  6785. if (map->isFixedWidth())
  6786. {
  6787. tgt.expr.setown(getSizetConstant(map->getFixedRecordSize()));
  6788. return;
  6789. }
  6790. throwError(HQLERR_CannotDetermineSizeVar);
  6791. }
  6792. break;
  6793. case type_void:
  6794. break;
  6795. default:
  6796. if ((type->getSize() != UNKNOWN_LENGTH) && (!selector || !selector->isConditional()))
  6797. {
  6798. tgt.expr.setown(getSizetConstant(type->getSize()));
  6799. e->Release();
  6800. return;
  6801. }
  6802. }
  6803. }
  6804. throw; // really an internal error - the parse should not have let it through....
  6805. }
  6806. }
  6807. void HqlCppTranslator::doBuildExprRowDiff(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr, IHqlExpression * rightRecord, IHqlExpression * leftSelector, IHqlExpression * rightSelector, StringBuffer & selectorText, bool isCount)
  6808. {
  6809. switch (expr->getOperator())
  6810. {
  6811. case no_field:
  6812. {
  6813. _ATOM name = expr->queryName();
  6814. IHqlSimpleScope * rightScope = rightRecord->querySimpleScope();
  6815. OwnedHqlExpr match = rightScope ? rightScope->lookupSymbol(name) : NULL;
  6816. if (!match)
  6817. return;
  6818. OwnedHqlExpr left = createSelectExpr(LINK(leftSelector), LINK(expr));
  6819. OwnedHqlExpr right = createSelectExpr(LINK(rightSelector), LINK(match));
  6820. ITypeInfo * leftType = expr->queryType()->queryPromotedType();
  6821. switch (leftType->getTypeCode())
  6822. {
  6823. case type_record:
  6824. case type_row:
  6825. {
  6826. StringBuffer subSelectorText;
  6827. subSelectorText.append(selectorText).append(expr->queryName()).append(".");
  6828. IHqlExpression * record = ::queryRecord(leftType);
  6829. doBuildExprRowDiff(ctx, target, record, left, right->queryRecord(), right, subSelectorText, isCount);
  6830. return;
  6831. }
  6832. break;
  6833. case type_table:
  6834. case type_groupedtable:
  6835. UNIMPLEMENTED;
  6836. }
  6837. StringBuffer fullName;
  6838. fullName.append(selectorText).append(name);
  6839. ITypeInfo * rightType = right->queryType()->queryPromotedType();
  6840. if (!leftType->assignableFrom(rightType))
  6841. throwError1(HQLERR_MismatchRowDiffType, fullName.str());
  6842. Owned<ITypeInfo> compareType = ::getPromotedECLType(leftType, rightType);
  6843. OwnedHqlExpr test = createBoolExpr(no_ne, ensureExprType(left, compareType), ensureExprType(right, compareType));
  6844. HqlExprArray args;
  6845. CHqlBoundExpr bound;
  6846. buildExpr(ctx, test, bound);
  6847. StringBuffer specialText;
  6848. generateExprCpp(specialText, target.length).append(",");
  6849. generateExprCpp(specialText, target.expr).append(".refextendstr()");
  6850. OwnedHqlExpr special = createQuoted(specialText.str(), makeBoolType());
  6851. BuildCtx condctx(ctx);
  6852. IHqlStmt * cond = condctx.addFilter(bound.expr);
  6853. //if differ...
  6854. args.append(*LINK(special));
  6855. if (isCount)
  6856. args.append(*createConstant(",1"));
  6857. else
  6858. {
  6859. StringBuffer temp;
  6860. temp.append(",").append(fullName);
  6861. args.append(*createConstant(temp));
  6862. }
  6863. buildFunctionCall(condctx, concatExtendAtom, args);
  6864. //else if same...
  6865. if (isCount)
  6866. {
  6867. condctx.selectElse(cond);
  6868. args.append(*LINK(special));
  6869. args.append(*createConstant(",0"));
  6870. buildFunctionCall(condctx, concatExtendAtom, args);
  6871. }
  6872. break;
  6873. }
  6874. case no_ifblock:
  6875. {
  6876. doBuildExprRowDiff(ctx, target, expr->queryChild(1), leftSelector, rightRecord, rightSelector, selectorText, isCount);
  6877. break;
  6878. }
  6879. case no_record:
  6880. {
  6881. ForEachChild(idx, expr)
  6882. doBuildExprRowDiff(ctx, target, expr->queryChild(idx), leftSelector, rightRecord, rightSelector, selectorText, isCount);
  6883. break;
  6884. }
  6885. case no_attr:
  6886. case no_attr_expr:
  6887. case no_attr_link:
  6888. break;
  6889. default:
  6890. UNIMPLEMENTED;
  6891. }
  6892. }
  6893. IHqlExpression * HqlCppTranslator::queryRecord(BuildCtx & ctx, IHqlExpression * expr)
  6894. {
  6895. return expr->queryRecord();
  6896. }
  6897. void HqlCppTranslator::doBuildExprRowDiff(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt)
  6898. {
  6899. CHqlBoundTarget tempTarget;
  6900. createTempFor(ctx, expr, tempTarget);
  6901. StringBuffer selectorText;
  6902. IHqlExpression * left = expr->queryChild(0);
  6903. IHqlExpression * leftRecord = queryRecord(ctx, left);
  6904. IHqlExpression * right = expr->queryChild(1);
  6905. IHqlExpression * rightRecord = queryRecord(ctx, right);
  6906. ctx.addAssign(tempTarget.length, queryZero());
  6907. doBuildExprRowDiff(ctx, tempTarget, leftRecord, left, rightRecord, right, selectorText, expr->hasProperty(countAtom));
  6908. OwnedHqlExpr result = createValue(no_substring, LINK(unknownStringType), tempTarget.getTranslatedExpr(), createValue(no_rangefrom, makeVoidType(), createConstant(2)));
  6909. buildExpr(ctx, result, tgt);
  6910. }
  6911. //---------------------------------------------------------------------------
  6912. ABoundActivity * HqlCppTranslator::doBuildActivityCacheAlias(BuildCtx & ctx, IHqlExpression * expr)
  6913. {
  6914. IHqlExpression * dataset = expr->queryChild(1);
  6915. return buildCachedActivity(ctx, dataset);
  6916. }
  6917. //---------------------------------------------------------------------------
  6918. // no_cloned
  6919. ABoundActivity * HqlCppTranslator::doBuildActivityCloned(BuildCtx & ctx, IHqlExpression * expr)
  6920. {
  6921. IHqlExpression * dataset = expr->queryChild(0);
  6922. return buildCachedActivity(ctx, dataset);
  6923. }
  6924. //---------------------------------------------------------------------------
  6925. // no_addfiles
  6926. static void unwindAddFiles(HqlExprArray & args, IHqlExpression * expr, bool isOrdered, bool isOrderedPull)
  6927. {
  6928. if ((expr->getOperator() == no_addfiles) && (expr->hasProperty(_ordered_Atom) == isOrdered) && (expr->hasProperty(_orderedPull_Atom) == isOrderedPull))
  6929. {
  6930. unwindAddFiles(args, expr->queryChild(0), isOrdered, isOrderedPull);
  6931. unwindAddFiles(args, expr->queryChild(1), isOrdered, isOrderedPull);
  6932. }
  6933. else
  6934. args.append(*LINK(expr));
  6935. }
  6936. ABoundActivity * HqlCppTranslator::doBuildActivityConcat(BuildCtx & ctx, IHqlExpression * expr)
  6937. {
  6938. HqlExprArray inExprs;
  6939. bool ordered = expr->hasProperty(_ordered_Atom);
  6940. bool orderedPull = expr->hasProperty(_orderedPull_Atom);
  6941. unwindAddFiles(inExprs, expr, ordered, orderedPull);
  6942. //If all coming from disk, probably better to pull them in order.
  6943. bool allFromDisk = options.orderDiskFunnel;
  6944. CIArray bound;
  6945. ForEachItemIn(idx, inExprs)
  6946. {
  6947. IHqlExpression * cur = &inExprs.item(idx);
  6948. bound.append(*buildCachedActivity(ctx, cur));
  6949. loop
  6950. {
  6951. node_operator curOp = cur->getOperator();
  6952. if ((curOp != no_nofold) && (curOp != no_section) && (curOp != no_sectioninput) && (curOp != no_preservemeta))
  6953. break;
  6954. cur = cur->queryChild(0);
  6955. }
  6956. switch (cur->getOperator())
  6957. {
  6958. case no_compound_diskread:
  6959. case no_compound_disknormalize:
  6960. case no_compound_diskaggregate:
  6961. case no_compound_diskcount:
  6962. case no_compound_diskgroupaggregate:
  6963. break;
  6964. case no_temptable:
  6965. case no_inlinetable:
  6966. case no_temprow:
  6967. case no_datasetfromrow:
  6968. case no_projectrow:
  6969. case no_createrow:
  6970. case no_typetransfer:
  6971. break;
  6972. case no_table:
  6973. switch (cur->queryChild(2)->getOperator())
  6974. {
  6975. case no_thor: case no_flat:
  6976. break;
  6977. default:
  6978. allFromDisk = false;
  6979. break;
  6980. }
  6981. break;
  6982. default:
  6983. allFromDisk = false;
  6984. break;
  6985. }
  6986. }
  6987. if (orderedPull || (allFromDisk && !targetRoxie()))
  6988. ordered = true;
  6989. bool useImplementationClass = options.minimizeActivityClasses && targetRoxie();
  6990. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKfunnel, expr, "Funnel");
  6991. if (useImplementationClass)
  6992. instance->setImplementationClass(newFunnelArgAtom);
  6993. buildActivityFramework(instance);
  6994. buildInstancePrefix(instance);
  6995. if (!useImplementationClass)
  6996. {
  6997. if (ordered)
  6998. doBuildBoolFunction(instance->classctx, "isOrdered", true);
  6999. if (orderedPull)
  7000. doBuildBoolFunction(instance->classctx, "pullSequentially", orderedPull);
  7001. }
  7002. else
  7003. {
  7004. instance->addConstructorParameter(queryBoolExpr(ordered));
  7005. instance->addConstructorParameter(queryBoolExpr(orderedPull));
  7006. }
  7007. buildInstanceSuffix(instance);
  7008. ForEachItemIn(idx2, bound)
  7009. buildConnectInputOutput(ctx, instance, (ABoundActivity *)&bound.item(idx2), 0, idx2);
  7010. return instance->getBoundActivity();
  7011. }
  7012. ABoundActivity * HqlCppTranslator::doBuildActivityMerge(BuildCtx & ctx, IHqlExpression * expr)
  7013. {
  7014. CIArrayOf<ABoundActivity> inputs;
  7015. ForEachChild(idx, expr)
  7016. {
  7017. IHqlExpression *cur = expr->queryChild(idx);
  7018. if (!cur->isAttribute())
  7019. inputs.append(*buildCachedActivity(ctx, cur));
  7020. }
  7021. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKmerge, expr, "Merge");
  7022. buildActivityFramework(instance);
  7023. buildInstancePrefix(instance);
  7024. IHqlExpression * dataset = expr->queryChild(0);
  7025. IHqlExpression * sortAttr = expr->queryProperty(sortedAtom);
  7026. HqlExprArray sorts;
  7027. unwindChildren(sorts, sortAttr);
  7028. if (sorts.ordinality() != 0)
  7029. {
  7030. OwnedHqlExpr sortOrder = createValueSafe(no_sortlist, makeSortListType(NULL), sorts);
  7031. instance->startctx.addQuoted("virtual ICompare * queryCompare() { return &compare; }");
  7032. DatasetReference dsRef(dataset, no_activetable, NULL);
  7033. buildCompareClass(instance->nestedctx, "compare", sortOrder, dsRef);
  7034. if (!instance->isLocal)
  7035. generateSerializeKey(instance->nestedctx, no_none, dsRef, sorts, !instance->isChildActivity(), true, false);
  7036. }
  7037. else
  7038. throwError(HQLERR_InputMergeNotSorted);
  7039. if (expr->hasProperty(dedupAtom))
  7040. doBuildBoolFunction(instance->classctx, "dedup", true);
  7041. buildInstanceSuffix(instance);
  7042. ForEachItemIn(idx2, inputs)
  7043. buildConnectInputOutput(ctx, instance, &inputs.item(idx2), 0, idx2);
  7044. return instance->getBoundActivity();
  7045. }
  7046. ABoundActivity * HqlCppTranslator::doBuildActivityRegroup(BuildCtx & ctx, IHqlExpression * expr)
  7047. {
  7048. HqlExprArray inExprs;
  7049. expr->unwindList(inExprs, no_regroup);
  7050. CIArray bound;
  7051. ForEachItemIn(idx, inExprs)
  7052. {
  7053. IHqlExpression & cur = inExprs.item(idx);
  7054. bound.append(*buildCachedActivity(ctx, &cur));
  7055. }
  7056. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKregroup, expr, "Regroup");
  7057. buildActivityFramework(instance);
  7058. buildInstancePrefix(instance);
  7059. buildInstanceSuffix(instance);
  7060. ForEachItemIn(idx2, bound)
  7061. buildConnectInputOutput(ctx, instance, (ABoundActivity *)&bound.item(idx2), 0, idx2);
  7062. return instance->getBoundActivity();
  7063. }
  7064. static void unwindNonEmpty(HqlExprCopyArray & args, IHqlExpression * expr, bool isLocal)
  7065. {
  7066. if ((expr->getOperator() == no_nonempty) && (expr->hasProperty(localAtom) == isLocal))
  7067. {
  7068. ForEachChild(i, expr)
  7069. unwindNonEmpty(args, expr->queryChild(i), isLocal);
  7070. }
  7071. else
  7072. args.append(*expr);
  7073. }
  7074. ABoundActivity * HqlCppTranslator::doBuildActivityNonEmpty(BuildCtx & ctx, IHqlExpression * expr)
  7075. {
  7076. HqlExprCopyArray inExprs;
  7077. unwindNonEmpty(inExprs, expr, expr->hasProperty(localAtom));
  7078. CIArray bound;
  7079. ForEachItemIn(idx, inExprs)
  7080. {
  7081. IHqlExpression * cur = &inExprs.item(idx);
  7082. if (!cur->isAttribute())
  7083. bound.append(*buildCachedActivity(ctx, cur));
  7084. }
  7085. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKnonempty, expr, "NonEmpty");
  7086. buildActivityFramework(instance);
  7087. buildInstancePrefix(instance);
  7088. buildInstanceSuffix(instance);
  7089. ForEachItemIn(idx2, bound)
  7090. buildConnectInputOutput(ctx, instance, (ABoundActivity *)&bound.item(idx2), 0, idx2);
  7091. return instance->getBoundActivity();
  7092. }
  7093. ABoundActivity * HqlCppTranslator::doBuildActivitySplit(BuildCtx & ctx, IHqlExpression * expr)
  7094. {
  7095. IHqlExpression * dataset = expr->queryChild(0);
  7096. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  7097. bool useImplementationClass = options.minimizeActivityClasses;
  7098. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKsplit, expr, "Split");
  7099. if (useImplementationClass)
  7100. instance->setImplementationClass(newSplitArgAtom);
  7101. buildActivityFramework(instance);
  7102. buildInstancePrefix(instance);
  7103. //IHqlExpression * numWays = expr->queryChild(1);
  7104. OwnedHqlExpr numWaysCallback = createUnknown(no_callback, LINK(sizetType), countAtom, instance->createOutputCountCallback());
  7105. OwnedHqlExpr numWays = createTranslated(numWaysCallback);
  7106. bool balanced = expr->hasProperty(balancedAtom);
  7107. if (!useImplementationClass)
  7108. {
  7109. if (!matchesConstantValue(numWays, 2))
  7110. doBuildUnsignedFunction(instance->classctx, "numBranches", numWays);
  7111. if (balanced)
  7112. doBuildBoolFunction(instance->classctx, "isBalanced", true);
  7113. }
  7114. else
  7115. {
  7116. instance->addConstructorParameter(numWays);
  7117. instance->addConstructorParameter(queryBoolExpr(balanced));
  7118. }
  7119. buildInstanceSuffix(instance);
  7120. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  7121. return instance->getBoundActivity();
  7122. }
  7123. ABoundActivity * HqlCppTranslator::doBuildActivitySpill(BuildCtx & ctx, IHqlExpression * expr)
  7124. {
  7125. return doBuildActivityOutput(ctx, expr, false);
  7126. }
  7127. //---------------------------------------------------------------------------
  7128. bool HqlCppTranslator::isCurrentActiveGraph(BuildCtx & ctx, IHqlExpression * graphTag)
  7129. {
  7130. SubGraphInfo * activeSubgraph = queryActiveSubGraph(ctx);
  7131. assertex(activeSubgraph);
  7132. return (graphTag == activeSubgraph->graphTag);
  7133. }
  7134. ABoundActivity * HqlCppTranslator::doBuildActivityLoop(BuildCtx & ctx, IHqlExpression * expr)
  7135. {
  7136. unsigned curArg = 0;
  7137. IHqlExpression * dataset = expr->queryChild(curArg++);
  7138. IHqlExpression * rhs = NULL;
  7139. if (expr->getOperator() == no_loop2)
  7140. rhs = expr->queryChild(curArg++);
  7141. IHqlExpression * count = queryRealChild(expr, curArg++);
  7142. IHqlExpression * filter = queryRealChild(expr, curArg++);
  7143. IHqlExpression * loopCond = queryRealChild(expr, curArg++);
  7144. IHqlExpression * body = expr->queryChild(curArg++);
  7145. assertex(body->getOperator() == no_loopbody);
  7146. IHqlExpression * rhsTransform = NULL;
  7147. if (rhs)
  7148. rhsTransform = expr->queryChild(curArg++);
  7149. IHqlExpression * counter = queryPropertyChild(expr, _countProject_Atom, 0);
  7150. IHqlExpression * rowsid = expr->queryProperty(_rowsid_Atom);
  7151. IHqlExpression * selSeq = querySelSeq(expr);
  7152. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  7153. ThorActivityKind kind = TAKnone;
  7154. //LOOP(dataset, count[, rowFilter])
  7155. //LOOP(dataset, <dataset-filter>, <rowfilter>)
  7156. //LOOP(dataset, <dataset-filter|rowfilter>
  7157. if (count)
  7158. kind = TAKloopcount;
  7159. else if (loopCond)
  7160. kind = TAKloopdataset;
  7161. else
  7162. kind = TAKlooprow;
  7163. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, kind, expr, "Loop");
  7164. buildActivityFramework(instance);
  7165. buildInstancePrefix(instance);
  7166. if (filter)
  7167. {
  7168. BuildCtx funcctx(instance->startctx);
  7169. funcctx.addQuotedCompound("virtual bool sendToLoop(unsigned counter, const void * _self)");
  7170. funcctx.addQuoted("unsigned char * self = (unsigned char *) _self;");
  7171. associateCounter(funcctx, counter, "counter");
  7172. bindTableCursor(funcctx, dataset, "self", no_left, selSeq);
  7173. buildReturn(funcctx, filter);
  7174. }
  7175. if (count)
  7176. doBuildUnsignedFunction(instance->startctx, "numIterations", count);
  7177. if (loopCond)
  7178. {
  7179. BuildCtx funcctx(instance->startctx);
  7180. funcctx.addQuotedCompound("virtual bool loopAgain(unsigned counter, unsigned numRows, const void * * _rows)");
  7181. funcctx.addQuoted("unsigned char * * rows = (unsigned char * *) _rows;");
  7182. associateCounter(funcctx, counter, "counter");
  7183. bindRows(funcctx, no_left, selSeq, rowsid, dataset, "numRows", "rows", options.mainRowsAreLinkCounted);
  7184. buildReturn(funcctx, loopCond);
  7185. }
  7186. IHqlExpression * parallel = expr->queryProperty(parallelAtom);
  7187. if (parallel && (targetHThor() || !count || loopCond))
  7188. parallel = NULL;
  7189. if (parallel)
  7190. {
  7191. IHqlExpression * arg0 = parallel->queryChild(0);
  7192. IHqlExpression * arg1 = parallel->queryChild(1);
  7193. LinkedHqlExpr parallelList;
  7194. LinkedHqlExpr numThreads;
  7195. if (arg0)
  7196. {
  7197. if (arg1)
  7198. {
  7199. parallelList.set(arg0);
  7200. numThreads.set(arg1);
  7201. }
  7202. else if (arg0->isList())
  7203. parallelList.set(arg0);
  7204. else
  7205. numThreads.set(arg0);
  7206. }
  7207. if (numThreads)
  7208. doBuildUnsignedFunction(instance->startctx, "defaultParallelIterations", numThreads);
  7209. if (parallelList)
  7210. {
  7211. Owned<ITypeInfo> setType = makeSetType(LINK(unsignedType));
  7212. BuildCtx funcctx(instance->startctx);
  7213. funcctx.addQuotedCompound("virtual void numParallelIterations(size32_t & __lenResult, void * & __result)");
  7214. funcctx.addQuoted("bool __isAllResult;");
  7215. doBuildFunctionReturn(funcctx, setType, parallelList);
  7216. }
  7217. }
  7218. StringBuffer flags;
  7219. if (counter) flags.append("|LFcounter");
  7220. if (parallel) flags.append("|LFparallel");
  7221. if (filter) flags.append("|LFfiltered");
  7222. if (flags.length())
  7223. doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);
  7224. BuildCtx subctx(instance->startctx);
  7225. subctx.addQuotedCompound("virtual void createParentExtract(rtlRowBuilder & builder)");
  7226. //Now need to generate the body of the loop.
  7227. //output dataset is result 0
  7228. //input dataset is fed in using result 1
  7229. //counter (if required) is fed in using result 2[0].counter;
  7230. unique_id_t loopId = buildLoopSubgraph(subctx, dataset, selSeq, rowsid, body->queryChild(0), counter, instance->activityId, (parallel != NULL));
  7231. instance->addAttributeInt("_loopid", loopId);
  7232. buildInstanceSuffix(instance);
  7233. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  7234. return instance->getBoundActivity();
  7235. }
  7236. //---------------------------------------------------------------------------
  7237. ABoundActivity * HqlCppTranslator::doBuildActivityGraphLoop(BuildCtx & ctx, IHqlExpression * expr)
  7238. {
  7239. unsigned curArg = 0;
  7240. IHqlExpression * dataset = expr->queryChild(curArg++);
  7241. IHqlExpression * count = expr->queryChild(curArg++);
  7242. IHqlExpression * body = expr->queryChild(curArg++);
  7243. assertex(body->getOperator() == no_loopbody);
  7244. IHqlExpression * counter = queryPropertyChild(expr, _countProject_Atom, 0);
  7245. IHqlExpression * rowsid = expr->queryProperty(_rowsid_Atom);
  7246. IHqlExpression * selSeq = querySelSeq(expr);
  7247. IHqlExpression * parallel = expr->queryProperty(parallelAtom);
  7248. if (parallel && targetHThor())
  7249. parallel = NULL;
  7250. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  7251. ThorActivityKind kind = parallel ? TAKparallelgraphloop : TAKgraphloop;
  7252. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, kind, expr, "GraphLoop");
  7253. buildActivityFramework(instance);
  7254. buildInstancePrefix(instance);
  7255. doBuildUnsignedFunction(instance->startctx, "numIterations", count);
  7256. StringBuffer flags;
  7257. if (counter) flags.append("|GLFcounter");
  7258. if (parallel) flags.append("|GLFparallel");
  7259. if (flags.length())
  7260. doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);
  7261. BuildCtx subctx(instance->startctx);
  7262. subctx.addQuotedCompound("virtual void createParentExtract(rtlRowBuilder & builder)");
  7263. //Now need to generate the body of the loop.
  7264. //output dataset is result 0
  7265. //input dataset is fed in using result 1
  7266. //counter (if required) is fed in using result 2[0].counter;
  7267. unique_id_t loopId = buildGraphLoopSubgraph(subctx, dataset, selSeq, rowsid, body->queryChild(0), counter, instance->activityId, (parallel != NULL));
  7268. instance->addAttributeInt("_loopid", loopId);
  7269. buildInstanceSuffix(instance);
  7270. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  7271. return instance->getBoundActivity();
  7272. }
  7273. //---------------------------------------------------------------------------
  7274. ABoundActivity * HqlCppTranslator::doBuildActivityRemote(BuildCtx & ctx, IHqlExpression * expr, bool isRoot)
  7275. {
  7276. IHqlExpression * child = expr->queryChild(0);
  7277. if (targetHThor() || (targetThor() && !insideChildGraph(ctx)))
  7278. {
  7279. if (!options.alwaysAllowAllNodes)
  7280. throwError(HQLERR_RemoteNoMeaning);
  7281. }
  7282. if (isGrouped(child))
  7283. throwError(HQLERR_RemoteGrouped);
  7284. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKremotegraph, expr, "Remote");
  7285. buildActivityFramework(instance, isRoot);
  7286. buildInstancePrefix(instance);
  7287. IHqlExpression * dataset = expr->queryChild(0);
  7288. IHqlExpression * rowlimit = expr->queryProperty(rowLimitAtom);
  7289. if (rowlimit)
  7290. {
  7291. doBuildUnsigned64Function(instance->startctx, "getRowLimit", rowlimit->queryChild(0));
  7292. IHqlExpression * fail = queryChildOperator(no_fail, rowlimit);
  7293. if (fail)
  7294. {
  7295. BuildCtx ctx(instance->startctx);
  7296. ctx.addQuotedCompound("virtual void onLimitExceeded()");
  7297. buildStmt(ctx, fail);
  7298. }
  7299. }
  7300. BuildCtx subctx(instance->startctx);
  7301. subctx.addQuotedCompound("virtual void createParentExtract(rtlRowBuilder & builder)");
  7302. //output dataset is result 0
  7303. unique_id_t remoteId = buildRemoteSubgraph(subctx, dataset, instance->activityId);
  7304. instance->addAttributeInt("_graphid", remoteId);
  7305. buildInstanceSuffix(instance);
  7306. return instance->getBoundActivity();
  7307. }
  7308. //---------------------------------------------------------------------------
  7309. // no_update
  7310. void HqlCppTranslator::doBuildStmtUpdate(BuildCtx & ctx, IHqlExpression * expr)
  7311. {
  7312. // MJH - CODE TO DO UPDATE GOES HERE
  7313. PrintLog("in HqlCppTranslator::doBuildStmtUpdate()");
  7314. }
  7315. IHqlExpression * HqlCppTranslator::createClearRowCall(BuildCtx & ctx, BoundRow * self)
  7316. {
  7317. IHqlExpression * record = self->querySelector()->queryRecord();
  7318. OwnedHqlExpr clearFunc = getClearRecordFunction(record, 0);
  7319. StringBuffer s;
  7320. ensureContextAvailable(ctx);
  7321. IHqlExpression * boundRow = self->queryBound();
  7322. OwnedHqlExpr rowPointer = getPointer(boundRow);
  7323. generateExprCpp(s, clearFunc).append("(");
  7324. if (self->queryBuilder())
  7325. {
  7326. generateExprCpp(s, self->queryBuilder());
  7327. }
  7328. else
  7329. {
  7330. StringBuffer builderName;
  7331. getUniqueId(builderName.append("rb"));
  7332. StringBuffer temp;
  7333. temp.append("RtlStaticRowBuilder ").append(builderName).append("(");
  7334. if (ctx.queryMatchExpr(constantMemberMarkerExpr))
  7335. temp.append("(byte *)");
  7336. generateExprCpp(temp, rowPointer);
  7337. temp.append(",").append(getMaxRecordSize(record)).append(");");
  7338. ctx.addQuoted(temp);
  7339. s.append(builderName);
  7340. }
  7341. s.append(", ctx)");
  7342. return createQuoted(s.str(), makeVoidType());
  7343. }
  7344. void HqlCppTranslator::associateSkipReturnMarker(BuildCtx & ctx, IHqlExpression * value, BoundRow * self)
  7345. {
  7346. ctx.associateExpr(skipReturnMarker, value);
  7347. }
  7348. void HqlCppTranslator::doBuildStmtSkip(BuildCtx & ctx, IHqlExpression * expr, bool * canReachFollowing)
  7349. {
  7350. HqlExprAssociation * match = ctx.queryMatchExpr(skipActionMarker);
  7351. IHqlExpression * cond = expr->queryChild(0);
  7352. if (cond && cond->isRecord())
  7353. cond = NULL;
  7354. if (canReachFollowing)
  7355. *canReachFollowing = true;
  7356. BuildCtx subctx(ctx);
  7357. if (match)
  7358. {
  7359. IHqlCodeCallback * callback = static_cast<IHqlCodeCallback *>(match->queryExpr()->queryUnknownExtra());
  7360. if (cond)
  7361. buildFilter(subctx, cond);
  7362. callback->buildCode(*this, subctx);
  7363. }
  7364. else
  7365. {
  7366. match = ctx.queryMatchExpr(skipReturnMarker);
  7367. if (match)
  7368. {
  7369. if (cond)
  7370. buildFilteredReturn(ctx, cond, match->queryExpr());
  7371. else
  7372. {
  7373. buildReturn(ctx, match->queryExpr());
  7374. if (canReachFollowing)
  7375. *canReachFollowing = false;
  7376. }
  7377. }
  7378. else
  7379. throwError(HQLERR_SkipNotValidHere);
  7380. }
  7381. }
  7382. void HqlCppTranslator::doBuildStmtAssert(BuildCtx & ctx, IHqlExpression * expr)
  7383. {
  7384. if (!options.checkAsserts)
  7385. return;
  7386. IHqlExpression * cond = expr->queryChild(0);
  7387. LinkedHqlExpr locationAttr = expr->queryProperty(_location_Atom);
  7388. if (!locationAttr)
  7389. {
  7390. IHqlExpression * activeNamedActivity = queryActiveNamedActivity();
  7391. if (activeNamedActivity)
  7392. {
  7393. ISourcePath * sourcePath = activeNamedActivity->querySourcePath();
  7394. if (sourcePath)
  7395. locationAttr.setown(createLocationAttr(sourcePath, 0, 0, 0));
  7396. }
  7397. }
  7398. LinkedHqlExpr msg = queryRealChild(expr, 1);
  7399. if (!msg)
  7400. msg.setown(createDefaultAssertMessage(cond));
  7401. if (expr->hasProperty(constAtom))
  7402. {
  7403. IValue * condValue = cond->queryValue();
  7404. assertex(condValue && msg->queryValue());
  7405. if (condValue->getBoolValue())
  7406. return;
  7407. StringBuffer msgText;
  7408. getStringValue(msgText, msg);
  7409. reportErrorDirect(locationAttr, ERR_ASSERTION_FAILS, msgText.str(), false);
  7410. return;
  7411. }
  7412. BuildCtx condctx(ctx);
  7413. OwnedHqlExpr inverse = getInverse(cond);
  7414. buildFilter(condctx, inverse);
  7415. OwnedHqlExpr action;
  7416. HqlExprArray args;
  7417. args.append(*getSizetConstant(ERR_ASSERTION_FAILS));
  7418. args.append(*LINK(msg));
  7419. ECLlocation location;
  7420. if (locationAttr)
  7421. location.extractLocationAttr(locationAttr);
  7422. if (location.sourcePath)
  7423. args.append(*createConstant(location.sourcePath->str()));
  7424. else
  7425. args.append(*getNullStringPointer(true));
  7426. args.append(*getSizetConstant(location.lineno));
  7427. args.append(*getSizetConstant(location.column));
  7428. args.append(*createConstant(expr->hasProperty(failAtom)));
  7429. action.setown(bindFunctionCall(addWorkunitAssertFailureAtom, args));
  7430. buildStmt(condctx, action);
  7431. }
  7432. void HqlCppTranslator::doBuildStmtCluster(BuildCtx & ctx, IHqlExpression * expr)
  7433. {
  7434. StringAttr prevClusterName;
  7435. pushCluster(ctx, expr->queryChild(1), prevClusterName);
  7436. buildStmt(ctx, expr->queryChild(0));
  7437. popCluster(ctx, prevClusterName);
  7438. }
  7439. //---------------------------------------------------------------------------
  7440. // no_apply
  7441. void HqlCppTranslator::doBuildSequenceFunc(BuildCtx & ctx, IHqlExpression * seq, bool ignoreInternal)
  7442. {
  7443. if (ignoreInternal && isInternalSeq(seq))
  7444. return;
  7445. //virtual unsigned getSequence() = 0;
  7446. StringBuffer s;
  7447. s.append("virtual int getSequence() { return ");
  7448. if (seq)
  7449. generateExprCpp(s, seq);
  7450. else
  7451. s.append("-1");
  7452. s.append("; }");
  7453. ctx.addQuoted(s);
  7454. }
  7455. class ApplyStmtBuilder
  7456. {
  7457. public:
  7458. ApplyStmtBuilder(HqlCppTranslator & _translator) : translator(_translator) {}
  7459. // ~ApplyStmtBuilder() { assertex(!builder); } // don't do this because throwing exceptions inside destructors is very messy
  7460. void flush(BuildCtx & ctx)
  7461. {
  7462. if (builder)
  7463. builder->generateGraph(ctx);
  7464. builder.clear();
  7465. }
  7466. bool requiresGraph(BuildCtx & ctx, IHqlExpression * expr)
  7467. {
  7468. if (!expr) return false;
  7469. switch (expr->getOperator())
  7470. {
  7471. case NO_ACTION_REQUIRES_GRAPH:
  7472. return true;
  7473. case no_parallel:
  7474. case no_sequential:
  7475. case no_actionlist:
  7476. case no_compound:
  7477. {
  7478. ForEachChild(idx, expr)
  7479. {
  7480. if (requiresGraph(ctx, expr->queryChild(idx)))
  7481. return true;
  7482. }
  7483. break;
  7484. }
  7485. case no_if:
  7486. return requiresGraph(ctx, expr->queryChild(1)) || requiresGraph(ctx, expr->queryChild(2));
  7487. }
  7488. return false;
  7489. }
  7490. void addGraphStmt(BuildCtx & ctx, IHqlExpression * expr)
  7491. {
  7492. if (!builder)
  7493. builder.setown(new ChildGraphBuilder(translator));
  7494. builder->buildStmt(ctx, expr);
  7495. }
  7496. void buildStmt(BuildCtx & ctx, IHqlExpression * expr)
  7497. {
  7498. node_operator op = expr->getOperator();
  7499. switch (expr->getOperator())
  7500. {
  7501. case NO_ACTION_REQUIRES_GRAPH:
  7502. addGraphStmt(ctx, expr);
  7503. break;
  7504. case no_parallel:
  7505. case no_sequential:
  7506. case no_actionlist:
  7507. case no_compound:
  7508. {
  7509. ForEachChild(idx, expr)
  7510. {
  7511. buildStmt(ctx, expr->queryChild(idx));
  7512. if (op == no_sequential)
  7513. flush(ctx);
  7514. }
  7515. break;
  7516. }
  7517. case no_attr:
  7518. case no_attr_link:
  7519. case no_attr_expr:
  7520. break;
  7521. case no_if:
  7522. if (requiresGraph(ctx, expr))
  7523. addGraphStmt(ctx, expr);
  7524. else
  7525. translator.buildStmt(ctx, expr);
  7526. break;
  7527. default:
  7528. translator.buildStmt(ctx, expr);
  7529. break;
  7530. }
  7531. }
  7532. protected:
  7533. HqlCppTranslator & translator;
  7534. Owned<ChildGraphBuilder> builder;
  7535. };
  7536. ABoundActivity * HqlCppTranslator::doBuildActivityApply(BuildCtx & ctx, IHqlExpression * expr, bool isRoot)
  7537. {
  7538. StringBuffer s;
  7539. IHqlExpression * dataset = expr->queryChild(0);
  7540. IHqlExpression * action = expr->queryChild(1);
  7541. IHqlExpression * start = expr->queryProperty(beforeAtom);
  7542. IHqlExpression * end = expr->queryProperty(afterAtom);
  7543. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  7544. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKapply, expr, "Apply");
  7545. buildActivityFramework(instance, isRoot);
  7546. buildInstancePrefix(instance);
  7547. doBuildSequenceFunc(instance->classctx, querySequence(expr), false);
  7548. BuildCtx transformctx(instance->startctx);
  7549. transformctx.addQuotedCompound("virtual void apply(const void * _self)");
  7550. s.clear().append("unsigned char * self = (unsigned char *) _self;");
  7551. transformctx.addQuoted(s);
  7552. bindTableCursor(transformctx, dataset, "self");
  7553. unsigned max = expr->numChildren();
  7554. ApplyStmtBuilder builder(*this);
  7555. for (unsigned i=1; i < max; i++)
  7556. builder.buildStmt(transformctx, expr->queryChild(i));
  7557. builder.flush(transformctx);
  7558. if (start)
  7559. {
  7560. BuildCtx startctx(instance->startctx);
  7561. startctx.addQuotedCompound("virtual void start()");
  7562. builder.buildStmt(startctx, start->queryChild(0));
  7563. builder.flush(startctx);
  7564. }
  7565. if (end)
  7566. {
  7567. BuildCtx endctx(instance->startctx);
  7568. endctx.addQuotedCompound("virtual void end()");
  7569. builder.buildStmt(endctx, end->queryChild(0));
  7570. builder.flush(endctx);
  7571. }
  7572. buildInstanceSuffix(instance);
  7573. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  7574. return instance->getBoundActivity();
  7575. }
  7576. //---------------------------------------------------------------------------
  7577. //-- no_thor --
  7578. ActivityInstance * HqlCppTranslator::queryCurrentActivity(BuildCtx & ctx)
  7579. {
  7580. return static_cast<ActivityInstance *>(ctx.queryFirstAssociation(AssocActivityInstance));
  7581. }
  7582. unique_id_t HqlCppTranslator::queryCurrentActivityId(BuildCtx & ctx)
  7583. {
  7584. ActivityInstance * activeActivity = queryCurrentActivity(ctx);
  7585. if (activeActivity)
  7586. return activeActivity->activityId;
  7587. HqlExprAssociation * match = ctx.queryMatchExpr(queryActivityIdMarker());
  7588. if (match)
  7589. return getIntValue(match->queryExpr());
  7590. return 0;
  7591. }
  7592. IHqlExpression * HqlCppTranslator::getCurrentActivityId(BuildCtx & ctx)
  7593. {
  7594. ActivityInstance * activeActivity = queryCurrentActivity(ctx);
  7595. if (activeActivity)
  7596. return getSizetConstant(activeActivity->activityId);
  7597. HqlExprAssociation * match = ctx.queryMatchExpr(queryActivityIdMarker());
  7598. if (match)
  7599. return LINK(match->queryExpr());
  7600. return getSizetConstant(0);
  7601. }
  7602. bool HqlCppTranslator::insideRemoteGraph(BuildCtx & ctx)
  7603. {
  7604. //Dubious... how about a remote child query?
  7605. SubGraphInfo * activeSubgraph = queryActiveSubGraph(ctx);
  7606. if (!activeSubgraph)
  7607. return false;
  7608. return (activeSubgraph->type == SubGraphRemote);
  7609. }
  7610. bool HqlCppTranslator::insideChildGraph(BuildCtx & ctx)
  7611. {
  7612. FilteredAssociationIterator iter(ctx, AssocSubGraph);
  7613. ForEach(iter)
  7614. {
  7615. SubGraphInfo & cur = static_cast<SubGraphInfo &>(iter.get());
  7616. if (cur.type == SubGraphChild)
  7617. return true;
  7618. }
  7619. return false;
  7620. }
  7621. unsigned HqlCppTranslator::curSubGraphId(BuildCtx & ctx)
  7622. {
  7623. SubGraphInfo * activeSubgraph = queryActiveSubGraph(ctx);
  7624. return activeSubgraph ? activeSubgraph->id : 0;
  7625. }
  7626. unsigned HqlCppTranslator::doBuildThorChildSubGraph(BuildCtx & ctx, IHqlExpression * expr, SubGraphType kind, unsigned reservedId, IHqlExpression * graphTag)
  7627. {
  7628. //NB: Need to create the graph in the correct order so the references to property trees we retain
  7629. // remain live..
  7630. SubGraphInfo * activeSubgraph = queryActiveSubGraph(ctx);
  7631. IPropertyTree * node = createPTree("node");
  7632. if (activeSubgraph)
  7633. node = activeSubgraph->tree->addPropTree("node", node);
  7634. else
  7635. node = graph->addPropTree("node", node);
  7636. unsigned thisId = reservedId ? reservedId : nextActivityId();
  7637. node->setPropInt("@id", thisId);
  7638. IPropertyTree * graphAttr = node->addPropTree("att", createPTree("att"));
  7639. IPropertyTree * subGraph = graphAttr->addPropTree("graph", createPTree("graph"));
  7640. Owned<SubGraphInfo> graphInfo = new SubGraphInfo(subGraph, thisId, graphTag, kind);
  7641. ctx.associate(*graphInfo);
  7642. IHqlExpression * numResultsAttr = expr->queryProperty(numResultsAtom);
  7643. if (numResultsAttr)
  7644. addGraphAttributeInt(subGraph, "_numResults", getIntValue(numResultsAttr->queryChild(0), 0));
  7645. if (expr->hasProperty(multiInstanceAtom))
  7646. subGraph->setPropBool("@multiInstance", true);
  7647. if (expr->hasProperty(delayedAtom))
  7648. subGraph->setPropBool("@delayed", true);
  7649. if (expr->queryProperty(childAtom))
  7650. subGraph->setPropBool("@child", true);
  7651. if (insideChildGraph(ctx))
  7652. {
  7653. graphAttr->setProp("@name", "_kind");
  7654. graphAttr->setPropInt("@value", TAKsubgraph);
  7655. ActivityInstance * curActivityInstance = queryCurrentActivity(ctx);
  7656. if (curActivityInstance)
  7657. addGraphAttributeInt(node, "_parentActivity", curActivityInstance->activityId);
  7658. }
  7659. OwnedHqlExpr idExpr = createConstant((__int64)thisId);
  7660. ctx.associateExpr(expr, idExpr);
  7661. BuildCtx subctx(ctx);
  7662. ForEachChild(idx, expr)
  7663. {
  7664. IHqlExpression * cur = expr->queryChild(idx);
  7665. switch (cur->getOperator())
  7666. {
  7667. case no_subgraph:
  7668. doBuildThorChildSubGraph(ctx, cur, SubGraphChild, 0, graphTag);
  7669. break;
  7670. case no_attr:
  7671. case no_attr_expr:
  7672. case no_attr_link:
  7673. break;
  7674. default:
  7675. buildRootActivity(subctx, cur);
  7676. break;
  7677. }
  7678. }
  7679. ctx.removeAssociation(graphInfo);
  7680. return thisId;
  7681. }
  7682. unsigned HqlCppTranslator::doBuildThorSubGraph(BuildCtx & ctx, IHqlExpression * expr, SubGraphType kind, unsigned reservedId, IHqlExpression * graphTag)
  7683. {
  7684. BuildCtx graphctx(ctx);
  7685. graphctx.addGroup();
  7686. bool needToCreateGraph = !graph;
  7687. if (!graphTag && outputLibraryId)
  7688. graphTag = outputLibraryId;
  7689. if (needToCreateGraph)
  7690. beginGraph(graphctx);
  7691. unsigned thisId = doBuildThorChildSubGraph(graphctx, expr, kind, reservedId, graphTag);
  7692. if (needToCreateGraph)
  7693. endGraph();
  7694. return thisId;
  7695. }
  7696. void HqlCppTranslator::beginGraph(BuildCtx & ctx, const char * _graphName)
  7697. {
  7698. if (activeGraphName)
  7699. {
  7700. //buildStmt(ctx, expr->queryChild(0));
  7701. //return;
  7702. throwError(HQLERR_NestedThorNodes);
  7703. }
  7704. graphSeqNumber++;
  7705. StringBuffer graphName;
  7706. if (!_graphName)
  7707. graphName.append("graph").append(graphSeqNumber);
  7708. else
  7709. graphName.append(_graphName);
  7710. activeGraphName.set(graphName.str());
  7711. graph.setown(createPTree("graph"));
  7712. if (graphLabel)
  7713. {
  7714. graph->setProp("@label", graphLabel);
  7715. graphLabel.clear();
  7716. }
  7717. if (insideLibrary())
  7718. graph->setPropBool("@library", true);
  7719. }
  7720. void HqlCppTranslator::endGraph()
  7721. {
  7722. graphs.append(* new GeneratedGraphInfo(activeGraphName, graph));
  7723. graph.clear();
  7724. activeGraphName.set(NULL);
  7725. }
  7726. void HqlCppTranslator::clearGraph()
  7727. {
  7728. graph.clear();
  7729. activeGraphName.clear();
  7730. }
  7731. /*
  7732. Tricky getting this in the correct order, problems are:
  7733. 1. generally best to move limits over projects and everything else
  7734. 2. Don't move a limit over a project if that will be part of a compound disk/index read.
  7735. 3. Need to make sure preloaded items aren't messed up by the resourcing.
  7736. 4. Resourcing ensures that shared items are explicitly shared if needed, it also ensures that
  7737. index-reads are not shared if it would be inefficient.
  7738. 5. Worth optimizing the graph before resourcing because it may move split points - e.g.,
  7739. moving a filter/choosen over a sort/spill greatly reducing the time taken.
  7740. 6. Optimizing may move items over conditions which can improve the executed code. E.g.,
  7741. where it allows a limit to be added to something.
  7742. */
  7743. IHqlExpression * HqlCppTranslator::optimizeCompoundSource(IHqlExpression * expr, unsigned flags)
  7744. {
  7745. unsigned time = msTick();
  7746. CompoundSourceTransformer transformer(*this, flags);
  7747. OwnedHqlExpr ret = transformer.process(expr);
  7748. DEBUG_TIMER("EclServer: tree transform: optimize disk read", msTick()-time);
  7749. return ret.getClear();
  7750. }
  7751. IHqlExpression * HqlCppTranslator::optimizeGraphPostResource(IHqlExpression * expr, unsigned csfFlags)
  7752. {
  7753. LinkedHqlExpr resourced = expr;
  7754. // Second attempt to spot compound disk reads - this time of spill files for thor.
  7755. resourced.setown(optimizeCompoundSource(resourced, csfFlags));
  7756. //insert projects after compound created...
  7757. if (options.optimizeResourcedProjects)
  7758. {
  7759. cycle_t time = msTick();
  7760. OwnedHqlExpr optimized = insertImplicitProjects(*this, resourced.get(), options.optimizeSpillProject);
  7761. DEBUG_TIMER("EclServer: implicit projects", msTick()-time);
  7762. traceExpression("AfterResourcedImplicit", resourced);
  7763. if (optimized != resourced)
  7764. resourced.setown(optimizeCompoundSource(optimized, csfFlags));
  7765. }
  7766. //Now call the optimizer again - the main purpose is to move projects over limits and into compound index/disk reads
  7767. if (options.optimizeGraph)
  7768. {
  7769. unsigned time = msTick();
  7770. traceExpression("BeforeOptimize2", resourced);
  7771. resourced.setown(optimizeHqlExpression(resourced, getOptimizeFlags()|HOOcompoundproject));
  7772. traceExpression("AfterOptimize2", resourced);
  7773. DEBUG_TIMER("EclServer: optimize graph", msTick()-time);
  7774. }
  7775. resourced.setown(optimizeCompoundSource(resourced, csfFlags));
  7776. return resourced.getClear();
  7777. }
  7778. IHqlExpression * HqlCppTranslator::getResourcedGraph(IHqlExpression * expr, IHqlExpression * graphIdExpr)
  7779. {
  7780. LinkedHqlExpr resourced = expr;
  7781. unsigned csfFlags = CSFindex|options.optimizeDiskFlag;
  7782. if (!targetRoxie())
  7783. csfFlags |= CSFcompoundSpill;
  7784. //Convert queries on preloaded into compound activities - before resourcing so keyed gets done correctly
  7785. checkNormalized(expr);
  7786. traceExpression("BeforeCompound", resourced);
  7787. if (true)
  7788. resourced.setown(optimizeCompoundSource(resourced, CSFpreload|csfFlags));
  7789. // Call optimizer before resourcing so items get moved over conditions, and remove other items
  7790. // which would otherwise cause extra spills.
  7791. traceExpression("BeforeOptimize", resourced);
  7792. unsigned optFlags = getOptimizeFlags();
  7793. checkNormalized(resourced);
  7794. if (options.optimizeGraph)
  7795. {
  7796. unsigned time = msTick();
  7797. resourced.setown(optimizeHqlExpression(resourced, optFlags|HOOfiltersharedproject));
  7798. //have the following on an "aggressive fold" option? If no_selects extract constants it can be quite impressive (jholt22.hql)
  7799. //resourced.setown(foldHqlExpression(resourced));
  7800. DEBUG_TIMER("EclServer: optimize graph", msTick()-time);
  7801. }
  7802. traceExpression("AfterOptimize", resourced);
  7803. checkNormalized(resourced);
  7804. if (true)
  7805. resourced.setown(optimizeCompoundSource(resourced, CSFpreload|csfFlags));
  7806. //Now resource the graph....
  7807. unsigned numNodes = 0;
  7808. // Owned<IConstWUClusterInfo> clusterInfo = wu()->getClusterInfo(curCluster);
  7809. // if (clusterInfo)
  7810. // numNodes = clusterInfo->getSize();
  7811. if (options.specifiedClusterSize != 0)
  7812. numNodes = options.specifiedClusterSize;
  7813. traceExpression("BeforeResourcing", resourced);
  7814. cycle_t time = msTick();
  7815. if (outputLibraryId)
  7816. {
  7817. unsigned numResults = outputLibrary->numResultsUsed();
  7818. resourced.setown(resourceLibraryGraph(*this, resourced, targetClusterType, numNodes, outputLibraryId, &numResults));
  7819. HqlExprArray children;
  7820. unwindCommaCompound(children, resourced);
  7821. children.append(*createAttribute(numResultsAtom, getSizetConstant(numResults)));
  7822. children.append(*createAttribute(multiInstanceAtom)); // since can be called from multiple places.
  7823. resourced.setown(createValue(no_subgraph, makeVoidType(), children));
  7824. }
  7825. else
  7826. resourced.setown(resourceThorGraph(*this, resourced, targetClusterType, numNodes, graphIdExpr));
  7827. if (!resourced)
  7828. return NULL;
  7829. DEBUG_TIMER("EclServer: resource graph", msTick()-time);
  7830. traceExpression("AfterResourcing", resourced);
  7831. if (options.regressionTest)
  7832. checkDependencyConsistency(resourced);
  7833. checkNormalized(resourced);
  7834. if (allowCountIndex())
  7835. {
  7836. //Second attempt to spot count index and count disk file -
  7837. //catches counts on persist files, and counts on indexes enabled by the optimizeHqlExpression
  7838. ITypeInfo * type = resourced->queryType();
  7839. if (type && (type->isScalar() || type->getTypeCode()==type_void))
  7840. {
  7841. ThorCountTransformer countTransformer(*this, false);
  7842. resourced.setown(countTransformer.transformRoot(resourced));
  7843. }
  7844. }
  7845. resourced.setown(optimizeGraphPostResource(resourced, csfFlags));
  7846. if (options.optimizeSpillProject)
  7847. {
  7848. resourced.setown(convertSpillsToActivities(resourced));
  7849. resourced.setown(optimizeGraphPostResource(resourced, csfFlags));
  7850. }
  7851. checkNormalized(resourced);
  7852. //Finally create a couple of special compound activities.
  7853. //e.g., filtered fetch, limited keyed join
  7854. {
  7855. unsigned time = msTick();
  7856. CompoundActivityTransformer transformer(targetClusterType);
  7857. resourced.setown(transformer.transformRoot(resourced));
  7858. traceExpression("AfterCompoundActivity", resourced);
  7859. DEBUG_TIMER("EclServer: tree transform: compound activity", msTick()-time);
  7860. }
  7861. resourced.setown(spotTableInvariant(resourced));
  7862. traceExpression("TableInvariant", resourced);
  7863. checkNormalized(resourced);
  7864. return resourced.getClear();
  7865. }
  7866. void HqlCppTranslator::doBuildThorGraph(BuildCtx & ctx, IHqlExpression * expr)
  7867. {
  7868. assertex(expr->queryType()->getTypeCode() == type_void);
  7869. if (outputLibrary)
  7870. buildLibraryGraph(ctx, expr, NULL);
  7871. else
  7872. {
  7873. beginGraph(ctx);
  7874. unsigned id = 0;
  7875. OwnedHqlExpr graphTag = NULL;//WIP:createAttribute(graphAtom, createConstant((__int64)id));
  7876. OwnedHqlExpr resourced = getResourcedGraph(expr->queryChild(0), graphTag);
  7877. if (resourced)
  7878. {
  7879. traceExpression("beforeGenerate", resourced);
  7880. BuildCtx graphctx(ctx);
  7881. graphctx.addGroup();
  7882. Owned<SubGraphInfo> graphInfo;
  7883. if (graphTag)
  7884. {
  7885. graphInfo.setown(new SubGraphInfo(graph, 0, graphTag, SubGraphRoot));
  7886. graphctx.associate(*graphInfo);
  7887. }
  7888. activeGraphCtx = &graphctx;
  7889. buildStmt(graphctx, resourced);
  7890. activeGraphCtx = NULL;
  7891. graphctx.removeAssociation(graphInfo);
  7892. HqlExprArray args;
  7893. args.append(*createConstant(activeGraphName));
  7894. args.append(*createConstant(targetThor()));
  7895. args.append(*createConstant(0));
  7896. args.append(*createValue(no_nullptr, makeReferenceModifier(makeRowType(queryNullRecord()->getType()))));
  7897. callProcedure(ctx, executeGraphAtom, args);
  7898. }
  7899. endGraph();
  7900. }
  7901. }
  7902. void HqlCppTranslator::buildReturnCsvValue(BuildCtx & ctx, IHqlExpression * _expr)
  7903. {
  7904. LinkedHqlExpr expr = _expr;
  7905. IValue * value = expr->queryValue();
  7906. if (value && isUnicodeType(value->queryType()))
  7907. {
  7908. StringBuffer temp;
  7909. value->getUTF8Value(temp);
  7910. expr.setown(createConstant(createStringValue(temp.str(), temp.length())));
  7911. }
  7912. buildReturn(ctx, expr, constUnknownVarStringType);
  7913. }
  7914. void HqlCppTranslator::buildCsvListFunc(BuildCtx & classctx, const char * func, IHqlExpression * attr, const char * defaultValue)
  7915. {
  7916. BuildCtx funcctx(classctx);
  7917. StringBuffer s;
  7918. s.clear().append("virtual const char * ").append(func).append("(unsigned idx)");
  7919. funcctx.addQuotedCompound(s);
  7920. if (attr || defaultValue)
  7921. {
  7922. OwnedHqlExpr idxVar = createVariable("idx", LINK(unsignedType));
  7923. IHqlExpression * value = attr ? attr->queryChild(0) : NULL;
  7924. if (!value || !isEmptyList(value))
  7925. {
  7926. IHqlStmt * caseStmt = funcctx.addSwitch(idxVar);
  7927. if (value)
  7928. {
  7929. if (!value->isList())
  7930. {
  7931. OwnedHqlExpr label = createConstant((__int64)0);
  7932. funcctx.addCase(caseStmt, label);
  7933. buildReturnCsvValue(funcctx, value);
  7934. }
  7935. else
  7936. {
  7937. ForEachChild(idx, value)
  7938. {
  7939. OwnedHqlExpr label = createConstant((__int64)idx);
  7940. funcctx.addCase(caseStmt, label);
  7941. buildReturnCsvValue(funcctx, value->queryChild(idx));
  7942. }
  7943. }
  7944. }
  7945. else
  7946. {
  7947. unsigned entry = 0;
  7948. const char * start = defaultValue;
  7949. loop
  7950. {
  7951. const char * end = strchr(start, '|');
  7952. if (!end) end = start+strlen(start);
  7953. s.clear().append("case ").append(entry++).append(": return ");
  7954. appendStringAsQuotedCPP(s, end-start, start, false);
  7955. s.append(";");
  7956. funcctx.addQuoted(s);
  7957. if (!*end)
  7958. break;
  7959. start = end+1;
  7960. }
  7961. }
  7962. funcctx.addDefault(caseStmt);
  7963. }
  7964. }
  7965. funcctx.addReturn(queryQuotedNullExpr());
  7966. }
  7967. static void expandDefaultString(StringBuffer & out, IHqlExpression * property, const char * defaultValue)
  7968. {
  7969. IHqlExpression * value = property ? property->queryChild(0) : NULL;
  7970. if (value && value->queryValue())
  7971. value->queryValue()->getStringValue(out);
  7972. else
  7973. out.append(defaultValue);
  7974. }
  7975. void HqlCppTranslator::buildCsvParameters(BuildCtx & subctx, IHqlExpression * csvAttr, IHqlExpression * record, bool isReading)
  7976. {
  7977. HqlExprArray attrs;
  7978. if (csvAttr)
  7979. unwindChildren(attrs, csvAttr);
  7980. BuildCtx classctx(subctx);
  7981. StringBuffer s;
  7982. beginNestedClass(classctx, "csv", "ICsvParameters");
  7983. doBuildBoolFunction(classctx, "queryEBCDIC", queryProperty(ebcdicAtom, attrs)!=NULL);
  7984. bool singleHeader = false;
  7985. bool manyHeader = false;
  7986. IHqlExpression * headerAttr = queryProperty(headerAtom, attrs);
  7987. IHqlExpression * terminator = queryProperty(terminatorAtom, attrs);
  7988. IHqlExpression * separator = queryProperty(separatorAtom, attrs);
  7989. if (headerAttr)
  7990. {
  7991. IHqlExpression * header = queryRealChild(headerAttr, 0);
  7992. if (header)
  7993. {
  7994. if (header->queryType()->isInteger())
  7995. {
  7996. classctx.addQuoted("virtual const char * queryHeader() { return NULL; }");
  7997. doBuildUnsignedFunction(classctx, "queryHeaderLen", header);
  7998. }
  7999. else
  8000. {
  8001. doBuildVarStringFunction(classctx, "queryHeader", header);
  8002. classctx.addQuoted("virtual unsigned queryHeaderLen() { return 1; }");
  8003. }
  8004. }
  8005. else
  8006. {
  8007. StringBuffer names;
  8008. if (!isReading)
  8009. {
  8010. StringBuffer comma;
  8011. expandDefaultString(comma, separator, ",");
  8012. expandFieldNames(names, record, comma.str(), queryPropertyChild(headerAttr, formatAtom, 0));
  8013. expandDefaultString(names, terminator, "\n");
  8014. }
  8015. OwnedHqlExpr namesExpr = createConstant(names.str());
  8016. doBuildVarStringFunction(classctx, "queryHeader", namesExpr);
  8017. classctx.addQuoted("virtual unsigned queryHeaderLen() { return 1; }");
  8018. }
  8019. if (isReading)
  8020. {
  8021. manyHeader = headerAttr->hasProperty(manyAtom) && !headerAttr->hasProperty(singleAtom);
  8022. singleHeader = !manyHeader;
  8023. }
  8024. else
  8025. {
  8026. if (queryRealChild(headerAttr, 1))
  8027. doBuildVarStringFunction(classctx, "queryFooter", headerAttr->queryChild(1));
  8028. if (headerAttr->hasProperty(singleAtom))
  8029. singleHeader = true;
  8030. else
  8031. manyHeader = true;
  8032. }
  8033. }
  8034. else
  8035. {
  8036. classctx.addQuoted("virtual const char * queryHeader() { return NULL; }");
  8037. classctx.addQuoted("virtual unsigned queryHeaderLen() { return 0; }");
  8038. }
  8039. doBuildSizetFunction(classctx, "queryMaxSize", getCsvMaxLength(csvAttr));
  8040. buildCsvListFunc(classctx, "queryQuote", queryProperty(quoteAtom, attrs), isReading ? "'" : NULL);
  8041. buildCsvListFunc(classctx, "querySeparator", separator, ",");
  8042. buildCsvListFunc(classctx, "queryTerminator", terminator, isReading ? "\r\n|\n" : "\n");
  8043. StringBuffer flags;
  8044. if (!queryProperty(quoteAtom, attrs)) flags.append("|defaultQuote");
  8045. if (!queryProperty(separatorAtom, attrs)) flags.append("|defaultSeparate");
  8046. if (!queryProperty(terminatorAtom, attrs)) flags.append("|defaultTerminate");
  8047. if (singleHeader) flags.append("|singleHeaderFooter");
  8048. if (manyHeader) flags.append("|manyHeaderFooter");
  8049. if (queryProperty(noTrimAtom, attrs)) flags.append("|preserveWhitespace");
  8050. if (flags.length() == 0) flags.append("|0");
  8051. doBuildUnsignedFunction(classctx, "getFlags", flags.str()+1);
  8052. endNestedClass();
  8053. subctx.addQuoted("virtual ICsvParameters * queryCsvParameters() { return &csv; }");
  8054. }
  8055. void HqlCppTranslator::buildCsvWriteScalar(BuildCtx & ctx, IHqlExpression * expr, _ATOM encoding)
  8056. {
  8057. ITypeInfo * type = expr->queryType()->queryPromotedType();
  8058. type_t tc = type->getTypeCode();
  8059. LinkedHqlExpr value = expr;
  8060. _ATOM func;
  8061. if (type->isInteger() || tc == type_boolean)
  8062. {
  8063. if (type->isSigned())
  8064. func = writeSignedAtom;
  8065. else
  8066. func = writeUnsignedAtom;
  8067. }
  8068. else if (tc == type_real)
  8069. func = writeRealAtom;
  8070. else if (tc == type_utf8)
  8071. {
  8072. func = writeUtf8Atom;
  8073. value.setown(createValue(no_trim, makeUtf8Type(UNKNOWN_LENGTH, NULL), LINK(value)));
  8074. }
  8075. else if (isUnicodeType(type))
  8076. {
  8077. func = writeUnicodeAtom;
  8078. value.setown(createValue(no_trim, makeUnicodeType(UNKNOWN_LENGTH, NULL), LINK(value)));
  8079. }
  8080. else
  8081. {
  8082. func = writeStringAtom;
  8083. value.setown(createValue(no_trim, LINK(unknownStringType), ensureExprType(value, unknownStringType)));
  8084. }
  8085. if (encoding == asciiAtom)
  8086. {
  8087. func = writeStringAtom;
  8088. Owned<ITypeInfo> type = makeStringType(UNKNOWN_LENGTH, getCharset(asciiAtom), NULL);
  8089. value.setown(ensureExprType(value, type));
  8090. }
  8091. else if (encoding == ebcdicAtom)
  8092. {
  8093. func = writeEbcdicAtom;
  8094. Owned<ITypeInfo> type = makeStringType(UNKNOWN_LENGTH, getCharset(ebcdicAtom), NULL);
  8095. value.setown(ensureExprType(value, type));
  8096. }
  8097. else if (encoding == unicodeAtom)
  8098. {
  8099. func = writeUnicodeAtom;
  8100. Owned<ITypeInfo> type = makeUnicodeType(UNKNOWN_LENGTH, NULL);
  8101. value.setown(ensureExprType(value, type));
  8102. }
  8103. HqlExprArray args;
  8104. args.append(*createVariable("out", makeBoolType()));
  8105. args.append(*LINK(value));
  8106. buildFunctionCall(ctx, func, args);
  8107. }
  8108. void HqlCppTranslator::buildCsvWriteTransform(BuildCtx & subctx, IHqlExpression * expr, IHqlExpression * selector, _ATOM encoding)
  8109. {
  8110. switch (expr->getOperator())
  8111. {
  8112. case no_field:
  8113. {
  8114. ITypeInfo * type = expr->queryType()->queryPromotedType();
  8115. OwnedHqlExpr selected = createSelectExpr(LINK(selector), LINK(expr));
  8116. if (type->getTypeCode() == type_row)
  8117. {
  8118. buildCsvWriteTransform(subctx, expr->queryRecord(), selected, encoding);
  8119. break;
  8120. }
  8121. if (expr->isDataset())
  8122. {
  8123. //May as well output datasets in csv in some way - a count followed by the expanded fields...
  8124. Owned<IHqlCppDatasetCursor> cursor = createDatasetSelector(subctx, selected);
  8125. CHqlBoundExpr boundCount;
  8126. cursor->buildCount(subctx, boundCount);
  8127. OwnedHqlExpr translatedCount = boundCount.getTranslatedExpr();
  8128. buildCsvWriteScalar(subctx, translatedCount, encoding);
  8129. BuildCtx loopctx(subctx);
  8130. BoundRow * row = cursor->buildIterateLoop(loopctx, false);
  8131. buildCsvWriteTransform(loopctx, expr->queryRecord(), selected, encoding);
  8132. return;
  8133. }
  8134. else if (type->getTypeCode() == type_set)
  8135. {
  8136. BuildCtx condctx(subctx);
  8137. Owned<IHqlCppSetCursor> cursor = createSetSelector(condctx, selected);
  8138. CHqlBoundExpr isAll;
  8139. cursor->buildIsAll(condctx, isAll);
  8140. IHqlStmt * stmt = condctx.addFilter(isAll.expr);
  8141. OwnedHqlExpr allText = createConstant("ALL");
  8142. buildCsvWriteScalar(condctx, allText, encoding);
  8143. condctx.selectElse(stmt);
  8144. CHqlBoundExpr boundCurElement;
  8145. cursor->buildIterateLoop(condctx, boundCurElement, false);
  8146. OwnedHqlExpr curElement = boundCurElement.getTranslatedExpr();
  8147. buildCsvWriteScalar(condctx, curElement, encoding);
  8148. }
  8149. else
  8150. buildCsvWriteScalar(subctx, selected, encoding);
  8151. break;
  8152. }
  8153. case no_ifblock:
  8154. {
  8155. OwnedHqlExpr cond = replaceSelector(expr->queryChild(0), querySelfReference(), selector);
  8156. BuildCtx condctx(subctx);
  8157. buildFilter(condctx, cond);
  8158. buildCsvWriteTransform(condctx, expr->queryChild(1), selector, encoding);
  8159. break;
  8160. }
  8161. case no_record:
  8162. {
  8163. ForEachChild(idx, expr)
  8164. buildCsvWriteTransform(subctx, expr->queryChild(idx), selector, encoding);
  8165. break;
  8166. }
  8167. case no_attr:
  8168. case no_attr_expr:
  8169. case no_attr_link:
  8170. break;
  8171. }
  8172. }
  8173. void HqlCppTranslator::buildCsvWriteTransform(BuildCtx & subctx, IHqlExpression * dataset, _ATOM encoding)
  8174. {
  8175. BuildCtx funcctx(subctx);
  8176. funcctx.addQuotedCompound("void writeRow(const byte * self, ITypedOutputStream * out)");
  8177. BoundRow * cursor = bindTableCursor(funcctx, dataset, "self");
  8178. buildCsvWriteTransform(funcctx, dataset->queryRecord(), cursor->querySelector(), encoding);
  8179. }
  8180. void HqlCppTranslator::buildExpiryHelper(BuildCtx & ctx, IHqlExpression * expireAttr)
  8181. {
  8182. if (expireAttr)
  8183. {
  8184. LinkedHqlExpr num = expireAttr->queryChild(0);
  8185. if (!num)
  8186. num.setown(getSizetConstant(DEFAULT_EXPIRY_PERIOD));
  8187. doBuildUnsignedFunction(ctx, "getExpiryDays", num);
  8188. }
  8189. }
  8190. void HqlCppTranslator::buildEncryptHelper(BuildCtx & ctx, IHqlExpression * encryptAttr, const char * funcname)
  8191. {
  8192. if (encryptAttr)
  8193. {
  8194. if (!funcname) funcname = "getEncryptKey";
  8195. doBuildDataFunction(ctx, funcname, encryptAttr->queryChild(0));
  8196. }
  8197. }
  8198. void HqlCppTranslator::buildUpdateHelper(BuildCtx & ctx, ActivityInstance & instance, IHqlExpression * input, IHqlExpression * updateAttr)
  8199. {
  8200. if (updateAttr)
  8201. {
  8202. BuildCtx subctx(ctx);
  8203. subctx.addQuotedCompound("virtual void getUpdateCRCs(unsigned & eclCrc, unsigned __int64 & totalCRC)");
  8204. OwnedHqlExpr eclCrcVar = createVariable("eclCrc", LINK(unsignedType));
  8205. OwnedHqlExpr totalCrcVar = createVariable("totalCRC", makeIntType(8, false));
  8206. IHqlExpression * originalCrc = updateAttr->queryChild(0);
  8207. DependenciesUsed dependencies(true);
  8208. IHqlExpression * filesRead = updateAttr->queryProperty(_files_Atom);
  8209. if (filesRead)
  8210. {
  8211. ForEachChild(i, filesRead)
  8212. dependencies.tablesRead.append(*getNormalizedFilename(filesRead->queryChild(i)));
  8213. }
  8214. IHqlExpression * resultsRead = updateAttr->queryProperty(_results_Atom);
  8215. if (resultsRead)
  8216. unwindChildren(dependencies.resultsRead, resultsRead);
  8217. OwnedHqlExpr crcExpr = calculatePersistInputCrc(subctx, dependencies);
  8218. buildAssignToTemp(subctx, eclCrcVar, originalCrc);
  8219. buildAssignToTemp(subctx, totalCrcVar, crcExpr);
  8220. if (!updateAttr->hasProperty(alwaysAtom))
  8221. instance.addAttributeBool("_updateIfChanged", true);
  8222. }
  8223. }
  8224. void HqlCppTranslator::buildClusterHelper(BuildCtx & ctx, IHqlExpression * expr)
  8225. {
  8226. IHqlExpression * cluster = expr->queryProperty(clusterAtom);
  8227. if (!cluster)
  8228. return;
  8229. BuildCtx funcctx(ctx);
  8230. funcctx.addQuotedCompound("virtual const char * queryCluster(unsigned idx)");
  8231. BuildCtx switchctx(funcctx);
  8232. OwnedHqlExpr var = createVariable("idx", LINK(unsignedType));
  8233. IHqlStmt * switchStmt = switchctx.addSwitch(var);
  8234. unsigned count = 0;
  8235. ForEachChild(i, cluster)
  8236. {
  8237. IHqlExpression * cur = queryRealChild(cluster, i);
  8238. if (cur)
  8239. {
  8240. BuildCtx casectx(switchctx);
  8241. OwnedHqlExpr label = getSizetConstant(count++);
  8242. casectx.addCase(switchStmt, label);
  8243. buildReturn(casectx, cur, constUnknownVarStringType);
  8244. // I think this is wrong: it forces the arguments of the CLUSTER on OUTPUT/BUILDINDEX to by topology clusters (i.e. query destinations) instead of thor clusters (i.e. file destinations)
  8245. //if (logger)
  8246. //{
  8247. // OwnedHqlExpr folded = foldHqlExpression(cur);
  8248. // if (folded->queryValue())
  8249. // {
  8250. // StringBuffer clusterText;
  8251. // folded->queryValue()->getStringValue(clusterText);
  8252. // logger->noteCluster(clusterText.str());
  8253. // }
  8254. //}
  8255. }
  8256. }
  8257. funcctx.addReturn(queryQuotedNullExpr());
  8258. }
  8259. void HqlCppTranslator::buildRecordEcl(BuildCtx & subctx, IHqlExpression * dataset, const char * methodName)
  8260. {
  8261. StringBuffer eclFuncName;
  8262. StringBuffer s;
  8263. //Ensure the ECL for the record reflects its serialized form, not the internal form
  8264. OwnedHqlExpr record = getSerializedForm(dataset->queryRecord());
  8265. appendUniqueId(eclFuncName.append("ecl"), getConsistentUID(record));
  8266. BuildCtx declarectx(*code, declareAtom);
  8267. OwnedHqlExpr attr = createAttribute(eclAtom, LINK(record));
  8268. HqlExprAssociation * match = declarectx.queryMatchExpr(attr);
  8269. if (!match)
  8270. {
  8271. StringBuffer eclText;
  8272. getRecordECL(record, eclText);
  8273. BuildCtx funcctx(declarectx);
  8274. funcctx.setNextPriority(EclTextPrio);
  8275. s.append("const char * ").append(eclFuncName).append("(ICodeContext * ctx)");
  8276. funcctx.addQuotedCompound(s);
  8277. OwnedHqlExpr v = addStringLiteral(eclText);
  8278. funcctx.addReturn(v);
  8279. OwnedHqlExpr temp = createVariable(eclFuncName.str(), makeVoidType());
  8280. declarectx.associateExpr(attr, temp);
  8281. if (options.spanMultipleCpp)
  8282. {
  8283. s.clear().append("extern const char * ").append(eclFuncName).append("(ICodeContext * ctx);");
  8284. BuildCtx protoctx(*code, mainprototypesAtom);
  8285. protoctx.addQuoted(s);
  8286. }
  8287. }
  8288. s.clear().append("virtual const char * ").append(methodName).append("() { return ").append(eclFuncName).append("(ctx); }");
  8289. subctx.addQuoted(s);
  8290. }
  8291. void HqlCppTranslator::buildFormatCrcFunction(BuildCtx & ctx, const char * name, IHqlExpression * dataset, IHqlExpression * expr, unsigned payloadDelta)
  8292. {
  8293. IHqlExpression * payload = expr ? expr->queryProperty(_payload_Atom) : NULL;
  8294. OwnedHqlExpr exprToCrc = getSerializedForm(dataset->queryRecord());
  8295. unsigned payloadSize = 1;
  8296. if (payload)
  8297. payloadSize = (unsigned)getIntValue(payload->queryChild(0)) + payloadDelta;
  8298. exprToCrc.setown(createComma(exprToCrc.getClear(), getSizetConstant(payloadSize)));
  8299. traceExpression("crc:", exprToCrc);
  8300. OwnedHqlExpr crc = getSizetConstant(getExpressionCRC(exprToCrc));
  8301. doBuildUnsignedFunction(ctx, name, crc);
  8302. }
  8303. static void createOutputIndexRecord(HqlMapTransformer & mapper, HqlExprArray & fields, IHqlExpression * record, bool isMainRecord, bool allowTranslate)
  8304. {
  8305. unsigned numFields = record->numChildren();
  8306. unsigned max = isMainRecord ? numFields-1 : numFields;
  8307. for (unsigned idx=0; idx < max; idx++)
  8308. {
  8309. IHqlExpression * cur = record->queryChild(idx);
  8310. IHqlExpression * newField = NULL;
  8311. switch (cur->getOperator())
  8312. {
  8313. case no_attr:
  8314. case no_attr_link:
  8315. case no_attr_expr:
  8316. newField = LINK(cur);
  8317. break;
  8318. case no_ifblock:
  8319. {
  8320. HqlExprArray newFields;
  8321. createOutputIndexRecord(mapper, newFields, cur->queryChild(1), false, false);
  8322. newField = createValue(no_ifblock, makeNullType(), mapper.transformRoot(cur->queryChild(0)), createRecord(newFields));
  8323. break;
  8324. }
  8325. case no_record:
  8326. {
  8327. HqlExprArray newFields;
  8328. createOutputIndexRecord(mapper, newFields, cur, false, allowTranslate);
  8329. newField = createRecord(newFields);
  8330. break;
  8331. }
  8332. case no_field:
  8333. if (cur->hasProperty(blobAtom))
  8334. {
  8335. newField = createField(cur->queryName(), makeIntType(8, false), NULL, NULL);
  8336. }
  8337. else if (allowTranslate)
  8338. {
  8339. if (cur->isDatarow() && !isInPayload())
  8340. {
  8341. HqlMapTransformer childMapper;
  8342. HqlExprArray newFields;
  8343. createOutputIndexRecord(childMapper, newFields, cur->queryRecord(), false, allowTranslate);
  8344. OwnedHqlExpr newRecord = createRecord(newFields);
  8345. HqlExprArray args;
  8346. unwindChildren(args, cur);
  8347. newField = createField(cur->queryName(), newRecord->getType(), args);
  8348. }
  8349. else
  8350. {
  8351. OwnedHqlExpr select = createSelectExpr(getActiveTableSelector(), LINK(cur));
  8352. OwnedHqlExpr value = getHozedKeyValue(select);
  8353. ITypeInfo * newType = value->getType();
  8354. newField = createField(cur->queryName(), newType, NULL, extractFieldAttrs(cur));
  8355. //Now set up the mappings for ifblocks
  8356. OwnedHqlExpr selfSelect = createSelectExpr(LINK(querySelfReference()), LINK(cur));
  8357. OwnedHqlExpr physicalSelect = createSelectExpr(LINK(querySelfReference()), LINK(newField));
  8358. OwnedHqlExpr keyValue = convertIndexPhysical2LogicalValue(cur, physicalSelect, true);
  8359. mapper.setMapping(selfSelect, keyValue);
  8360. }
  8361. }
  8362. else
  8363. newField = LINK(cur);
  8364. break;
  8365. }
  8366. fields.append(*newField);
  8367. }
  8368. }
  8369. static void createOutputIndexTransform(HqlExprArray & assigns, IHqlExpression * self, IHqlExpression * tgtRecord, IHqlExpression * srcRecord, IHqlExpression* srcDataset, bool isMainRecord, bool allowTranslate)
  8370. {
  8371. unsigned numFields = srcRecord->numChildren();
  8372. unsigned max = isMainRecord ? numFields-1 : numFields;
  8373. for (unsigned idx=0; idx < max; idx++)
  8374. {
  8375. IHqlExpression * cur = srcRecord->queryChild(idx);
  8376. IHqlExpression * curNew = tgtRecord->queryChild(idx);
  8377. switch (cur->getOperator())
  8378. {
  8379. case no_ifblock:
  8380. createOutputIndexTransform(assigns, self, curNew->queryChild(1), cur->queryChild(1), srcDataset, false, false);
  8381. break;
  8382. case no_record:
  8383. createOutputIndexTransform(assigns, self, curNew, cur, srcDataset, false, allowTranslate);
  8384. break;
  8385. case no_field:
  8386. {
  8387. OwnedHqlExpr select = createSelectExpr(LINK(srcDataset), LINK(cur));
  8388. OwnedHqlExpr value;
  8389. if (cur->hasProperty(blobAtom))
  8390. {
  8391. value.setown(createValue(no_blob2id, curNew->getType(), LINK(select)));
  8392. }
  8393. else if (cur->isDatarow() && !isInPayload())
  8394. {
  8395. OwnedHqlExpr childSelf = createSelectExpr(LINK(self), LINK(curNew));
  8396. createOutputIndexTransform(assigns, childSelf, curNew->queryRecord(), cur->queryRecord(), select, false, allowTranslate);
  8397. }
  8398. else
  8399. {
  8400. if (allowTranslate)
  8401. value.setown(getHozedKeyValue(select));
  8402. else
  8403. value.set(select);
  8404. }
  8405. if (value)
  8406. assigns.append(*createAssign(createSelectExpr(LINK(self), LINK(curNew)), LINK(value)));
  8407. break;
  8408. }
  8409. }
  8410. }
  8411. }
  8412. void HqlCppTranslator::doBuildIndexOutputTransform(BuildCtx & ctx, IHqlExpression * record, OwnedHqlExpr & rawRecord)
  8413. {
  8414. OwnedHqlExpr srcDataset = createDataset(no_anon, LINK(record));
  8415. HqlExprArray fields;
  8416. HqlExprArray assigns;
  8417. HqlMapTransformer mapper;
  8418. createOutputIndexRecord(mapper, fields, record, true, true);
  8419. OwnedHqlExpr newRecord = createRecord(fields);
  8420. rawRecord.set(newRecord);
  8421. OwnedHqlExpr self = getSelf(newRecord);
  8422. createOutputIndexTransform(assigns, self, newRecord, record, srcDataset, true, true);
  8423. OwnedHqlExpr tgtDataset = createDataset(no_anon, newRecord.getLink());
  8424. OwnedHqlExpr transform = createValue(no_newtransform, makeTransformType(newRecord->getType()), assigns);
  8425. BuildCtx subctx(ctx);
  8426. subctx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf, const void * _left, IBlobCreator * blobs, unsigned __int64 & filepos)");
  8427. ensureRowAllocated(subctx, "crSelf");
  8428. subctx.addQuoted("const unsigned char * left = (const unsigned char *) _left;");
  8429. associateBlobHelper(subctx, srcDataset, "blobs");
  8430. BoundRow * selfCursor = bindSelf(subctx, tgtDataset, "crSelf");
  8431. bindTableCursor(subctx, srcDataset, "left");
  8432. associateSkipReturnMarker(subctx, queryZero(), selfCursor);
  8433. doTransform(subctx, transform, selfCursor);
  8434. OwnedHqlExpr fposVar = createVariable("filepos", makeIntType(8, false));
  8435. OwnedHqlExpr fposField = createSelectExpr(LINK(srcDataset), LINK(queryLastField(record)));
  8436. buildAssignToTemp(subctx, fposVar, fposField);
  8437. buildReturnRecordSize(subctx, selfCursor);
  8438. buildMetaMember(ctx, tgtDataset, "queryDiskRecordSize");
  8439. }
  8440. class TranslatorMaxSizeCallback : public IMaxSizeCallback
  8441. {
  8442. public:
  8443. enum { sizeofFposField = 8 };
  8444. TranslatorMaxSizeCallback(HqlCppTranslator & _translator) : translator(_translator) {}
  8445. virtual size32_t getMaxSize(IHqlExpression * record)
  8446. {
  8447. bool isKnownSize, usedDefaultMaxSize;
  8448. size32_t maxSize = getMaxRecordSize(record, translator.getDefaultMaxRecordSize(), isKnownSize, usedDefaultMaxSize);
  8449. if (usedDefaultMaxSize)
  8450. maxSize += sizeofFposField; //adjust maximum size for the fileposition, so consistent with raw record size
  8451. return maxSize;
  8452. }
  8453. protected:
  8454. HqlCppTranslator & translator;
  8455. };
  8456. static HqlTransformerInfo cHqlBlobTransformerInfo("CHqlBlobTransformer");
  8457. class CHqlBlobTransformer : public QuickHqlTransformer
  8458. {
  8459. public:
  8460. CHqlBlobTransformer() : QuickHqlTransformer(cHqlBlobTransformerInfo, NULL) {}
  8461. virtual IHqlExpression * createTransformed(IHqlExpression * expr)
  8462. {
  8463. OwnedHqlExpr transformed = QuickHqlTransformer::createTransformed(expr);
  8464. if ((expr->getOperator() == no_field) && expr->hasProperty(blobAtom))
  8465. return appendOwnedOperand(transformed, createAttribute(_isBlobInIndex_Atom));
  8466. return transformed.getClear();
  8467. }
  8468. };
  8469. IHqlExpression * annotateIndexBlobs(IHqlExpression * expr)
  8470. {
  8471. CHqlBlobTransformer transformer;
  8472. return transformer.transform(expr);
  8473. }
  8474. IDefRecordElement * HqlCppTranslator::createMetaRecord(IHqlExpression * record)
  8475. {
  8476. TranslatorMaxSizeCallback callback(*this);
  8477. return ::createMetaRecord(record, &callback);
  8478. }
  8479. IHqlExpression * HqlCppTranslator::getSerializedLayoutFunction(IHqlExpression * record, unsigned numKeyedFields)
  8480. {
  8481. OwnedHqlExpr serializedRecord = getSerializedForm(record);
  8482. OwnedHqlExpr search = createAttribute(indexLayoutMarkerAtom, LINK(serializedRecord), getSizetConstant(numKeyedFields));
  8483. BuildCtx declarectx(*code, declareAtom);
  8484. HqlExprAssociation * match = declarectx.queryMatchExpr(search);
  8485. if (match)
  8486. return LINK(match->queryExpr());
  8487. //Modify the record, so that blob fields are tagged as represented as they are in the index
  8488. OwnedHqlExpr indexRecord = annotateIndexBlobs(serializedRecord);
  8489. Owned<IDefRecordElement> re = createMetaRecord(indexRecord);
  8490. if (!re)
  8491. return NULL;
  8492. StringBuffer functionName;
  8493. getUniqueId(functionName.append("getLayout"));
  8494. BuildCtx layoutctx(declarectx);
  8495. StringBuffer s;
  8496. s.append("void ").append(functionName).append("(size32_t & __lenResult, void * & __result, IResourceContext * ctx)");
  8497. layoutctx.setNextPriority(RowMetaPrio);
  8498. layoutctx.addQuotedCompound(s);
  8499. Owned<IDefRecordMeta> meta = createDefRecordMeta(re, numKeyedFields);
  8500. MemoryBuffer serialized;
  8501. serializeRecordMeta(serialized, meta, true);
  8502. OwnedHqlExpr metaExpr = createConstant(createDataValue(serialized.toByteArray(), serialized.length()));
  8503. Owned<ITypeInfo> type = makeDataType(UNKNOWN_LENGTH);
  8504. doBuildFunctionReturn(layoutctx, type, metaExpr);
  8505. if (options.showMetaText)
  8506. {
  8507. s.clear().append("/*").newline();
  8508. getRecordMetaAsString(s, meta);
  8509. s.append("*/");
  8510. layoutctx.addQuoted(s.str());
  8511. }
  8512. if (options.spanMultipleCpp)
  8513. {
  8514. s.clear().append("extern void ").append(functionName).append("(size32_t & __lenResult, void * & __result, IResourceContext * ctx);");
  8515. BuildCtx protoctx(*code, mainprototypesAtom);
  8516. protoctx.addQuoted(s);
  8517. }
  8518. OwnedHqlExpr temp = createVariable(functionName, makeVoidType());
  8519. declarectx.associateExpr(search, temp);
  8520. return temp.getLink();
  8521. }
  8522. void HqlCppTranslator::buildSerializedLayoutMember(BuildCtx & ctx, IHqlExpression * record, const char * name, unsigned numKeyedFields)
  8523. {
  8524. OwnedHqlExpr func = getSerializedLayoutFunction(record, numKeyedFields);
  8525. if (func)
  8526. {
  8527. StringBuffer s;
  8528. s.append("virtual bool ").append(name).append("(size32_t & __lenResult, void * & __result) { ");
  8529. generateExprCpp(s, func).append("(__lenResult, __result, ctx); return true; }");
  8530. ctx.addQuoted(s);
  8531. }
  8532. }
  8533. ABoundActivity * HqlCppTranslator::doBuildActivityOutputIndex(BuildCtx & ctx, IHqlExpression * expr, bool isRoot)
  8534. {
  8535. IHqlExpression * dataset = expr->queryChild(0);
  8536. IHqlExpression * filename = queryRealChild(expr, 1);
  8537. IHqlExpression * record = dataset->queryRecord();
  8538. IHqlDataset * baseTable = dataset->queryDataset()->queryRootTable();
  8539. if (targetRoxie() && options.checkRoxieRestrictions)
  8540. throwError1(HQLERR_NotSupportInRoxie, "BUILDINDEX");
  8541. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  8542. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKindexwrite, expr, "IndexWrite");
  8543. buildActivityFramework(instance, isRoot);
  8544. buildInstancePrefix(instance);
  8545. //virtual const char * getFileName() { return "x.d00"; }
  8546. buildFilenameFunction(*instance, instance->startctx, "getFileName", filename, hasDynamicFilename(expr));
  8547. //virtual const char * getDatasetName() { return "x.d00"; }
  8548. IHqlExpression * tableName = expr->queryProperty(nameAtom);
  8549. if (tableName)
  8550. doBuildVarStringFunction(instance->startctx, "getDatasetName", tableName->queryChild(0));
  8551. //virtual unsigned getFlags() = 0;
  8552. IHqlExpression * updateAttr = expr->queryProperty(updateAtom);
  8553. IHqlExpression * compressAttr = expr->queryProperty(compressedAtom);
  8554. IHqlExpression * widthExpr = queryPropertyChild(expr, widthAtom, 0);
  8555. bool hasTLK = !expr->hasProperty(noRootAtom);
  8556. bool singlePart = expr->hasProperty(fewAtom);
  8557. if (matchesConstantValue(widthExpr, 1))
  8558. {
  8559. singlePart = true;
  8560. widthExpr = NULL;
  8561. }
  8562. StringBuffer s;
  8563. StringBuffer flags;
  8564. if (expr->hasProperty(overwriteAtom)) flags.append("|TIWoverwrite");
  8565. if (expr->hasProperty(noOverwriteAtom)) flags.append("|TIWnooverwrite");
  8566. if (expr->hasProperty(backupAtom)) flags.append("|TIWbackup");
  8567. if (!filename->isConstant()) flags.append("|TIWvarfilename");
  8568. if (singlePart) flags.append("|TIWsmall");
  8569. if (updateAttr) flags.append("|TIWupdatecrc");
  8570. if (updateAttr && !updateAttr->queryProperty(alwaysAtom)) flags.append("|TIWupdate");
  8571. if (!hasTLK && !singlePart) flags.append("|TIWlocal");
  8572. if (compressAttr)
  8573. {
  8574. if (compressAttr->hasProperty(rowAtom)) flags.append("|TIWrowcompress");
  8575. if (!compressAttr->hasProperty(lzwAtom)) flags.append("|TIWnolzwcompress");
  8576. }
  8577. if (widthExpr) flags.append("|TIWhaswidth");
  8578. if (flags.length())
  8579. doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);
  8580. IHqlExpression * indexNameAttr = expr->queryProperty(indexAtom);
  8581. if (indexNameAttr)
  8582. buildFilenameFunction(*instance, instance->startctx, "getDistributeIndexName", indexNameAttr->queryChild(0), hasDynamicFilename(expr));
  8583. buildExpiryHelper(instance->createctx, expr->queryProperty(expireAtom));
  8584. buildUpdateHelper(instance->createctx, *instance, dataset, updateAttr);
  8585. buildClusterHelper(instance->classctx, expr);
  8586. // virtual unsigned getKeyedSize()
  8587. HqlExprArray fields;
  8588. unwindChildren(fields, record);
  8589. removeProperties(fields);
  8590. fields.popn(numPayloadFields(expr));
  8591. OwnedHqlExpr keyedRecord = createRecord(fields); // must be fixed length => no maxlength
  8592. if (expr->hasProperty(_payload_Atom))
  8593. instance->classctx.addQuoted(s.clear().append("virtual unsigned getKeyedSize() { return ").append(getFixedRecordSize(keyedRecord)).append("; }"));
  8594. else
  8595. instance->classctx.addQuoted(s.clear().append("virtual unsigned getKeyedSize() { return (unsigned) -1; }"));
  8596. //virtual const char * queryRecordECL() = 0;
  8597. buildRecordEcl(instance->createctx, dataset, "queryRecordECL");
  8598. doBuildSequenceFunc(instance->classctx, querySequence(expr), false);
  8599. Owned<IWUResult> result = createDatasetResultSchema(querySequence(expr), queryResultName(expr), dataset->queryRecord(), false, true);
  8600. if (expr->hasProperty(setAtom))
  8601. {
  8602. BuildCtx subctx(instance->startctx);
  8603. subctx.addQuotedCompound("virtual bool getIndexMeta(size32_t & lenName, char * & name, size32_t & lenValue, char * & value, unsigned idx)");
  8604. CHqlBoundTarget nameTarget, valueTarget;
  8605. initBoundStringTarget(nameTarget, unknownStringType, "lenName", "name");
  8606. //more should probably be utf-8 rather than string
  8607. initBoundStringTarget(valueTarget, unknownStringType, "lenValue", "value");
  8608. OwnedHqlExpr idxVar = createVariable("idx", LINK(sizetType));
  8609. BuildCtx casectx(subctx);
  8610. IHqlStmt * switchStmt = casectx.addSwitch(idxVar);
  8611. unsigned count = 0;
  8612. ForEachChild(i, expr)
  8613. {
  8614. IHqlExpression * cur = expr->queryChild(i);
  8615. if (cur->isAttribute() && cur->queryName() == setAtom)
  8616. {
  8617. OwnedHqlExpr label = getSizetConstant(count++);
  8618. casectx.addCase(switchStmt, label);
  8619. buildExprAssign(casectx, nameTarget, cur->queryChild(0));
  8620. buildExprAssign(casectx, valueTarget, cur->queryChild(1));
  8621. buildReturn(casectx, queryBoolExpr(true));
  8622. }
  8623. }
  8624. buildReturn(subctx, queryBoolExpr(false));
  8625. }
  8626. OwnedHqlExpr rawRecord;
  8627. doBuildIndexOutputTransform(instance->startctx, record, rawRecord);
  8628. buildFormatCrcFunction(instance->classctx, "getFormatCrc", rawRecord, expr, 0);
  8629. if (compressAttr && compressAttr->hasProperty(rowAtom))
  8630. {
  8631. if (!isFixedWidthDataset(rawRecord))
  8632. throwError(HQLERR_RowCompressRequireFixedSize);
  8633. }
  8634. if (!expr->hasProperty(fixedAtom))
  8635. buildSerializedLayoutMember(instance->classctx, record, "getIndexLayout", fields.ordinality());
  8636. if (widthExpr)
  8637. {
  8638. doBuildUnsignedFunction(instance->startctx, "getWidth", widthExpr);
  8639. if (!hasTLK)
  8640. {
  8641. HqlExprArray sorts;
  8642. gatherIndexBuildSortOrder(sorts, expr, options.sortIndexPayload);
  8643. OwnedHqlExpr sortOrder = createValueSafe(no_sortlist, makeSortListType(NULL), sorts);
  8644. instance->startctx.addQuoted("virtual ICompare * queryCompare() { return &compare; }");
  8645. DatasetReference dsRef(dataset);
  8646. buildCompareClass(instance->nestedctx, "compare", sortOrder, dsRef);
  8647. }
  8648. }
  8649. buildInstanceSuffix(instance);
  8650. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  8651. OwnedHqlExpr dependency = createAttribute(fileAtom, getNormalizedFilename(filename));
  8652. Owned<ABoundActivity> bound = instance->getBoundActivity();
  8653. OwnedHqlExpr boundUnknown = createUnknown(no_attr, NULL, NULL, LINK(bound));
  8654. ctx.associateExpr(dependency, boundUnknown);
  8655. return instance->getBoundActivity();
  8656. }
  8657. void HqlCppTranslator::buildCsvWriteMembers(ActivityInstance * instance, IHqlExpression * dataset, IHqlExpression * csvAttr)
  8658. {
  8659. buildCsvParameters(instance->nestedctx, csvAttr, dataset->queryRecord(), false);
  8660. buildCsvWriteTransform(instance->startctx, dataset, queryCsvEncoding(csvAttr));
  8661. }
  8662. void HqlCppTranslator::buildXmlWriteMembers(ActivityInstance * instance, IHqlExpression * dataset, IHqlExpression * xmlAttr)
  8663. {
  8664. buildXmlSerialize(instance->startctx, dataset, "toXML", false);
  8665. IHqlExpression * rowAttr = xmlAttr->queryProperty(rowAtom);
  8666. if (rowAttr)
  8667. doBuildVarStringFunction(instance->startctx, "queryIteratorPath", rowAttr->queryChild(0));
  8668. IHqlExpression * headerAttr = xmlAttr->queryProperty(headerAtom);
  8669. if (headerAttr)
  8670. {
  8671. doBuildVarStringFunction(instance->startctx, "queryHeader", headerAttr->queryChild(0));
  8672. doBuildVarStringFunction(instance->startctx, "queryFooter", headerAttr->queryChild(1));
  8673. }
  8674. StringBuffer xmlFlags;
  8675. if (xmlAttr->hasProperty(trimAtom))
  8676. xmlFlags.append("|XWFtrim");
  8677. if (xmlAttr->hasProperty(optAtom))
  8678. xmlFlags.append("|XWFopt");
  8679. if (xmlFlags.length())
  8680. {
  8681. StringBuffer s;
  8682. s.append("virtual unsigned getXmlFlags() { return ").append(xmlFlags.str()+1).append("; }");
  8683. instance->classctx.addQuoted(s);
  8684. }
  8685. }
  8686. ABoundActivity * HqlCppTranslator::doBuildActivityOutput(BuildCtx & ctx, IHqlExpression * expr, bool isRoot)
  8687. {
  8688. IHqlExpression * dataset = expr->queryChild(0);
  8689. IHqlExpression * filename = queryRealChild(expr, 1);
  8690. if (!filename)
  8691. return doBuildActivityOutputWorkunit(ctx, expr, isRoot);
  8692. IHqlExpression * program = queryRealChild(expr, 2);
  8693. IHqlExpression * csvAttr = expr->queryProperty(csvAtom);
  8694. IHqlExpression * xmlAttr = expr->queryProperty(xmlAtom);
  8695. IHqlExpression * expireAttr = expr->queryProperty(expireAtom);
  8696. IHqlExpression * seq = querySequence(expr);
  8697. IHqlExpression *pipe = NULL;
  8698. if (program)
  8699. {
  8700. if (program->getOperator()==no_pipe)
  8701. pipe = program->queryChild(0);
  8702. }
  8703. else if (filename->getOperator()==no_pipe)
  8704. pipe = filename->queryChild(0);
  8705. if (pipe)
  8706. checkPipeAllowed();
  8707. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  8708. ThorActivityKind kind = TAKdiskwrite;
  8709. const char * activity = "DiskWrite";
  8710. if (expr->getOperator() == no_spill)
  8711. {
  8712. kind = TAKspill;
  8713. activity = "Spill";
  8714. }
  8715. else if (pipe)
  8716. {
  8717. kind = TAKpipewrite;
  8718. activity = "PipeWrite";
  8719. }
  8720. else if (csvAttr)
  8721. {
  8722. kind = TAKcsvwrite;
  8723. activity = "CsvWrite";
  8724. }
  8725. else if (xmlAttr)
  8726. {
  8727. kind = TAKxmlwrite;
  8728. activity = "XmlWrite";
  8729. }
  8730. bool useImplementationClass = options.minimizeActivityClasses && targetRoxie() && expr->hasProperty(_spill_Atom);
  8731. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, kind, expr, activity);
  8732. //Output to a variable filename is either a user result, or a computed workflow spill, both need evaluating.
  8733. if (useImplementationClass)
  8734. instance->setImplementationClass(newMemorySpillSplitArgAtom);
  8735. if ((kind == TAKdiskwrite) && filename->queryValue())
  8736. {
  8737. StringBuffer s;
  8738. s.append(getActivityText(kind));
  8739. if (expr->hasProperty(_spill_Atom))
  8740. s.append("\nSpill File");
  8741. else
  8742. filename->toString(s.append("\n"));
  8743. instance->graphLabel.set(s.str());
  8744. }
  8745. if (pipe)
  8746. {
  8747. if (csvAttr)
  8748. instance->addBaseClass("IHThorCsvWriteExtra", true);
  8749. else if (xmlAttr)
  8750. instance->addBaseClass("IHThorXmlWriteExtra", true);
  8751. }
  8752. buildActivityFramework(instance, isRoot && !isInternalSeq(seq));
  8753. buildInstancePrefix(instance);
  8754. noteResultDefined(ctx, instance, seq, filename, isRoot);
  8755. //virtual const char * getFileName() { return "x.d00"; }
  8756. OwnedHqlExpr tempCount;
  8757. if (expr->hasProperty(_spill_Atom) || expr->hasProperty(jobTempAtom))
  8758. {
  8759. IPropertyTree * graphNode = NULL;
  8760. if (targetRoxie() && expr->hasProperty(jobTempAtom))
  8761. graphNode = instance->graphNode;
  8762. GlobalFileTracker * tracker = new GlobalFileTracker(filename, graphNode);
  8763. globalFiles.append(*tracker);
  8764. OwnedHqlExpr callback = createUnknown(no_callback, LINK(unsignedType), globalAtom, LINK(tracker));
  8765. tempCount.setown(createTranslated(callback));
  8766. }
  8767. if (!useImplementationClass)
  8768. {
  8769. if (pipe)
  8770. {
  8771. //MORE or pipe name is dependent on the input dataset - !constant is not sufficient
  8772. if (expr->hasProperty(repeatAtom))
  8773. {
  8774. //virtual const char * getPipeProgram() { return "grep"; }
  8775. instance->startctx.addQuoted("virtual char * getPipeProgram() { return NULL; }");
  8776. BuildCtx pipeCtx(instance->startctx);
  8777. pipeCtx.addQuotedCompound("virtual char * getNameFromRow(const void * _self)");
  8778. pipeCtx.addQuoted("const unsigned char * self = (const unsigned char *) _self;");
  8779. bindTableCursor(pipeCtx, dataset, "self");
  8780. buildReturn(pipeCtx, pipe, unknownVarStringType);
  8781. }
  8782. else
  8783. {
  8784. //virtual const char * getPipeProgram() { return "grep"; }
  8785. BuildCtx pipeCtx(instance->startctx);
  8786. pipeCtx.addQuotedCompound("virtual char * getPipeProgram()");
  8787. buildReturn(pipeCtx, pipe, unknownVarStringType);
  8788. }
  8789. if (csvAttr)
  8790. instance->classctx.addQuoted("virtual IHThorCsvWriteExtra * queryCsvOutput() { return this; }");
  8791. if (xmlAttr)
  8792. instance->classctx.addQuoted("virtual IHThorXmlWriteExtra * queryXmlOutput() { return this; }");
  8793. StringBuffer flags;
  8794. if (expr->hasProperty(repeatAtom))
  8795. flags.append("|TPFrecreateeachrow");
  8796. if (expr->hasProperty(optAtom))
  8797. flags.append("|TPFnofail");
  8798. if (csvAttr)
  8799. flags.append("|TPFwritecsvtopipe");
  8800. if (xmlAttr)
  8801. flags.append("|TPFwritexmltopipe");
  8802. if (flags.length())
  8803. doBuildUnsignedFunction(instance->classctx, "getPipeFlags", flags.str()+1);
  8804. }
  8805. else
  8806. {
  8807. bool constFilename = true;
  8808. //virtual const char * getFileName() = 0;
  8809. if (filename && filename->getOperator() != no_pipe)
  8810. {
  8811. buildFilenameFunction(*instance, instance->startctx, "getFileName", filename, hasDynamicFilename(expr));
  8812. if (!filename->isConstant())
  8813. constFilename = false;
  8814. }
  8815. else
  8816. {
  8817. BuildCtx getNameCtx(instance->startctx);
  8818. getNameCtx.addQuotedCompound("virtual const char * getFileName()");
  8819. getNameCtx.addReturn(queryQuotedNullExpr());
  8820. }
  8821. //virtual unsigned getFlags() = 0;
  8822. IHqlExpression * updateAttr = expr->queryProperty(updateAtom);
  8823. StringBuffer s;
  8824. StringBuffer flags;
  8825. if (expr->hasProperty(_spill_Atom)) flags.append("|TDXtemporary");
  8826. if (expr->hasProperty(groupedAtom)) flags.append("|TDXgrouped");
  8827. if (expr->hasProperty(compressedAtom)) flags.append("|TDWnewcompress");
  8828. if (expr->hasProperty(__compressed__Atom)) flags.append("|TDXcompress");
  8829. if (expr->hasProperty(extendAtom)) flags.append("|TDWextend");
  8830. if (expr->hasProperty(overwriteAtom)) flags.append("|TDWoverwrite");
  8831. if (expr->hasProperty(noOverwriteAtom)) flags.append("|TDWnooverwrite");
  8832. if (expr->hasProperty(_workflowPersist_Atom)) flags.append("|TDWpersist");
  8833. if (expr->hasProperty(_noReplicate_Atom)) flags.append("|TDWnoreplicate");
  8834. if (expr->hasProperty(backupAtom)) flags.append("|TDWbackup");
  8835. if (expr->hasProperty(resultAtom)) flags.append("|TDWowned|TDWresult");
  8836. if (expr->hasProperty(ownedAtom)) flags.append("|TDWowned");
  8837. if (!constFilename) flags.append("|TDXvarfilename");
  8838. if (hasDynamicFilename(expr)) flags.append("|TDXdynamicfilename");
  8839. if (expr->hasProperty(jobTempAtom)) flags.append("|TDXjobtemp");
  8840. if (updateAttr) flags.append("|TDWupdatecrc");
  8841. if (updateAttr && !updateAttr->queryProperty(alwaysAtom)) flags.append("|TDWupdate");
  8842. if (flags.length())
  8843. doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);
  8844. //virtual const char * queryRecordECL() = 0;
  8845. buildRecordEcl(instance->createctx, dataset, "queryRecordECL");
  8846. buildExpiryHelper(instance->createctx, expireAttr);
  8847. buildUpdateHelper(instance->createctx, *instance, dataset, updateAttr);
  8848. }
  8849. doBuildSequenceFunc(instance->classctx, seq, true);
  8850. if (tempCount)
  8851. {
  8852. if ((kind != TAKspill) || !matchesConstantValue(tempCount, 1))
  8853. {
  8854. BuildCtx ctx1(instance->classctx);
  8855. ctx1.addQuotedCompound("virtual unsigned getTempUsageCount()");
  8856. buildReturn(ctx1, tempCount, unsignedType);
  8857. }
  8858. }
  8859. Owned<IWUResult> result = createDatasetResultSchema(seq, queryResultName(expr), dataset->queryRecord(), (kind != TAKcsvwrite) && (kind != TAKxmlwrite), true);
  8860. if (expr->hasProperty(resultAtom))
  8861. result->setResultRowLimit(-1);
  8862. buildFormatCrcFunction(instance->classctx, "getFormatCrc", dataset, NULL, 0);
  8863. LinkedHqlExpr diskDataset = dataset;
  8864. if (!expr->hasProperty(groupedAtom) && isGroupedActivity(dataset))
  8865. diskDataset.setown(createDataset(no_group, LINK(dataset), NULL));
  8866. if ((kind != TAKspill) || (diskDataset->queryType() != expr->queryType()))
  8867. buildMetaMember(instance->classctx, diskDataset, "queryDiskRecordSize");
  8868. buildClusterHelper(instance->classctx, expr);
  8869. //Both csv write and pipe with csv/xml format
  8870. if (csvAttr)
  8871. buildCsvWriteMembers(instance, dataset, csvAttr);
  8872. if (xmlAttr)
  8873. buildXmlWriteMembers(instance, dataset, xmlAttr);
  8874. buildEncryptHelper(instance->startctx, expr->queryProperty(encryptAtom));
  8875. }
  8876. else
  8877. {
  8878. assertex(tempCount.get() && !hasDynamic(expr));
  8879. instance->addConstructorParameter(tempCount);
  8880. addFilenameConstructorParameter(*instance, "getFileName", filename);
  8881. }
  8882. instance->addAttributeBool("_isSpill", expr->hasProperty(_spill_Atom));
  8883. if (targetRoxie())
  8884. instance->addAttributeBool("_isSpillGlobal", expr->hasProperty(jobTempAtom));
  8885. buildInstanceSuffix(instance);
  8886. if (boundDataset)
  8887. {
  8888. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  8889. }
  8890. OwnedHqlExpr dependency = createAttribute(fileAtom, getNormalizedFilename(filename));
  8891. Owned<ABoundActivity> bound = instance->getBoundActivity();
  8892. OwnedHqlExpr boundUnknown = createUnknown(no_attr, NULL, NULL, LINK(bound));
  8893. activeGraphCtx->associateExpr(dependency, boundUnknown);
  8894. IHqlExpression * name = queryPropertyChild(expr, namedAtom, 0);
  8895. if (name)
  8896. associateRemoteResult(*instance, seq, name);
  8897. return instance->getBoundActivity();
  8898. }
  8899. void HqlCppTranslator::addSchemaField(IHqlExpression *field, MemoryBuffer &schema, IHqlExpression *selector)
  8900. {
  8901. _ATOM name = field->queryName();
  8902. StringBuffer schemaName;
  8903. if (name)
  8904. {
  8905. schemaName.append(name->str());
  8906. }
  8907. else
  8908. {
  8909. schemaName.append("unknown_name");
  8910. getUniqueId(schemaName);
  8911. }
  8912. schemaName.toLowerCase();
  8913. StringBuffer funcname;
  8914. Linked<ITypeInfo> schemaType = queryUnqualifiedType(field->queryType());
  8915. switch(schemaType->getTypeCode())
  8916. {
  8917. case type_alien:
  8918. schemaType.set(schemaType->queryChildType());
  8919. break;
  8920. case type_bitfield:
  8921. schemaType.set(schemaType->queryPromotedType());
  8922. //fall through;
  8923. case type_table:
  8924. case type_groupedtable:
  8925. case type_row:
  8926. schemaType.setown(makeStringType(UNKNOWN_LENGTH, NULL, NULL));
  8927. break;
  8928. }
  8929. schema.append(schemaName.str());
  8930. schemaType->serialize(schema);
  8931. }
  8932. void HqlCppTranslator::doAddSchemaFields(IHqlExpression * record, MemoryBuffer &schema, IHqlExpression *selector)
  8933. {
  8934. assertex(record->getOperator() == no_record);
  8935. ForEachChild(idx, record)
  8936. {
  8937. IHqlExpression *field = record->queryChild(idx);
  8938. switch (field->getOperator())
  8939. {
  8940. case no_ifblock:
  8941. doAddSchemaFields(field->queryChild(1), schema, selector);
  8942. break;
  8943. case no_record:
  8944. doAddSchemaFields(field, schema, selector);
  8945. break;
  8946. case no_field:
  8947. if (field->isDatarow())
  8948. {
  8949. // MORE - should I record this nesting in the schema?
  8950. // MORE - this does not yet work...
  8951. OwnedHqlExpr subselector = createSelectExpr(LINK(selector), LINK(field));
  8952. doAddSchemaFields(field->queryRecord(), schema, subselector);
  8953. }
  8954. else
  8955. addSchemaField(field, schema, selector);
  8956. break;
  8957. }
  8958. }
  8959. }
  8960. void HqlCppTranslator::getRecordECL(IHqlExpression * deserializedRecord, StringBuffer & eclText)
  8961. {
  8962. OwnedHqlExpr record = getSerializedForm(deserializedRecord);
  8963. if ((options.maxRecordSize != MAX_RECORD_SIZE) && maxRecordSizeUsesDefault(record))
  8964. {
  8965. //Add an explicit record size if default max record size
  8966. size32_t maxSize = getMaxRecordSize(record);
  8967. HqlExprArray args;
  8968. unwindChildren(args, record);
  8969. args.append(*createAttribute(maxLengthAtom, getSizetConstant(maxSize)));
  8970. OwnedHqlExpr annotatedRecord = record->clone(args);
  8971. ::getRecordECL(annotatedRecord, eclText);
  8972. }
  8973. else
  8974. ::getRecordECL(record, eclText);
  8975. }
  8976. void HqlCppTranslator::addSchemaFields(IHqlExpression * record, MemoryBuffer &schema, IHqlExpression *selector)
  8977. {
  8978. doAddSchemaFields(record, schema, selector);
  8979. schema.append("").append((unsigned char) type_void);
  8980. StringBuffer eclText;
  8981. getRecordECL(record, eclText);
  8982. schema.append((unsigned)eclText.length());
  8983. schema.append(eclText.length(), eclText.str()); // could compress at this point...
  8984. }
  8985. void HqlCppTranslator::addSchemaResource(int seq, const char * name, IHqlExpression * record)
  8986. {
  8987. StringBuffer xml;
  8988. getRecordXmlSchema(xml, record, true);
  8989. addSchemaResource(seq, name, xml.length()+1, xml.str());
  8990. }
  8991. void HqlCppTranslator::addSchemaResource(int seq, const char * name, unsigned len, const char * schemaXml)
  8992. {
  8993. Owned<IPropertyTree> manifestEntry = createPTree("Resource");
  8994. manifestEntry->setProp("@name", name);
  8995. manifestEntry->setPropInt("@seq", seq);
  8996. code->addCompressResource("RESULT_XSD", len, schemaXml, manifestEntry);
  8997. }
  8998. void HqlCppTranslator::finalizeResources()
  8999. {
  9000. }
  9001. IWUResult * HqlCppTranslator::createDatasetResultSchema(IHqlExpression * sequenceExpr, IHqlExpression * name, IHqlExpression * record, bool createTransformer, bool isFile)
  9002. {
  9003. //Some spills have no sequence attached
  9004. if (!sequenceExpr)
  9005. return NULL;
  9006. int sequence = (int)getIntValue(sequenceExpr);
  9007. Owned<IWUResult> result = createWorkunitResult(sequence, name);
  9008. if (!result)
  9009. return NULL;
  9010. MemoryBuffer schema;
  9011. OwnedHqlExpr self = getSelf(record);
  9012. addSchemaFields(record, schema, self);
  9013. SCMStringBuffer resultName;
  9014. result->getResultName(resultName);
  9015. addSchemaResource(sequence, resultName.str(), record);
  9016. result->setResultSchemaRaw(schema.length(), schema.toByteArray());
  9017. result->setResultScalar(false);
  9018. OwnedHqlExpr serialRecord = getSerializedForm(record);
  9019. OwnedHqlExpr ds = createDataset(no_anon, LINK(serialRecord));
  9020. MetaInstance meta(*this, ds);
  9021. buildMetaInfo(meta);
  9022. result->setResultRecordSizeEntry(meta.metaFactoryName);
  9023. if (targetRoxie() && (sequence >= 0) && !isFile)
  9024. result->setResultFormat(ResultFormatXml);
  9025. if (createTransformer)
  9026. {
  9027. OwnedHqlExpr noVirtualRecord = removeVirtualAttributes(serialRecord);
  9028. Owned<IHqlExpression> transformedRecord = getSimplifiedRecord(noVirtualRecord, false);
  9029. if (transformedRecord)
  9030. {
  9031. OwnedHqlExpr ds = createDataset(no_anon, LINK(noVirtualRecord));
  9032. OwnedHqlExpr seq = createDummySelectorSequence();
  9033. OwnedHqlExpr leftSelect = createSelector(no_left, ds, seq);
  9034. OwnedHqlExpr transform = getSimplifiedTransform(transformedRecord, noVirtualRecord, leftSelect);
  9035. OwnedHqlExpr tds = createDataset(no_anon, LINK(transformedRecord));
  9036. StringBuffer s, name;
  9037. getUniqueId(name.append("tf"));
  9038. BuildCtx transformctx(*code, declareAtom);
  9039. transformctx.setNextPriority(RecordTranslatorPrio);
  9040. s.clear().append("extern \"C\" ECL_API size32_t ").append(name).append("(ARowBuilder & crSelf, const byte * src)");
  9041. transformctx.addQuotedCompound(s);
  9042. BoundRow * selfCursor = bindSelf(transformctx, tds, "crSelf");
  9043. bindTableCursor(transformctx, ds, "src", no_left, seq);
  9044. associateSkipReturnMarker(transformctx, queryZero(), selfCursor);
  9045. ensureRowAllocated(transformctx, "crSelf");
  9046. doTransform(transformctx, transform, selfCursor);
  9047. buildReturnRecordSize(transformctx, selfCursor);
  9048. result->setResultTransformerEntry(name.str());
  9049. }
  9050. }
  9051. return result.getClear();
  9052. }
  9053. //-------------------------------------------------------------------------------------------------------------------
  9054. void HqlCppTranslator::buildXmlSerializeSetValues(BuildCtx & ctx, IHqlExpression * value, IHqlExpression * itemName, bool includeAll)
  9055. {
  9056. OwnedHqlExpr simpleValue = simplifyFixedLengthList(value);
  9057. BuildCtx subctx(ctx);
  9058. Owned<IHqlCppSetCursor> cursor = createSetSelector(ctx, simpleValue);
  9059. if (includeAll)
  9060. {
  9061. CHqlBoundExpr isAll;
  9062. cursor->buildIsAll(ctx, isAll);
  9063. IHqlStmt * stmt = subctx.addFilter(isAll.expr);
  9064. HqlExprArray args;
  9065. args.append(*createVariable("out", makeBoolType()));
  9066. callProcedure(subctx, outputXmlSetAllAtom, args);
  9067. subctx.selectElse(stmt);
  9068. }
  9069. CHqlBoundExpr boundCurElement;
  9070. cursor->buildIterateLoop(subctx, boundCurElement, false);
  9071. OwnedHqlExpr curElement = boundCurElement.getTranslatedExpr();
  9072. buildXmlSerializeScalar(subctx, curElement, itemName);
  9073. }
  9074. void HqlCppTranslator::buildXmlSerializeBeginNested(BuildCtx & ctx, IHqlExpression * name, bool doIndent)
  9075. {
  9076. if (name)
  9077. {
  9078. HqlExprArray args;
  9079. args.append(*createVariable("out", makeBoolType()));
  9080. args.append(*LINK(name));
  9081. args.append(*createConstant(false));
  9082. callProcedure(ctx, outputXmlBeginNestedAtom, args);
  9083. }
  9084. }
  9085. void HqlCppTranslator::buildXmlSerializeEndNested(BuildCtx & ctx, IHqlExpression * name)
  9086. {
  9087. if (name)
  9088. {
  9089. HqlExprArray args;
  9090. args.append(*createVariable("out", makeBoolType()));
  9091. args.append(*LINK(name));
  9092. callProcedure(ctx, outputXmlEndNestedAtom, args);
  9093. }
  9094. }
  9095. void HqlCppTranslator::buildXmlSerializeSet(BuildCtx & ctx, IHqlExpression * field, IHqlExpression * value)
  9096. {
  9097. OwnedHqlExpr name, itemName;
  9098. extractXmlName(name, &itemName, NULL, field, "Item", false);
  9099. HqlExprArray args;
  9100. buildXmlSerializeBeginNested(ctx, name, false);
  9101. buildXmlSerializeSetValues(ctx, value, itemName, (name != NULL));
  9102. buildXmlSerializeEndNested(ctx, name);
  9103. }
  9104. void HqlCppTranslator::buildXmlSerializeDataset(BuildCtx & ctx, IHqlExpression * field, IHqlExpression * value, HqlExprArray * assigns)
  9105. {
  9106. OwnedHqlExpr name, rowName;
  9107. extractXmlName(name, &rowName, NULL, field, "Row", false);
  9108. HqlExprArray args;
  9109. buildXmlSerializeBeginNested(ctx, name, false);
  9110. Owned<IHqlCppDatasetCursor> cursor = createDatasetSelector(ctx, value);
  9111. BuildCtx subctx(ctx);
  9112. BoundRow * sourceRow = cursor->buildIterateLoop(subctx, false);
  9113. buildXmlSerializeBeginNested(subctx, rowName, true);
  9114. StringBuffer boundRowText;
  9115. generateExprCpp(boundRowText, sourceRow->queryBound());
  9116. OwnedHqlExpr ds = createDataset(no_null, LINK(field->queryRecord()));
  9117. buildXmlSerializeUsingMeta(subctx, ds, boundRowText.str());
  9118. buildXmlSerializeEndNested(subctx, rowName);
  9119. buildXmlSerializeEndNested(ctx, name);
  9120. }
  9121. void HqlCppTranslator::buildXmlSerializeScalar(BuildCtx & ctx, IHqlExpression * selected, IHqlExpression * name)
  9122. {
  9123. ITypeInfo * type = selected->queryType()->queryPromotedType();
  9124. LinkedHqlExpr value = selected;
  9125. _ATOM func;
  9126. switch (type->getTypeCode())
  9127. {
  9128. case type_boolean:
  9129. func = outputXmlBoolAtom;
  9130. break;
  9131. case type_string:
  9132. case type_varstring:
  9133. func = outputXmlStringAtom;
  9134. break;
  9135. case type_qstring:
  9136. func = outputXmlQStringAtom;
  9137. break;
  9138. case type_data:
  9139. func = outputXmlDataAtom;
  9140. break;
  9141. case type_unicode:
  9142. case type_varunicode:
  9143. func = outputXmlUnicodeAtom;
  9144. break;
  9145. case type_utf8:
  9146. func = outputXmlUtf8Atom;
  9147. break;
  9148. case type_real:
  9149. func = outputXmlRealAtom;
  9150. break;
  9151. case type_int:
  9152. case type_swapint:
  9153. case type_packedint:
  9154. case type_bitfield:
  9155. if (type->isSigned())
  9156. func = outputXmlIntAtom;
  9157. else
  9158. func = outputXmlUIntAtom;
  9159. break;
  9160. case type_decimal:
  9161. value.setown(ensureExprType(value, unknownStringType));
  9162. func = outputXmlStringAtom;
  9163. break;
  9164. default:
  9165. UNIMPLEMENTED;
  9166. }
  9167. HqlExprArray args;
  9168. args.append(*createVariable("out", makeBoolType()));
  9169. args.append(*value.getLink());
  9170. if (name)
  9171. args.append(*LINK(name));
  9172. else
  9173. args.append(*getNullStringPointer(true));
  9174. buildFunctionCall(ctx, func, args);
  9175. }
  9176. void HqlCppTranslator::buildXmlSerialize(BuildCtx & subctx, IHqlExpression * expr, IHqlExpression * selector, HqlExprArray * assigns, unsigned pass, unsigned & expectedIndex)
  9177. {
  9178. if (anyXmlGeneratedForPass(expr, pass))
  9179. {
  9180. switch (expr->getOperator())
  9181. {
  9182. case no_field:
  9183. {
  9184. OwnedHqlExpr name;
  9185. extractXmlName(name, NULL, NULL, expr, NULL, false);
  9186. LinkedHqlExpr value;
  9187. OwnedHqlExpr selected;
  9188. ITypeInfo * type = expr->queryType()->queryPromotedType();
  9189. if (assigns)
  9190. {
  9191. OwnedHqlExpr match = getExtractMatchingAssign(*assigns, expr, expectedIndex, selector);
  9192. if (!match)
  9193. {
  9194. StringBuffer s;
  9195. expr->toString(s);
  9196. throwError2(HQLERR_MissingTransformAssignXX, s.str(), expr);
  9197. }
  9198. selected.set(match->queryChild(0));
  9199. value.setown(ensureExprType(match->queryChild(1), type));
  9200. }
  9201. else
  9202. {
  9203. selected.setown(createSelectExpr(LINK(selector), LINK(expr)));
  9204. value.set(selected);
  9205. }
  9206. switch (type->getTypeCode())
  9207. {
  9208. case type_row:
  9209. {
  9210. IHqlExpression * record = queryOriginalRecord(type);
  9211. buildXmlSerializeBeginNested(subctx, name, false);
  9212. if (assigns)
  9213. {
  9214. if (value->getOperator() == no_createrow)
  9215. {
  9216. HqlExprArray childAssigns;
  9217. filterExpandAssignments(subctx, NULL, childAssigns, value->queryChild(0));
  9218. OwnedHqlExpr childSelf = createSelector(no_self, value, NULL);
  9219. if (name)
  9220. buildXmlSerialize(subctx, record, childSelf, &childAssigns);
  9221. else
  9222. buildXmlSerialize(subctx, record, childSelf, &childAssigns, pass, expectedIndex);
  9223. }
  9224. else
  9225. {
  9226. CHqlBoundExpr bound;
  9227. Owned<IReferenceSelector> ref = buildNewRow(subctx, value);
  9228. if (name)
  9229. buildXmlSerialize(subctx, record, value, NULL);
  9230. else
  9231. buildXmlSerialize(subctx, record, value, NULL, pass, expectedIndex);
  9232. }
  9233. }
  9234. else
  9235. {
  9236. if (name)
  9237. buildXmlSerialize(subctx, record, selected, assigns);
  9238. else
  9239. buildXmlSerialize(subctx, record, selected, assigns, pass, expectedIndex);
  9240. }
  9241. buildXmlSerializeEndNested(subctx, name);
  9242. return;
  9243. }
  9244. break;
  9245. case type_set:
  9246. buildXmlSerializeSet(subctx, expr, value);
  9247. break;
  9248. case type_table:
  9249. case type_groupedtable:
  9250. buildXmlSerializeDataset(subctx, expr, value, assigns);
  9251. break;
  9252. default:
  9253. buildXmlSerializeScalar(subctx, value, name);
  9254. break;
  9255. }
  9256. }
  9257. break;
  9258. case no_ifblock:
  9259. {
  9260. OwnedHqlExpr cond = replaceSelector(expr->queryChild(0), querySelfReference(), selector);
  9261. BuildCtx condctx(subctx);
  9262. buildFilter(condctx, cond);
  9263. buildXmlSerialize(condctx, expr->queryChild(1), selector, assigns, pass, expectedIndex);
  9264. }
  9265. break;
  9266. case no_record:
  9267. {
  9268. ForEachChild(idx, expr)
  9269. buildXmlSerialize(subctx, expr->queryChild(idx), selector, assigns, pass, expectedIndex);
  9270. }
  9271. break;
  9272. case no_attr:
  9273. case no_attr_expr:
  9274. case no_attr_link:
  9275. break;
  9276. default:
  9277. UNIMPLEMENTED;
  9278. }
  9279. }
  9280. }
  9281. void HqlCppTranslator::buildXmlSerialize(BuildCtx & subctx, IHqlExpression * expr, IHqlExpression * selector, HqlExprArray * assigns)
  9282. {
  9283. unsigned expectedIndex = 0;
  9284. buildXmlSerialize(subctx, expr, selector, assigns, 0, expectedIndex);
  9285. expectedIndex = 0;
  9286. buildXmlSerialize(subctx, expr, selector, assigns, 1, expectedIndex);
  9287. }
  9288. void HqlCppTranslator::buildXmlSerialize(BuildCtx & ctx, IHqlExpression * dataset, const char * funcName, bool isMeta)
  9289. {
  9290. StringBuffer s;
  9291. BuildCtx funcctx(ctx);
  9292. funcctx.addQuotedCompound(s.append("virtual void ").append(funcName).append("(const byte * self, IXmlWriter & out)"));
  9293. if (!isMeta)
  9294. {
  9295. buildXmlSerializeUsingMeta(funcctx, dataset, "self");
  9296. }
  9297. else
  9298. {
  9299. BoundRow * selfCursor = bindTableCursor(funcctx, dataset, "self");
  9300. buildXmlSerialize(funcctx, dataset->queryRecord(), selfCursor->querySelector(), NULL);
  9301. }
  9302. }
  9303. void HqlCppTranslator::buildXmlSerializeUsingMeta(BuildCtx & ctx, IHqlExpression * dataset, const char * self)
  9304. {
  9305. MetaInstance meta(*this, dataset);
  9306. buildMetaInfo(meta);
  9307. StringBuffer s;
  9308. ctx.addQuoted(s.append(meta.queryInstanceObject()).append(".toXML(").append(self).append(", out);"));
  9309. }
  9310. //-------------------------------------------------------------------------------------------------------------------
  9311. //-------------------------------------------------------------------------------------------------------------------
  9312. ABoundActivity * HqlCppTranslator::doBuildActivityOutputWorkunit(BuildCtx & ctx, IHqlExpression * expr, bool isRoot)
  9313. {
  9314. IHqlExpression * dataset = expr->queryChild(0);
  9315. IHqlExpression * record = dataset->queryRecord();
  9316. IHqlExpression * seq = querySequence(expr);
  9317. IHqlExpression * name = queryResultName(expr);
  9318. int sequence = (int)getIntValue(seq, ResultSequenceInternal);
  9319. if (expr->hasProperty(diskAtom))
  9320. {
  9321. StringBuffer suffix;
  9322. suffix.append("_").append(sequence);
  9323. IHqlExpression * newName = createConstant("~result::");
  9324. newName = createValue(no_concat, LINK(unknownStringType), newName, createValue(no_wuid, LINK(unknownStringType)));
  9325. newName = createValue(no_concat, LINK(unknownStringType), newName, createConstant(suffix));
  9326. HqlExprArray args;
  9327. unwindChildren(args, expr);
  9328. args.add(*newName, 1);
  9329. args.append(*createAttribute(overwriteAtom));
  9330. args.append(*createAttribute(resultAtom));
  9331. OwnedHqlExpr outputToTemp = expr->clone(args);
  9332. return buildActivity(ctx, outputToTemp, isRoot);
  9333. }
  9334. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  9335. StringBuffer graphLabel;
  9336. bool useImplementationClass = options.minimizeActivityClasses && (sequence == ResultSequenceInternal);
  9337. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKworkunitwrite, expr, "WorkUnitWrite");
  9338. if (useImplementationClass)
  9339. instance->setImplementationClass(newWorkUnitWriteArgAtom);
  9340. graphLabel.append(getActivityText(instance->kind)).append("\n");
  9341. getStoredDescription(graphLabel, seq, name, true);
  9342. instance->graphLabel.set(graphLabel.str());
  9343. buildActivityFramework(instance, isRoot && !isInternalSeq(seq));
  9344. buildInstancePrefix(instance);
  9345. noteResultDefined(ctx, instance, seq, name, isRoot);
  9346. //virtual unsigned getFlags()
  9347. StringBuffer flags;
  9348. if (expr->hasProperty(extendAtom))
  9349. flags.append("|POFextend");
  9350. if (expr->hasProperty(groupedAtom))
  9351. flags.append("|POFgrouped");
  9352. if (!useImplementationClass)
  9353. {
  9354. doBuildSequenceFunc(instance->classctx, seq, true);
  9355. if (name)
  9356. {
  9357. BuildCtx namectx(instance->startctx);
  9358. namectx.addQuotedCompound("virtual const char * queryName()");
  9359. buildReturn(namectx, name, constUnknownVarStringType);
  9360. }
  9361. Owned<IWUResult> result = createDatasetResultSchema(seq, name, record, true, false);
  9362. if (result)
  9363. {
  9364. result->setResultRowLimit(-1);
  9365. if (sequence >= 0)
  9366. buildXmlSerialize(instance->startctx, dataset, "serializeXml", false);
  9367. }
  9368. if (flags.length())
  9369. doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);
  9370. }
  9371. else
  9372. {
  9373. if (flags.length() == 0)
  9374. flags.append("|0");
  9375. OwnedHqlExpr flagsExpr = createQuoted(flags.str()+1, LINK(unsignedType));
  9376. instance->addConstructorParameter(name);
  9377. instance->addConstructorParameter(flagsExpr);
  9378. }
  9379. buildInstanceSuffix(instance);
  9380. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  9381. associateRemoteResult(*instance, seq, name);
  9382. return instance->getBoundActivity();
  9383. }
  9384. void HqlCppTranslator::doBuildStmtOutput(BuildCtx & ctx, IHqlExpression * expr)
  9385. {
  9386. if (queryCurrentActivity(ctx))
  9387. {
  9388. ApplyStmtBuilder builder(*this);
  9389. builder.buildStmt(ctx, expr);
  9390. builder.flush(ctx);
  9391. return;
  9392. }
  9393. IHqlExpression * dataset = expr->queryChild(0);
  9394. if (expr->hasProperty(groupedAtom) && (dataset->getOperator() != no_null))
  9395. throwError1(HQLERR_NotSupportedInsideNoThor, "Grouped OUTPUT");
  9396. LinkedHqlExpr seq = querySequence(expr);
  9397. LinkedHqlExpr name = queryResultName(expr);
  9398. assertex(seq);
  9399. int sequence = (int)getIntValue(seq, (int)ResultSequenceInternal);
  9400. if (!seq)
  9401. seq.setown(getSizetConstant(sequence));
  9402. if (!name)
  9403. name.setown(createQuoted("NULL", LINK(constUnknownVarStringType)));
  9404. Owned<IWUResult> result = createDatasetResultSchema(seq, name, dataset->queryRecord(), true, false);
  9405. CHqlBoundExpr bound;
  9406. buildDataset(ctx, dataset, bound, FormatNatural);
  9407. OwnedHqlExpr count = getBoundCount(bound);
  9408. HqlExprArray args;
  9409. args.append(*LINK(name));
  9410. args.append(*LINK(seq));
  9411. args.append(*bound.getTranslatedExpr());
  9412. args.append(*createTranslated(count));
  9413. args.append(*LINK(queryBoolExpr(expr->hasProperty(extendAtom))));
  9414. buildFunctionCall(ctx, setResultDatasetAtom, args);
  9415. }
  9416. //---------------------------------------------------------------------------
  9417. ABoundActivity * HqlCppTranslator::doBuildActivityPipeThrough(BuildCtx & ctx, IHqlExpression * expr)
  9418. {
  9419. checkPipeAllowed();
  9420. IHqlExpression * dataset = expr->queryChild(0);
  9421. IHqlExpression * pipe = expr->queryChild(1);
  9422. IHqlExpression * output = expr->queryProperty(outputAtom);
  9423. IHqlExpression * csvToPipe = output ? output->queryProperty(csvAtom) : NULL;
  9424. IHqlExpression * xmlToPipe = output ? output->queryProperty(xmlAtom) : NULL;
  9425. IHqlExpression * csvFromPipe = expr->queryProperty(csvAtom);
  9426. IHqlExpression * xmlFromPipe = expr->queryProperty(xmlAtom);
  9427. //MORE: Could optimize dataset to not use LCR rows - if it is coming from a disk file
  9428. //Some other activities could similarly benefit (e.g., SORT), but they might need two
  9429. //metas and more intelligence in the activities...
  9430. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  9431. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKpipethrough, expr, "PipeThrough");
  9432. if (csvToPipe)
  9433. instance->addBaseClass("IHThorCsvWriteExtra", true);
  9434. else if (xmlToPipe)
  9435. instance->addBaseClass("IHThorXmlWriteExtra", true);
  9436. buildActivityFramework(instance);
  9437. buildInstancePrefix(instance);
  9438. if (expr->hasProperty(repeatAtom))
  9439. {
  9440. //virtual char * getPipeProgram() { return "grep"; }
  9441. instance->startctx.addQuoted("virtual char * getPipeProgram() { return NULL; }");
  9442. BuildCtx pipeCtx(instance->startctx);
  9443. pipeCtx.addQuotedCompound("virtual char * getNameFromRow(const void * _self)");
  9444. pipeCtx.addQuoted("const unsigned char * self = (const unsigned char *) _self;");
  9445. bindTableCursor(pipeCtx, dataset, "self");
  9446. buildReturn(pipeCtx, pipe, unknownVarStringType);
  9447. }
  9448. else
  9449. {
  9450. //virtual char * getPipeProgram() { return "grep"; }
  9451. BuildCtx pipeCtx(instance->startctx);
  9452. pipeCtx.addQuotedCompound("virtual char * getPipeProgram()");
  9453. buildReturn(pipeCtx, pipe, unknownVarStringType);
  9454. }
  9455. if (csvToPipe)
  9456. {
  9457. buildCsvWriteMembers(instance, dataset, csvToPipe);
  9458. instance->classctx.addQuoted("virtual IHThorCsvWriteExtra * queryCsvOutput() { return this; }");
  9459. }
  9460. if (xmlToPipe)
  9461. {
  9462. buildXmlWriteMembers(instance, dataset, xmlToPipe);
  9463. instance->classctx.addQuoted("virtual IHThorXmlWriteExtra * queryXmlOutput() { return this; }");
  9464. }
  9465. bool usesContents = false;
  9466. if (csvFromPipe)
  9467. {
  9468. if (isValidCsvRecord(expr->queryRecord()))
  9469. {
  9470. StringBuffer csvInstanceName;
  9471. buildCsvReadTransformer(expr, csvInstanceName, csvFromPipe);
  9472. StringBuffer s;
  9473. s.append("virtual ICsvToRowTransformer * queryCsvTransformer() { return &").append(csvInstanceName).append("; }");
  9474. instance->classctx.addQuoted(s);
  9475. }
  9476. else
  9477. {
  9478. throwUnexpected(); // should be caught earlier
  9479. }
  9480. }
  9481. else if (xmlFromPipe)
  9482. {
  9483. doBuildXmlReadMember(*instance, expr, "queryXmlTransformer", usesContents);
  9484. doBuildVarStringFunction(instance->classctx, "queryXmlIteratorPath", queryPropertyChild(xmlFromPipe, rowAtom, 0));
  9485. }
  9486. StringBuffer flags;
  9487. if (expr->hasProperty(repeatAtom))
  9488. flags.append("|TPFrecreateeachrow");
  9489. if (expr->hasProperty(groupAtom))
  9490. flags.append("|TPFgroupeachrow");
  9491. if (expr->hasProperty(optAtom))
  9492. flags.append("|TPFnofail");
  9493. if (csvToPipe)
  9494. flags.append("|TPFwritecsvtopipe");
  9495. if (xmlToPipe)
  9496. flags.append("|TPFwritexmltopipe");
  9497. if (csvFromPipe)
  9498. flags.append("|TPFreadcsvfrompipe");
  9499. if (xmlFromPipe)
  9500. flags.append("|TPFreadxmlfrompipe");
  9501. if (usesContents)
  9502. flags.append("|TPFreadusexmlcontents");
  9503. if (xmlToPipe && xmlToPipe->hasProperty(noRootAtom))
  9504. flags.append("|TPFwritenoroot");
  9505. if (xmlFromPipe && xmlFromPipe->hasProperty(noRootAtom))
  9506. flags.append("|TPFreadnoroot");
  9507. if (flags.length())
  9508. doBuildUnsignedFunction(instance->classctx, "getPipeFlags", flags.str()+1);
  9509. buildInstanceSuffix(instance);
  9510. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  9511. return instance->getBoundActivity();
  9512. }
  9513. //---------------------------------------------------------------------------
  9514. //-- no_join [JOIN] --
  9515. /* in parms: NOT linked */
  9516. void HqlCppTranslator::doCompareLeftRight(BuildCtx & ctx, const char * funcname, const DatasetReference & datasetLeft, const DatasetReference & datasetRight, HqlExprArray & left, HqlExprArray & right)
  9517. {
  9518. OwnedHqlExpr selSeq = createDummySelectorSequence();
  9519. OwnedHqlExpr leftList = createValueSafe(no_sortlist, makeSortListType(NULL), left);
  9520. OwnedHqlExpr leftSelect = datasetLeft.getSelector(no_left, selSeq);
  9521. OwnedHqlExpr leftResolved = datasetLeft.mapCompound(leftList, leftSelect);
  9522. OwnedHqlExpr rightList = createValueSafe(no_sortlist, makeSortListType(NULL), right);
  9523. OwnedHqlExpr rightSelect = datasetRight.getSelector(no_right, selSeq);
  9524. OwnedHqlExpr rightResolved = datasetRight.mapCompound(rightList, rightSelect);
  9525. OwnedHqlExpr order = createValue(no_order, LINK(signedType), LINK(leftResolved), LINK(rightResolved));
  9526. buildCompareMemberLR(ctx, funcname, order, datasetLeft.queryDataset(), datasetRight.queryDataset(), selSeq);
  9527. }
  9528. void HqlCppTranslator::buildSlidingMatchFunction(BuildCtx & ctx, HqlExprArray & leftEq, HqlExprArray & rightEq, HqlExprArray & slidingMatches, const char * funcname, unsigned childIndex, const DatasetReference & datasetL, const DatasetReference & datasetR)
  9529. {
  9530. HqlExprArray left, right;
  9531. unsigned numSimple = leftEq.ordinality() - slidingMatches.ordinality();
  9532. for (unsigned j=0; j<numSimple; j++)
  9533. {
  9534. left.append(OLINK(leftEq.item(j)));
  9535. right.append(OLINK(rightEq.item(j)));
  9536. }
  9537. ForEachItemIn(i, slidingMatches)
  9538. {
  9539. IHqlExpression & cur = slidingMatches.item(i);
  9540. left.append(*LINK(cur.queryChild(0)));
  9541. right.append(*LINK(cur.queryChild(childIndex)));
  9542. }
  9543. doCompareLeftRight(ctx, funcname, datasetL, datasetR, left, right);
  9544. }
  9545. void HqlCppTranslator::generateSortCompare(BuildCtx & nestedctx, BuildCtx & ctx, node_operator side, const DatasetReference & dataset, HqlExprArray & sorts, bool canRemoveSort, IHqlExpression * noSortAttr, bool canReuseLeft, bool isLightweight)
  9546. {
  9547. StringBuffer s, compareName;
  9548. const char * sideText = (side == no_left) ? "Left" : "Right";
  9549. compareName.append("compare").append(sideText);
  9550. assertex(dataset.querySide() == no_activetable);
  9551. bool noNeedToSort = canRemoveSort && isAlreadySorted(dataset.queryDataset(), sorts, canRemoveSort, true);
  9552. if (noSortAttr)
  9553. {
  9554. IHqlExpression * child = noSortAttr->queryChild(0);
  9555. if (!child)
  9556. noNeedToSort = true;
  9557. else
  9558. {
  9559. if (side == no_left)
  9560. noNeedToSort = child->queryName() == leftAtom;
  9561. else if (side == no_left)
  9562. noNeedToSort = child->queryName() == rightAtom;
  9563. }
  9564. }
  9565. if (noNeedToSort || isLightweight)
  9566. {
  9567. if (!noNeedToSort)
  9568. {
  9569. DBGLOG("Lightweight true, but code generator didn't think sort was required");
  9570. ctx.addQuoted("//Forced by lightweight");
  9571. }
  9572. s.clear().append("virtual bool is").append(sideText).append("AlreadySorted() { return true; }");
  9573. ctx.addQuoted(s);
  9574. }
  9575. if (canReuseLeft)
  9576. {
  9577. s.clear().append("virtual ICompare * queryCompare").append(sideText).append("() { return &compareLeft; }");
  9578. ctx.addQuoted(s);
  9579. }
  9580. else
  9581. {
  9582. s.clear().append("virtual ICompare * queryCompare").append(sideText).append("() { return &").append(compareName).append("; }");
  9583. ctx.addQuoted(s);
  9584. BuildCtx classctx(nestedctx);
  9585. beginNestedClass(classctx, compareName.str(), "ICompare");
  9586. BuildCtx funcctx(classctx);
  9587. funcctx.addQuotedCompound("virtual int docompare(const void * _left, const void * _right) const");
  9588. funcctx.addQuoted("const unsigned char * left = (const unsigned char *) _left;");
  9589. funcctx.addQuoted("const unsigned char * right = (const unsigned char *) _right;");
  9590. funcctx.associateExpr(constantMemberMarkerExpr, constantMemberMarkerExpr);
  9591. OwnedHqlExpr groupOrder = createValueSafe(no_sortlist, makeSortListType(NULL), sorts);
  9592. OwnedHqlExpr diff = doCompare(funcctx, groupOrder, dataset);
  9593. funcctx.setNextDestructor();
  9594. funcctx.addReturn(diff);
  9595. endNestedClass();
  9596. }
  9597. }
  9598. void HqlCppTranslator::generateSerializeAssigns(BuildCtx & ctx, IHqlExpression * record, IHqlExpression * selector, IHqlExpression * selfSelect, IHqlExpression * leftSelect, const DatasetReference & srcDataset, const DatasetReference & tgtDataset, HqlExprArray & srcSelects, HqlExprArray & tgtSelects, bool needToClear)
  9599. {
  9600. ForEachChild(i, record)
  9601. {
  9602. IHqlExpression * cur = record->queryChild(i);
  9603. switch (cur->getOperator())
  9604. {
  9605. case no_field:
  9606. {
  9607. OwnedHqlExpr selected = createSelectExpr(LINK(selector), LINK(cur));
  9608. unsigned matchIndex = tgtSelects.find(*selected);
  9609. if (matchIndex != NotFound)
  9610. {
  9611. Owned<IHqlExpression> self = tgtDataset.mapScalar(&tgtSelects.item(matchIndex), selfSelect);
  9612. Owned<IHqlExpression> left = srcDataset.mapScalar(&srcSelects.item(matchIndex), leftSelect);
  9613. buildAssign(ctx, self, left);
  9614. //Note, we could stop here if needToClear and all fields have been assigned, and all the following fields are fixed width.
  9615. // but not really sure it is worth it.
  9616. }
  9617. else if (cur->isDatarow())
  9618. {
  9619. generateSerializeAssigns(ctx, cur->queryRecord(), selected, selfSelect, leftSelect, srcDataset, tgtDataset, srcSelects, tgtSelects, needToClear);
  9620. }
  9621. else if (needToClear)
  9622. {
  9623. //MORE: Might want to recurse if a record
  9624. Owned<IHqlExpression> self = tgtDataset.mapScalar(selected, selfSelect);
  9625. buildClear(ctx, self);
  9626. }
  9627. break;
  9628. }
  9629. case no_record:
  9630. generateSerializeAssigns(ctx, cur, selector, selfSelect, leftSelect, srcDataset, tgtDataset, srcSelects, tgtSelects, needToClear);
  9631. break;
  9632. case no_ifblock:
  9633. //Filter on target...
  9634. UNIMPLEMENTED;
  9635. generateSerializeAssigns(ctx, cur->queryChild(1), selector, selfSelect, leftSelect, srcDataset, tgtDataset, srcSelects, tgtSelects, needToClear);
  9636. break;
  9637. }
  9638. }
  9639. }
  9640. void HqlCppTranslator::generateSerializeFunction(BuildCtx & ctx, const char * funcName, bool serialize, const DatasetReference & srcDataset, const DatasetReference & tgtDataset, HqlExprArray & srcSelects, HqlExprArray & tgtSelects)
  9641. {
  9642. StringBuffer s;
  9643. BuildCtx r2kctx(ctx);
  9644. s.append("virtual unsigned ").append(funcName).append("(ARowBuilder & crSelf, const void * _src, unsigned & thisRecordSize)");
  9645. r2kctx.addQuotedCompound(s);
  9646. ensureRowAllocated(r2kctx, "crSelf");
  9647. r2kctx.addQuoted("const unsigned char * src = (const unsigned char *) _src;");
  9648. OwnedHqlExpr selSeq = createDummySelectorSequence();
  9649. BoundRow * tgtCursor = bindSelf(ctx, tgtDataset.queryDataset(), "crSelf");
  9650. BoundRow * srcCursor = bindTableCursor(ctx, srcDataset.queryDataset(), "src", no_left, selSeq);
  9651. IHqlExpression * leftSelect = srcCursor->querySelector();
  9652. IHqlExpression * selfSelect = tgtCursor->querySelector();
  9653. IHqlExpression * record = tgtDataset.queryDataset()->queryRecord();
  9654. generateSerializeAssigns(r2kctx, record, tgtDataset.querySelector(), selfSelect, leftSelect, srcDataset, tgtDataset, srcSelects, tgtSelects, !isFixedRecordSize(record));
  9655. BoundRow * recordCursor = serialize ? srcCursor : tgtCursor;
  9656. OwnedHqlExpr recordSize = getRecordSize(recordCursor->querySelector());
  9657. OwnedHqlExpr recordSizeVar = createVariable("thisRecordSize", LINK(unsignedType));
  9658. buildAssignToTemp(r2kctx, recordSizeVar, recordSize);
  9659. buildReturnRecordSize(r2kctx, serialize ? tgtCursor : srcCursor);
  9660. }
  9661. void HqlCppTranslator::generateSerializeKey(BuildCtx & nestedctx, node_operator side, const DatasetReference & dataset, HqlExprArray & sorts, bool isGlobal, bool generateCompares, bool canReuseLeft)
  9662. {
  9663. //check if there are any ifblocks, and if so don't allow it. Even more accurate would be no join fields used in ifblocks
  9664. IHqlExpression * record = dataset.queryDataset()->queryRecord();
  9665. bool canSerialize = targetThor() && isGlobal && !recordContainsIfBlock(record);
  9666. const char * sideText = (side == no_none) ? "" : (side == no_left) ? "Left" : "Right";
  9667. StringBuffer s, s2;
  9668. HqlExprArray keyFields;
  9669. HqlExprArray keySelects;
  9670. HqlExprArray datasetSelects;
  9671. HqlExprArray keyCompares;
  9672. if (canSerialize)
  9673. {
  9674. ForEachItemIn(idx, sorts)
  9675. {
  9676. //MORE: Nested - this won't serialize the key if sorting by a field in a nested record
  9677. // If this is a problem we will need to create new fields for each value.
  9678. IHqlExpression & cur = sorts.item(idx);
  9679. IHqlExpression * value = &cur;
  9680. if (value->getOperator() == no_negate)
  9681. value=value->queryChild(0);
  9682. if ((value->getOperator() == no_select) && (value->queryChild(0)->queryNormalizedSelector() == dataset.querySelector()))
  9683. {
  9684. if (value->queryType()->getTypeCode() == type_alien)
  9685. {
  9686. //MORE: Really should check if a self contained alien data type.
  9687. canSerialize = false;
  9688. break;
  9689. }
  9690. OwnedHqlExpr serializedField = getSerializedForm(value->queryChild(1));
  9691. OwnedHqlExpr mappedSelect = dataset.mapScalar(value,queryActiveTableSelector());
  9692. keyFields.append(*LINK(serializedField));
  9693. keySelects.append(*createSelectExpr(LINK(mappedSelect->queryChild(0)), LINK(serializedField)));
  9694. datasetSelects.append(*LINK(value));
  9695. keyCompares.append(*dataset.mapScalar(&cur,queryActiveTableSelector()));
  9696. }
  9697. else if (!value->isConstant())
  9698. {
  9699. canSerialize = false;
  9700. break;
  9701. }
  9702. }
  9703. }
  9704. //The following test will need to change if we serialize when nested fields are used (see above)
  9705. if (sorts.ordinality() >= getFlatFieldCount(record))
  9706. canSerialize = false;
  9707. if (canSerialize)
  9708. {
  9709. if (canReuseLeft)
  9710. {
  9711. assertex(!generateCompares);
  9712. s.clear().append("virtual ISortKeySerializer * querySerialize").append(sideText).append("() { return &serializerLeft; }");
  9713. nestedctx.addQuoted(s);
  9714. }
  9715. else
  9716. {
  9717. StringBuffer memberName;
  9718. memberName.append("serializer").append(sideText);
  9719. BuildCtx classctx(nestedctx);
  9720. beginNestedClass(classctx, memberName, "ISortKeySerializer");
  9721. IHqlExpression * keyRecord = createRecordInheritMaxLength(keyFields, record);
  9722. Owned<IHqlExpression> keyDataset = createDataset(no_anon, keyRecord);
  9723. DatasetReference keyActiveRef(keyDataset, no_activetable, NULL);
  9724. generateSerializeFunction(classctx, "recordToKey", true, dataset, keyActiveRef, datasetSelects, keySelects);
  9725. generateSerializeFunction(classctx, "keyToRecord", false, keyActiveRef, dataset, keySelects, datasetSelects);
  9726. buildMetaMember(classctx, keyDataset, "queryRecordSize");
  9727. endNestedClass();
  9728. s.clear().append("virtual ISortKeySerializer * querySerialize").append(sideText).append("() { return &serializer").append(sideText).append("; }");
  9729. nestedctx.addQuoted(s);
  9730. if (generateCompares)
  9731. {
  9732. OwnedHqlExpr keyOrder = createValueSafe(no_sortlist, makeSortListType(NULL), keyCompares);
  9733. buildCompareMember(nestedctx, "CompareKey", keyOrder, keyActiveRef);
  9734. doCompareLeftRight(nestedctx, "CompareRowKey", dataset, keyActiveRef, sorts, keyCompares);
  9735. }
  9736. }
  9737. }
  9738. }
  9739. IHqlExpression * HqlCppTranslator::createFailMessage(const char * prefix, IHqlExpression * limit, IHqlExpression * filename, unique_id_t id)
  9740. {
  9741. StringBuffer s;
  9742. HqlExprArray values;
  9743. values.append(*createConstant(s.clear().append(prefix)));
  9744. if (limit)
  9745. {
  9746. values.append(*createConstant("("));
  9747. values.append(*ensureExprType(limit, unknownStringType));
  9748. values.append(*createConstant(")"));
  9749. }
  9750. if (filename)
  9751. {
  9752. values.append(*createConstant(" file '"));
  9753. values.append(*ensureExprType(filename, unknownStringType));
  9754. values.append(*createConstant("'"));
  9755. }
  9756. if (id)
  9757. values.append(*createConstant(s.clear().append(" [id=").append(id).append("]")));
  9758. OwnedHqlExpr errorText = createBalanced(no_concat, unknownStringType, values);
  9759. return foldHqlExpression(errorText);
  9760. }
  9761. IHqlExpression * HqlCppTranslator::createFailAction(const char * prefix, IHqlExpression * limit, IHqlExpression * filename, unique_id_t id)
  9762. {
  9763. IHqlExpression * msg = createFailMessage(prefix, limit, filename, id);
  9764. return createValue(no_fail, makeVoidType(), msg, getDefaultAttr());
  9765. }
  9766. void HqlCppTranslator::doBuildJoinRowLimitHelper(ActivityInstance & instance, IHqlExpression * rowlimit, IHqlExpression * filename, bool generateImplicitLimit)
  9767. {
  9768. if (rowlimit)
  9769. {
  9770. doBuildUnsignedFunction(instance.startctx, "getMatchAbortLimit", rowlimit->queryChild(0));
  9771. if (!rowlimit->hasProperty(skipAtom))
  9772. {
  9773. LinkedHqlExpr fail = queryChildOperator(no_fail, rowlimit);
  9774. if (!fail)
  9775. fail.setown(createFailAction("JOIN limit exceeded", rowlimit->queryChild(0), filename, instance.activityId));
  9776. BuildCtx ctx(instance.startctx);
  9777. ctx.addQuotedCompound("virtual void onMatchAbortLimitExceeded()");
  9778. buildStmt(ctx, fail);
  9779. }
  9780. }
  9781. else if (generateImplicitLimit)
  9782. {
  9783. OwnedHqlExpr implicitLimit = getSizetConstant(options.defaultImplicitKeyedJoinLimit);
  9784. doBuildUnsignedFunction(instance.startctx, "getMatchAbortLimit", implicitLimit);
  9785. if (options.warnOnImplicitJoinLimit)
  9786. {
  9787. StringBuffer fname;
  9788. if (filename)
  9789. getExprECL(filename, fname.append(" "));
  9790. WARNING2(HQLWRN_ImplicitJoinLimit, options.defaultImplicitKeyedJoinLimit, fname.str());
  9791. }
  9792. }
  9793. }
  9794. ABoundActivity * HqlCppTranslator::doBuildActivityJoinOrDenormalize(BuildCtx & ctx, IHqlExpression * expr)
  9795. {
  9796. node_operator op = expr->getOperator();
  9797. assertex(op==no_join || op==no_selfjoin || op==no_denormalize || op==no_denormalizegroup);
  9798. LinkedHqlExpr dataset1 = expr->queryChild(0);
  9799. LinkedHqlExpr dataset2 = queryJoinRhs(expr);
  9800. IHqlExpression * condition = expr->queryChild(2);
  9801. IHqlExpression * transform = expr->queryChild(3);
  9802. IHqlExpression * noSortAttr = expr->queryProperty(noSortAtom);
  9803. IHqlExpression * rowlimit = expr->queryProperty(rowLimitAtom);
  9804. IHqlExpression * selSeq = querySelSeq(expr);
  9805. bool isLeftOuter = false;
  9806. bool isRightOuter = false;
  9807. bool excludeMatches = false;
  9808. bool isAllJoin = false;
  9809. bool isLightweight = expr->hasProperty(_lightweight_Atom);
  9810. bool isManyLookup = expr->hasProperty(manyAtom);
  9811. if (expr->hasProperty(leftouterAtom))
  9812. isLeftOuter = true;
  9813. if (expr->hasProperty(rightouterAtom))
  9814. isRightOuter = true;
  9815. if (expr->hasProperty(fullouterAtom))
  9816. {
  9817. isLeftOuter = true;
  9818. isRightOuter = true;
  9819. }
  9820. if (expr->hasProperty(leftonlyAtom))
  9821. {
  9822. isLeftOuter = true;
  9823. excludeMatches = true;
  9824. }
  9825. if (expr->hasProperty(rightonlyAtom))
  9826. {
  9827. isRightOuter = true;
  9828. excludeMatches = true;
  9829. }
  9830. if (expr->hasProperty(fullonlyAtom))
  9831. {
  9832. isLeftOuter = true;
  9833. isRightOuter = true;
  9834. excludeMatches = true;
  9835. }
  9836. if (expr->hasProperty(allAtom))
  9837. isAllJoin = true;
  9838. bool isLookupJoin = expr->hasProperty(lookupAtom);
  9839. bool isHashJoin = targetThor() && expr->hasProperty(hashAtom);
  9840. bool isLocalJoin = !isHashJoin && expr->hasProperty(localAtom);
  9841. bool joinToSelf = (op == no_selfjoin);
  9842. bool allowAllToLookupConvert = !options.noAllToLookupConversion;
  9843. IHqlExpression * atmost = expr->queryProperty(atmostAtom);
  9844. //Delay removing ungroups until this point because they can be useful for reducing the size of spill files.
  9845. if (isUngroup(dataset1) && !isLookupJoin)
  9846. dataset1.set(dataset1->queryChild(0));
  9847. if (isUngroup(dataset2))
  9848. dataset2.set(dataset2->queryChild(0));
  9849. if (expr->hasProperty(groupedAtom) && targetThor())
  9850. WARNING(HQLWRN_GroupedJoinIsLookupJoin);
  9851. if ((op == no_denormalize || op == no_denormalizegroup) && targetThor() && options.checkThorRestrictions)
  9852. {
  9853. if (isHashJoin)
  9854. throwError1(HQLERR_ThorDenormNoFeatureX, "HASH");
  9855. if (expr->hasProperty(firstAtom))
  9856. throwError1(HQLERR_ThorDenormNoFeatureX, "FIRST");
  9857. if (expr->hasProperty(firstLeftAtom))
  9858. throwError1(HQLERR_ThorDenormNoFeatureX, "FIRST LEFT");
  9859. if (expr->hasProperty(firstRightAtom))
  9860. throwError1(HQLERR_ThorDenormNoFeatureX, "FIRST RIGHT");
  9861. if (expr->hasProperty(partitionRightAtom))
  9862. throwError1(HQLERR_ThorDenormNoFeatureX, "PARTITION RIGHT");
  9863. }
  9864. OwnedHqlExpr atmostCond, atmostLimit;
  9865. extractAtmostArgs(atmost, atmostCond, atmostLimit);
  9866. HqlExprArray leftSorts, rightSorts, slidingMatches;
  9867. bool slidingAllowed = options.slidingJoins && canBeSlidingJoin(expr);
  9868. OwnedHqlExpr match;
  9869. OwnedHqlExpr fuzzy, hard;
  9870. bool isLimitedSubstringJoin;
  9871. splitFuzzyCondition(condition, atmostCond, fuzzy, hard);
  9872. match.setown(findJoinSortOrders(hard, dataset1, dataset2, selSeq, leftSorts, rightSorts, isLimitedSubstringJoin, slidingAllowed ? &slidingMatches : NULL));
  9873. if (atmost && match)
  9874. {
  9875. if (isAllJoin)
  9876. allowAllToLookupConvert = false;
  9877. else
  9878. {
  9879. StringBuffer s;
  9880. throwError1(HQLERR_BadJoinConditionAtMost,getExprECL(match, s.append(" (")).append(")").str());
  9881. }
  9882. }
  9883. extendConditionOwn(match, no_and, fuzzy.getClear());
  9884. if (isAllJoin)
  9885. {
  9886. if (leftSorts.ordinality() && allowAllToLookupConvert)
  9887. {
  9888. //Convert an all join to a many lookup if it can be done that way - more efficient, and same resourcing/semantics ...
  9889. isManyLookup = true;
  9890. isAllJoin = false;
  9891. isLookupJoin = true;
  9892. }
  9893. }
  9894. else if (leftSorts.ordinality() == 0)
  9895. {
  9896. if (expr->hasProperty(_conditionFolded_Atom))
  9897. {
  9898. isAllJoin = true;
  9899. WARNING(HQLWRN_JoinConditionFoldedNowAll);
  9900. }
  9901. else
  9902. {
  9903. StringBuffer name;
  9904. if (expr->queryName())
  9905. name.append(" ").append(expr->queryName());
  9906. throwError1(HQLERR_JoinXTooComplex, name.str());
  9907. }
  9908. }
  9909. Owned<ABoundActivity> boundDataset1 = buildCachedActivity(ctx, dataset1);
  9910. Owned<ABoundActivity> boundDataset2;
  9911. if (!joinToSelf)
  9912. boundDataset2.setown(buildCachedActivity(ctx, dataset2));
  9913. const char * argName;
  9914. ThorActivityKind kind;
  9915. if (op == no_selfjoin)
  9916. {
  9917. if (isLightweight)
  9918. kind = TAKselfjoinlight;
  9919. else
  9920. kind = TAKselfjoin;
  9921. argName = "Join";
  9922. }
  9923. else if (op == no_join)
  9924. {
  9925. if (isAllJoin)
  9926. {
  9927. kind = TAKalljoin;
  9928. argName = "AllJoin";
  9929. }
  9930. else if (isLookupJoin)
  9931. {
  9932. kind = TAKlookupjoin;
  9933. argName = "HashJoin";
  9934. }
  9935. else if (isHashJoin)
  9936. {
  9937. kind = TAKhashjoin;
  9938. argName = "HashJoin";
  9939. }
  9940. else
  9941. {
  9942. kind = TAKjoin;
  9943. argName = "Join";
  9944. }
  9945. }
  9946. else if (op == no_denormalize)
  9947. {
  9948. if (isAllJoin)
  9949. {
  9950. kind = TAKalldenormalize;
  9951. argName = "AllDenormalize";
  9952. }
  9953. else if (isLookupJoin)
  9954. {
  9955. kind = TAKlookupdenormalize;
  9956. argName = "HashDenormalize";
  9957. }
  9958. else if (isHashJoin)
  9959. {
  9960. kind = TAKhashdenormalize;
  9961. argName = "HashDenormalize";
  9962. }
  9963. else
  9964. {
  9965. kind = TAKdenormalize;
  9966. argName = "Denormalize";
  9967. }
  9968. }
  9969. else
  9970. {
  9971. if (isAllJoin)
  9972. {
  9973. kind = TAKalldenormalizegroup;
  9974. argName = "AllDenormalizeGroup";
  9975. }
  9976. else if (isLookupJoin)
  9977. {
  9978. kind = TAKlookupdenormalizegroup;
  9979. argName = "HashDenormalizeGroup";
  9980. }
  9981. else if (isHashJoin)
  9982. {
  9983. kind = TAKhashdenormalizegroup;
  9984. argName = "HashDenormalizeGroup";
  9985. }
  9986. else
  9987. {
  9988. kind = TAKdenormalizegroup;
  9989. argName = "DenormalizeGroup";
  9990. }
  9991. }
  9992. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, kind, expr, argName);
  9993. if (isLightweight)
  9994. {
  9995. if ((kind == TAKselfjoinlight) || (kind == TAKselfjoin))
  9996. instance->graphLabel.set("Lightweight Self Join");
  9997. else
  9998. instance->graphLabel.set("Lightweight Join");
  9999. }
  10000. instance->setLocal(isLocalJoin);
  10001. buildActivityFramework(instance);
  10002. buildInstancePrefix(instance);
  10003. StringBuffer s,temp;
  10004. LinkedHqlExpr keepLimit = queryPropertyChild(expr, keepAtom, 0);
  10005. DatasetReference lhsDsRef(dataset1, no_activetable, NULL);
  10006. DatasetReference rhsDsRef(dataset2, no_activetable, NULL);
  10007. bool couldBeKeepOne = keepLimit && (!keepLimit->queryValue() || (keepLimit->queryValue()->getIntValue() <= 1));
  10008. if (dataset1->queryRecord() == dataset2->queryRecord())
  10009. {
  10010. //more could use the compareLeftRight function instead of generating the same code
  10011. //several time....
  10012. }
  10013. bool canReuseLeft = recordTypesMatch(dataset1, dataset2) && arraysMatch(leftSorts, rightSorts);
  10014. if (!isAllJoin)
  10015. {
  10016. bool canRemoveSort = isLocalJoin || !targetThor();
  10017. //Lookup join doesn't need the left sort (unless it is reused elsewhere), or the right sort unless it is deduping.
  10018. if (canReuseLeft || !isLookupJoin)
  10019. generateSortCompare(instance->nestedctx, instance->classctx, no_left, lhsDsRef, leftSorts, canRemoveSort, noSortAttr, false, isLightweight);
  10020. if (!(isLookupJoin && isManyLookup && !couldBeKeepOne && !targetThor())) // many lookup doesn't need to dedup the rhs
  10021. generateSortCompare(instance->nestedctx, instance->classctx, no_right, rhsDsRef, rightSorts, canRemoveSort, noSortAttr, canReuseLeft, isLightweight);
  10022. bool isGlobal = !isLocalJoin && !instance->isChildActivity();
  10023. generateSerializeKey(instance->nestedctx, no_left, lhsDsRef, leftSorts, isGlobal, false, false);
  10024. generateSerializeKey(instance->nestedctx, no_right, rhsDsRef, rightSorts, isGlobal, false, canReuseLeft);
  10025. }
  10026. StringBuffer flags;
  10027. if (excludeMatches) flags.append("|JFexclude");
  10028. if (isLeftOuter) flags.append("|JFleftouter");
  10029. if (isRightOuter) flags.append("|JFrightouter");
  10030. if (expr->hasProperty(firstAtom)) flags.append("|JFfirst");
  10031. if (expr->hasProperty(firstLeftAtom)) flags.append("|JFfirstleft");
  10032. if (expr->hasProperty(firstRightAtom)) flags.append("|JFfirstright");
  10033. if (expr->hasProperty(partitionRightAtom)) flags.append("|JFpartitionright");
  10034. if (expr->hasProperty(parallelAtom)) flags.append("|JFparallel");
  10035. if (expr->hasProperty(sequentialAtom)) flags.append("|JFsequential");
  10036. if (transformContainsSkip(transform))
  10037. flags.append("|JFtransformMaySkip");
  10038. if (rowlimit && rowlimit->hasProperty(skipAtom))
  10039. flags.append("|JFmatchAbortLimitSkips");
  10040. if (rowlimit && rowlimit->hasProperty(countAtom))
  10041. flags.append("|JFcountmatchabortlimit");
  10042. if (slidingMatches.ordinality()) flags.append("|JFslidingmatch");
  10043. if (match) flags.append("|JFmatchrequired");
  10044. if (isLookupJoin && isManyLookup) flags.append("|JFmanylookup");
  10045. if (expr->hasProperty(onFailAtom))
  10046. flags.append("|JFonfail");
  10047. if (transformReturnsSide(expr, no_left, 0))
  10048. flags.append("|JFtransformmatchesleft");
  10049. if (isLimitedSubstringJoin)
  10050. flags.append("|JFlimitedprefixjoin");
  10051. if (flags.length())
  10052. doBuildUnsignedFunction(instance->classctx, "getJoinFlags", flags.str()+1);
  10053. if (!isAllJoin)
  10054. {
  10055. buildSkewThresholdMembers(instance->classctx, expr);
  10056. if (!isZero(atmostLimit))
  10057. doBuildUnsignedFunction(instance->startctx, "getJoinLimit", atmostLimit);
  10058. }
  10059. if (keepLimit)
  10060. doBuildUnsignedFunction(instance->startctx, "getKeepLimit", keepLimit);
  10061. // The transform function is pretty standard - no need for copies here
  10062. BuildCtx transformctx(instance->startctx);
  10063. switch (op)
  10064. {
  10065. case no_denormalize:
  10066. {
  10067. transformctx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf, const void * _left, const void * _right, unsigned counter)");
  10068. ensureRowAllocated(transformctx, "crSelf");
  10069. IHqlExpression * counter = queryPropertyChild(expr, _countProject_Atom, 0);
  10070. associateCounter(transformctx, counter, "counter");
  10071. buildTransformBody(transformctx, transform, dataset1, dataset2, instance->dataset, selSeq);
  10072. break;
  10073. }
  10074. case no_denormalizegroup:
  10075. {
  10076. transformctx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf, const void * _left, const void * _right, unsigned numRows, const void * * _rows)");
  10077. ensureRowAllocated(transformctx, "crSelf");
  10078. transformctx.addQuoted("unsigned char * * rows = (unsigned char * *) _rows;");
  10079. BoundRow * selfCursor = buildTransformCursors(transformctx, transform, dataset1, dataset2, instance->dataset, selSeq);
  10080. bindRows(transformctx, no_right, selSeq, expr->queryProperty(_rowsid_Atom), dataset2, "numRows", "rows", options.mainRowsAreLinkCounted);
  10081. doBuildTransformBody(transformctx, transform, selfCursor);
  10082. break;
  10083. }
  10084. case no_join:
  10085. case no_selfjoin:
  10086. {
  10087. transformctx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf, const void * _left, const void * _right)");
  10088. ensureRowAllocated(transformctx, "crSelf");
  10089. buildTransformBody(transformctx, transform, dataset1, dataset2, instance->dataset, selSeq);
  10090. break;
  10091. }
  10092. }
  10093. IHqlExpression * onFail = expr->queryProperty(onFailAtom);
  10094. if (onFail)
  10095. {
  10096. BuildCtx funcctx(instance->startctx);
  10097. funcctx.addQuotedCompound("virtual size32_t onFailTransform(ARowBuilder & crSelf, const void * _left, const void * _right, IException * except)");
  10098. ensureRowAllocated(funcctx, "crSelf");
  10099. associateLocalFailure(funcctx, "except");
  10100. buildTransformBody(funcctx, onFail->queryChild(0), dataset1, dataset2, instance->dataset, selSeq);
  10101. }
  10102. // The collate function is used to work out which side to read from or if we have a potentially matching record
  10103. if (!isAllJoin)
  10104. {
  10105. //if left and right match, then leftright compare function is also the same
  10106. if (isLimitedSubstringJoin)
  10107. {
  10108. HqlExprArray compareLeftSorts, compareRightSorts;
  10109. unsigned max = leftSorts.ordinality()-1;
  10110. for (unsigned i=0; i < max; i++)
  10111. {
  10112. compareLeftSorts.append(OLINK(leftSorts.item(i)));
  10113. compareRightSorts.append(OLINK(rightSorts.item(i)));
  10114. }
  10115. doCompareLeftRight(instance->nestedctx, "CompareLeftRight", lhsDsRef, rhsDsRef, compareLeftSorts, compareRightSorts);
  10116. }
  10117. else if (canReuseLeft)
  10118. instance->nestedctx.addQuoted("virtual ICompare * queryCompareLeftRight() { return &compareLeft; }");
  10119. else
  10120. doCompareLeftRight(instance->nestedctx, "CompareLeftRight", lhsDsRef, rhsDsRef, leftSorts, rightSorts);
  10121. }
  10122. doBuildJoinRowLimitHelper(*instance, rowlimit, NULL, false);
  10123. //--function to clear left, used for right outer join and vice-versa
  10124. bool createDefaultRight = onFail || isLeftOuter;
  10125. if (isRightOuter)
  10126. buildClearRecordMember(instance->createctx, "Left", dataset1);
  10127. if (createDefaultRight)
  10128. buildClearRecordMember(instance->createctx, "Right", dataset2);
  10129. buildJoinMatchFunction(instance->startctx, "match", dataset1, dataset2, match, selSeq);
  10130. if (slidingMatches.ordinality())
  10131. {
  10132. buildSlidingMatchFunction(instance->nestedctx, leftSorts, rightSorts, slidingMatches, "CompareLeftRightLower", 1, lhsDsRef, rhsDsRef);
  10133. buildSlidingMatchFunction(instance->nestedctx, leftSorts, rightSorts, slidingMatches, "CompareLeftRightUpper", 2, lhsDsRef, rhsDsRef);
  10134. }
  10135. if (isHashJoin||isLookupJoin)
  10136. {
  10137. OwnedHqlExpr leftList = createValueSafe(no_sortlist, makeSortListType(NULL), leftSorts);
  10138. buildHashOfExprsClass(instance->nestedctx, "HashLeft", leftList, lhsDsRef, false);
  10139. if (!canReuseLeft)
  10140. {
  10141. OwnedHqlExpr rightList = createValueSafe(no_sortlist, makeSortListType(NULL), rightSorts);
  10142. buildHashOfExprsClass(instance->nestedctx, "HashRight", rightList, rhsDsRef, false);
  10143. }
  10144. else
  10145. instance->nestedctx.addQuoted("virtual IHash * queryHashRight() { return &HashLeft; }");
  10146. }
  10147. if (isLimitedSubstringJoin)
  10148. {
  10149. OwnedHqlExpr leftSelect = createSelector(no_left, dataset1, selSeq);
  10150. OwnedHqlExpr rightSelect = createSelector(no_right, dataset2, selSeq);
  10151. HqlExprArray args;
  10152. args.append(*lhsDsRef.mapCompound(&leftSorts.tos(), leftSelect));
  10153. args.append(*rhsDsRef.mapCompound(&rightSorts.tos(), rightSelect));
  10154. _ATOM func = prefixDiffStrAtom;
  10155. OwnedHqlExpr compare = bindFunctionCall(func, args);
  10156. buildCompareMemberLR(instance->nestedctx, "PrefixCompare", compare, dataset1, dataset2, selSeq);
  10157. }
  10158. buildInstanceSuffix(instance);
  10159. buildConnectInputOutput(ctx, instance, boundDataset1, 0, 0, boundDataset2 ? "LEFT" : NULL);
  10160. if (boundDataset2)
  10161. buildConnectInputOutput(ctx, instance, boundDataset2, 0, 1, "RIGHT");
  10162. return instance->getBoundActivity();
  10163. }
  10164. ABoundActivity * HqlCppTranslator::doBuildActivityJoin(BuildCtx & ctx, IHqlExpression * expr)
  10165. {
  10166. if (isKeyedJoin(expr))
  10167. return doBuildActivityKeyedJoinOrDenormalize(ctx, expr);
  10168. else
  10169. return doBuildActivityJoinOrDenormalize(ctx, expr);
  10170. }
  10171. //---------------------------------------------------------------------------
  10172. void HqlCppTranslator::doUserTransform(BuildCtx & ctx, IHqlExpression * transform, BoundRow * self)
  10173. {
  10174. node_operator transformOp = transform->getOperator();
  10175. assertex(transformOp == no_call || transformOp == no_externalcall);
  10176. //Ugly, but target selector is passed in as the target.expr. Should possibly have an extra parameter.
  10177. CHqlBoundTarget target;
  10178. target.expr.set(self->querySelector());
  10179. doBuildCall(ctx, &target, transform, NULL);
  10180. }
  10181. void HqlCppTranslator::doTransform(BuildCtx & ctx, IHqlExpression * transform, BoundRow * self)
  10182. {
  10183. HqlExprArray assigns;
  10184. IHqlExpression * record = self->queryRecord();
  10185. TransformBuilder builder(*this, ctx, record, self, assigns);
  10186. builder.doTransform(ctx, transform, self);
  10187. }
  10188. void HqlCppTranslator::doUpdateTransform(BuildCtx & ctx, IHqlExpression * transform, BoundRow * self, BoundRow * previous, bool alwaysNextRow)
  10189. {
  10190. HqlExprArray assigns;
  10191. IHqlExpression * record = self->queryRecord();
  10192. UpdateTransformBuilder builder(*this, ctx, record, self, previous->querySelector(), assigns, alwaysNextRow);
  10193. builder.doTransform(ctx, transform, self);
  10194. }
  10195. void HqlCppTranslator::doInlineTransform(BuildCtx & ctx, IHqlExpression * transform, BoundRow * targetRow)
  10196. {
  10197. Owned<BoundRow> rowBuilder = createRowBuilder(ctx, targetRow);
  10198. doTransform(ctx, transform, rowBuilder);
  10199. finalizeTempRow(ctx, targetRow, rowBuilder);
  10200. }
  10201. void HqlCppTranslator::precalculateFieldOffsets(BuildCtx & ctx, IHqlExpression * expr, BoundRow * cursor)
  10202. {
  10203. if (!cursor)
  10204. return;
  10205. if (isFixedRecordSize(cursor->queryRecord()))
  10206. return;
  10207. IHqlExpression * lastField;
  10208. IHqlExpression * selector = cursor->querySelector();
  10209. {
  10210. FieldAccessAnalyser analyser(selector);
  10211. analyser.analyse(expr, 0);
  10212. lastField = analyser.queryLastFieldAccessed();
  10213. }
  10214. if (!lastField)
  10215. return;
  10216. OwnedHqlExpr selected = createSelectExpr(LINK(selector), LINK(lastField));
  10217. Owned<IReferenceSelector> ref = buildReference(ctx, selected);
  10218. CHqlBoundExpr boundOffset;
  10219. ref->buildAddress(ctx, boundOffset);
  10220. }
  10221. BoundRow * HqlCppTranslator::buildTransformCursors(BuildCtx & ctx, IHqlExpression * transform, IHqlExpression * left, IHqlExpression * right, IHqlExpression * self, IHqlExpression * selSeq)
  10222. {
  10223. if (transform->getOperator() == no_skip)
  10224. return NULL;
  10225. assertex(recordTypesMatch(self->queryRecord(), transform->queryRecord()));
  10226. if (left)
  10227. ctx.addQuoted("const unsigned char * left = (const unsigned char *) _left;");
  10228. if (right)
  10229. ctx.addQuoted("const unsigned char * right = (const unsigned char *) _right;");
  10230. // Bind left to "left" and right to RIGHT
  10231. BoundRow * leftRow = NULL;
  10232. BoundRow * rightRow = NULL;
  10233. if (left)
  10234. leftRow = bindTableCursor(ctx, left, "left", no_left, selSeq);
  10235. if (right)
  10236. rightRow = bindTableCursor(ctx, right, "right", no_right, selSeq);
  10237. if (options.precalculateFieldOffsets)
  10238. precalculateFieldOffsets(ctx, transform, leftRow);
  10239. return bindSelf(ctx, self, "crSelf");
  10240. }
  10241. void HqlCppTranslator::doBuildTransformBody(BuildCtx & ctx, IHqlExpression * transform, BoundRow * selfCursor)
  10242. {
  10243. if (transform->getOperator() == no_skip)
  10244. {
  10245. ctx.addReturn(queryZero());
  10246. return;
  10247. }
  10248. associateSkipReturnMarker(ctx, queryZero(), selfCursor);
  10249. doTransform(ctx, transform, selfCursor);
  10250. buildReturnRecordSize(ctx, selfCursor);
  10251. }
  10252. void HqlCppTranslator::buildTransformBody(BuildCtx & ctx, IHqlExpression * transform, IHqlExpression * left, IHqlExpression * right, IHqlExpression * self, IHqlExpression * selSeq)
  10253. {
  10254. BoundRow * selfCursor = buildTransformCursors(ctx, transform, left, right, self, selSeq);
  10255. doBuildTransformBody(ctx, transform, selfCursor);
  10256. }
  10257. void HqlCppTranslator::buildIterateTransformFunction(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * transform, IHqlExpression * counter, IHqlExpression * selSeq)
  10258. {
  10259. BuildCtx funcctx(ctx);
  10260. funcctx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf, const void * _left, const void * _right, unsigned __int64 counter)");
  10261. ensureRowAllocated(funcctx, "crSelf");
  10262. associateCounter(funcctx, counter, "counter");
  10263. buildTransformBody(funcctx, transform, dataset, dataset, dataset, selSeq);
  10264. }
  10265. void HqlCppTranslator::buildRollupTransformFunction(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * transform, IHqlExpression * selSeq)
  10266. {
  10267. BuildCtx funcctx(ctx);
  10268. funcctx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf, const void * _left, const void * _right)");
  10269. ensureRowAllocated(funcctx, "crSelf");
  10270. buildTransformBody(funcctx, transform, dataset, dataset, dataset, selSeq);
  10271. }
  10272. ABoundActivity * HqlCppTranslator::doBuildActivityIterate(BuildCtx & ctx, IHqlExpression * expr)
  10273. {
  10274. IHqlExpression * dataset = expr->queryChild(0);
  10275. IHqlExpression * transform = expr->queryChild(1);
  10276. IHqlExpression * counter = queryPropertyChild(expr, _countProject_Atom, 0);
  10277. IHqlExpression * selSeq = querySelSeq(expr);
  10278. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  10279. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKiterate, expr, "Iterate");
  10280. buildActivityFramework(instance);
  10281. buildInstancePrefix(instance);
  10282. buildIterateTransformFunction(instance->startctx, dataset, transform, counter, selSeq);
  10283. buildClearRecordMember(instance->createctx, "", dataset);
  10284. if (transformContainsSkip(transform))
  10285. doBuildBoolFunction(instance->classctx, "canFilter", true);
  10286. buildInstanceSuffix(instance);
  10287. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  10288. return instance->getBoundActivity();
  10289. }
  10290. //---------------------------------------------------------------------------
  10291. IHqlExpression * HqlCppTranslator::queryExpandAliasScope(BuildCtx & ctx, IHqlExpression * expr)
  10292. {
  10293. while (expr->getOperator() == no_alias_scope)
  10294. {
  10295. expandAliasScope(ctx, expr);
  10296. expr = expr->queryChild(0);
  10297. }
  10298. return expr;
  10299. }
  10300. void HqlCppTranslator::buildProcessTransformFunction(BuildCtx & ctx, IHqlExpression * expr)
  10301. {
  10302. IHqlExpression * dataset = expr->queryChild(0);
  10303. IHqlExpression * right = expr->queryChild(1);
  10304. IHqlExpression * transformRow = expr->queryChild(2);
  10305. IHqlExpression * transformRight = expr->queryChild(3);
  10306. IHqlExpression * counter = queryPropertyChild(expr, _countProject_Atom, 0);
  10307. IHqlExpression * selSeq = querySelSeq(expr);
  10308. BuildCtx funcctx(ctx);
  10309. funcctx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf, ARowBuilder & crSelfRight, const void * _left, const void * _right, unsigned __int64 counter)");
  10310. associateCounter(funcctx, counter, "counter");
  10311. if ((transformRow->getOperator() == no_skip) || (transformRight->getOperator() == no_skip))
  10312. {
  10313. funcctx.addReturn(queryZero());
  10314. return;
  10315. }
  10316. ensureRowAllocated(funcctx, "crSelf");
  10317. ensureRowAllocated(funcctx, "crSelfRight");
  10318. funcctx.addQuoted("const unsigned char * left = (const unsigned char *) _left;");
  10319. funcctx.addQuoted("const unsigned char * right = (const unsigned char *) _right;");
  10320. bindTableCursor(funcctx, dataset, "left", no_left, selSeq);
  10321. bindTableCursor(funcctx, right, "right", no_right, selSeq);
  10322. LinkedHqlExpr skipReturnValue = queryZero();
  10323. associateSkipReturnMarker(funcctx, skipReturnValue, NULL);
  10324. if (!recordTypesMatch(dataset, right))
  10325. {
  10326. //self won't clash, so can generate efficient code.
  10327. //Perform cse on both transforms
  10328. OwnedHqlExpr comma = createComma(LINK(transformRow), LINK(transformRight));
  10329. comma.setown(spotScalarCSE(comma));
  10330. if (comma->getOperator() == no_alias_scope)
  10331. comma.set(comma->queryChild(0));
  10332. HqlExprArray unwound;
  10333. comma->unwindList(unwound, no_comma);
  10334. unsigned max = unwound.ordinality();
  10335. BoundRow * selfCursor = bindSelf(funcctx, dataset, "crSelf");
  10336. BoundRow * selfRowCursor = bindSelf(funcctx, right, "crSelfRight");
  10337. for (unsigned i=0; i<max-2; i++)
  10338. buildStmt(funcctx, &unwound.item(i));
  10339. IHqlExpression * newTransformRow = queryExpandAliasScope(funcctx, &unwound.item(max-2));
  10340. IHqlExpression * newTransformRight = queryExpandAliasScope(funcctx, &unwound.item(max-1));
  10341. assertex(newTransformRow->getOperator() == no_transform && newTransformRight->getOperator() == no_transform);
  10342. doTransform(funcctx, newTransformRow, selfCursor);
  10343. doTransform(funcctx, newTransformRight, selfRowCursor);
  10344. buildReturnRecordSize(funcctx, selfCursor);
  10345. }
  10346. else
  10347. {
  10348. BuildCtx ctx1(funcctx);
  10349. ctx1.addGroup();
  10350. BoundRow * selfRowCursor = bindSelf(ctx1, right, "crSelfRight");
  10351. doTransform(ctx1, transformRight, selfRowCursor);
  10352. BuildCtx ctx2(funcctx);
  10353. ctx2.addGroup();
  10354. BoundRow * selfCursor = bindSelf(ctx2, dataset, "crSelf");
  10355. doTransform(ctx2, transformRow, selfCursor);
  10356. buildReturnRecordSize(ctx2, selfCursor);
  10357. }
  10358. }
  10359. ABoundActivity * HqlCppTranslator::doBuildActivityProcess(BuildCtx & ctx, IHqlExpression * expr)
  10360. {
  10361. IHqlExpression * dataset = expr->queryChild(0);
  10362. IHqlExpression * right = expr->queryChild(1);
  10363. IHqlExpression * transformRow = expr->queryChild(2);
  10364. IHqlExpression * transformRight = expr->queryChild(3);
  10365. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  10366. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKprocess, expr, "Process");
  10367. buildActivityFramework(instance);
  10368. buildInstancePrefix(instance);
  10369. buildMetaMember(instance->classctx, right->queryRecord(), "queryRightRecordSize");
  10370. {
  10371. BuildCtx initialctx(instance->startctx);
  10372. initialctx.addQuotedCompound("virtual size32_t createInitialRight(ARowBuilder & crSelf)");
  10373. ensureRowAllocated(initialctx, "crSelf");
  10374. BoundRow * cursor = bindSelf(initialctx, right, "crSelf");
  10375. Owned<IReferenceSelector> createdRef = createReferenceSelector(cursor);
  10376. buildRowAssign(initialctx, createdRef, right);
  10377. buildReturnRecordSize(initialctx, cursor);
  10378. }
  10379. buildProcessTransformFunction(instance->startctx, expr);
  10380. if (transformContainsSkip(transformRow) || transformContainsSkip(transformRight))
  10381. doBuildBoolFunction(instance->classctx, "canFilter", true);
  10382. buildInstanceSuffix(instance);
  10383. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  10384. return instance->getBoundActivity();
  10385. }
  10386. //---------------------------------------------------------------------------
  10387. ABoundActivity * HqlCppTranslator::doBuildActivitySelectNth(BuildCtx & ctx, IHqlExpression * expr)
  10388. {
  10389. IHqlExpression * dataset = expr->queryChild(0);
  10390. IHqlExpression * index = expr->queryChild(1);
  10391. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  10392. //If selecting 1st element of a non-grouped aggregate (which can only produce one row) then don't need to
  10393. //add the selectNth operator.
  10394. IHqlExpression * search = dataset;
  10395. if (hasSingleRow(dataset))
  10396. {
  10397. IValue * indexValue = index->queryValue();
  10398. if (indexValue && (indexValue->getIntValue() == 1))
  10399. {
  10400. //index first element - don't need to do anything...
  10401. //if x[n] is ever used as a dataset this assumption is invalid....
  10402. return LINK(boundDataset);
  10403. }
  10404. }
  10405. #if 0
  10406. //MORE: Should optimize left.child[1] and probably others - e.g., localresult[n]
  10407. switch (dataset->getOperator())
  10408. {
  10409. case no_select:
  10410. if (!isNewSelector(dataset))
  10411. return doBuildActivityChildDataset(ctx, expr);
  10412. break;
  10413. //MORE: What other selects are worth special casing?
  10414. }
  10415. #endif
  10416. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKselectn, expr, "SelectN");
  10417. buildActivityFramework(instance);
  10418. if (matchesConstantValue(index, 1))
  10419. instance->graphLabel.set("Select 1st");
  10420. buildInstancePrefix(instance);
  10421. BuildCtx funcctx(instance->startctx);
  10422. funcctx.addQuotedCompound("virtual unsigned __int64 getRowToSelect()");
  10423. buildReturn(funcctx, index);
  10424. buildClearRecordMember(instance->createctx, "", dataset);
  10425. buildInstanceSuffix(instance);
  10426. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  10427. return instance->getBoundActivity();
  10428. }
  10429. //---------------------------------------------------------------------------
  10430. void HqlCppTranslator::doBuildAggregateClearFunc(BuildCtx & ctx, IHqlExpression * expr)
  10431. {
  10432. IHqlExpression * tgtRecord = expr->queryChild(1);
  10433. IHqlExpression * transform = expr->queryChild(2);
  10434. BuildCtx funcctx(ctx);
  10435. funcctx.addQuotedCompound("virtual size32_t clearAggregate(ARowBuilder & crSelf)");
  10436. OwnedHqlExpr resultDataset = createDataset(no_anon, LINK(tgtRecord));
  10437. BoundRow * selfRow = bindSelf(funcctx, resultDataset, "crSelf");
  10438. if (!isKnownTransform(transform))
  10439. {
  10440. OwnedHqlExpr clearCall = createClearRowCall(funcctx, selfRow);
  10441. funcctx.addReturn(clearCall);
  10442. return;
  10443. }
  10444. ensureRowAllocated(funcctx, "crSelf");
  10445. unsigned numAggregates = transform->numChildren();
  10446. unsigned idx;
  10447. OwnedHqlExpr self = getSelf(tgtRecord);
  10448. for (idx = 0; idx < numAggregates; idx++)
  10449. {
  10450. IHqlExpression * cur = transform->queryChild(idx);
  10451. OwnedHqlExpr target = selfRow->bindToRow(cur->queryChild(0), self);
  10452. IHqlExpression * src = cur->queryChild(1);
  10453. switch (src->getOperator())
  10454. {
  10455. case no_countgroup:
  10456. case no_maxgroup:
  10457. case no_mingroup:
  10458. case no_sumgroup:
  10459. case no_existsgroup:
  10460. buildClear(funcctx, target);
  10461. break;
  10462. case no_notexistsgroup:
  10463. buildAssign(funcctx, target, queryBoolExpr(true));
  10464. break;
  10465. default:
  10466. if (src->isConstant())
  10467. buildAssign(funcctx, target, src);
  10468. else
  10469. buildClear(funcctx, target);
  10470. break;
  10471. }
  10472. }
  10473. buildReturnRecordSize(funcctx, selfRow);
  10474. }
  10475. void HqlCppTranslator::doBuildAggregateFirstFunc(BuildCtx & ctx, IHqlExpression * expr)
  10476. {
  10477. IHqlExpression * dataset = expr->queryChild(0);
  10478. IHqlExpression * tgtRecord = expr->queryChild(1);
  10479. IHqlExpression * transform = expr->queryChild(2);
  10480. BuildCtx funcctx(ctx);
  10481. funcctx.addQuotedCompound("virtual size32_t processFirst(ARowBuilder & crSelf, const void * _src)");
  10482. ensureRowAllocated(funcctx, "crSelf");
  10483. funcctx.addQuoted("unsigned char * src = (unsigned char *) _src;");
  10484. //NOTE: no_throughaggregate recordof(expr) != tgtRecord => we need to create a temporary dataset
  10485. OwnedHqlExpr resultDataset = createDataset(no_anon, LINK(tgtRecord));
  10486. BoundRow * selfRow = bindSelf(funcctx, resultDataset, "crSelf");
  10487. bindTableCursor(funcctx, dataset, "src");
  10488. doBuildAggregateProcessTransform(funcctx, selfRow, expr, queryBoolExpr(false));
  10489. buildReturnRecordSize(funcctx, selfRow);
  10490. }
  10491. void HqlCppTranslator::doBuildAggregateNextFunc(BuildCtx & ctx, IHqlExpression * expr)
  10492. {
  10493. IHqlExpression * dataset = expr->queryChild(0);
  10494. IHqlExpression * tgtRecord = expr->queryChild(1);
  10495. BuildCtx funcctx(ctx);
  10496. funcctx.addQuotedCompound("virtual size32_t processNext(ARowBuilder & crSelf, const void * _src)");
  10497. //no need ensureRowAllocated(funcctx, "crSelf");
  10498. funcctx.addQuoted("unsigned char * src = (unsigned char *) _src;");
  10499. OwnedHqlExpr resultDataset = createDataset(no_anon, LINK(tgtRecord));
  10500. BoundRow * selfRow = bindSelf(funcctx, resultDataset, "crSelf");
  10501. bindTableCursor(funcctx, dataset, "src");
  10502. doBuildAggregateProcessTransform(funcctx, selfRow, expr, queryBoolExpr(true));
  10503. buildReturnRecordSize(funcctx, selfRow);
  10504. }
  10505. void HqlCppTranslator::doBuildAggregateProcessTransform(BuildCtx & ctx, BoundRow * selfRow, IHqlExpression * expr, IHqlExpression * alreadyDoneExpr)
  10506. {
  10507. bool alwaysFirstRow = matchesBoolean(alreadyDoneExpr, false);
  10508. bool alwaysNextRow = matchesBoolean(alreadyDoneExpr, true);
  10509. OwnedHqlExpr notAlreadyDone = getInverse(alreadyDoneExpr);
  10510. IHqlExpression * transform = expr->queryChild(2);
  10511. unsigned numAggregates = transform->numChildren();
  10512. unsigned idx;
  10513. bool isVariableOffset = false;
  10514. OwnedHqlExpr self = getSelf(expr->queryChild(1));
  10515. for (idx = 0; idx < numAggregates; idx++)
  10516. {
  10517. IHqlExpression * cur = transform->queryChild(idx);
  10518. if (cur->isAttribute())
  10519. continue;
  10520. OwnedHqlExpr target = selfRow->bindToRow(cur->queryChild(0), self);
  10521. IHqlExpression * src = cur->queryChild(1);
  10522. IHqlExpression * arg = src->queryChild(0);
  10523. IHqlExpression * cond = src->queryChild(1);
  10524. BuildCtx condctx(ctx);
  10525. node_operator srcOp = src->getOperator();
  10526. switch (srcOp)
  10527. {
  10528. case no_countgroup:
  10529. {
  10530. assertex(!(arg && isVariableOffset));
  10531. if (arg)
  10532. buildFilter(condctx, arg);
  10533. OwnedHqlExpr one = createConstant(createIntValue(1,8,true));
  10534. if (alwaysFirstRow)
  10535. {
  10536. buildAssign(condctx, target, one);
  10537. }
  10538. else
  10539. {
  10540. if (!alwaysNextRow && isVariableOffset)
  10541. {
  10542. IHqlStmt * ifStmt = condctx.addFilter(notAlreadyDone);
  10543. buildAssign(condctx, target, one);
  10544. condctx.selectElse(ifStmt);
  10545. }
  10546. buildIncrementAssign(condctx, target, one);
  10547. }
  10548. }
  10549. break;
  10550. case no_sumgroup:
  10551. {
  10552. assertex(!(cond && isVariableOffset));
  10553. if (cond)
  10554. buildFilter(condctx, cond);
  10555. if (alwaysFirstRow)
  10556. {
  10557. buildAssign(condctx, target, arg);
  10558. }
  10559. else
  10560. {
  10561. if (!alwaysNextRow && isVariableOffset)
  10562. {
  10563. IHqlStmt * ifStmt = condctx.addFilter(notAlreadyDone);
  10564. buildAssign(condctx, target, arg);
  10565. condctx.selectElse(ifStmt);
  10566. }
  10567. OwnedITypeInfo type = getPromotedECLType(target->queryType(), arg->queryType());
  10568. buildIncrementAssign(condctx, target, arg);
  10569. }
  10570. }
  10571. break;
  10572. case no_maxgroup:
  10573. case no_mingroup:
  10574. {
  10575. node_operator compareOp = (srcOp == no_maxgroup) ? no_gt : no_lt;
  10576. assertex(!cond);
  10577. OwnedHqlExpr castArg = ensureExprType(arg, target->queryType()); // cast to correct type, assume it can fit in the target type.
  10578. if (!alwaysFirstRow)
  10579. {
  10580. castArg.setown(buildSimplifyExpr(condctx, castArg));
  10581. OwnedHqlExpr compare = createBoolExpr (compareOp, LINK(castArg), LINK(target));
  10582. if (!alwaysNextRow)
  10583. compare.setown(createBoolExpr(no_or, LINK(notAlreadyDone), LINK(compare)));
  10584. buildFilter(condctx, compare);
  10585. }
  10586. buildAssign(condctx, target, castArg);
  10587. }
  10588. break;
  10589. case no_existsgroup:
  10590. case no_notexistsgroup:
  10591. assertex(!(arg && isVariableOffset));
  10592. cond = arg;
  10593. if (cond || !alwaysNextRow)
  10594. {
  10595. //The assign is conditional because unconditionally it is done in the AggregateFirst
  10596. if (cond)
  10597. buildFilter(condctx, cond);
  10598. buildAssign(condctx, target, queryBoolExpr(srcOp == no_existsgroup));
  10599. }
  10600. break;
  10601. default:
  10602. if (!src->isConstant() || isVariableOffset)
  10603. {
  10604. if (!alwaysNextRow)
  10605. {
  10606. if (!alwaysFirstRow)
  10607. buildFilter(condctx, notAlreadyDone);
  10608. buildAssign(condctx, target, src);
  10609. }
  10610. }
  10611. break;
  10612. }
  10613. if (target->queryType()->getSize() == UNKNOWN_LENGTH)
  10614. isVariableOffset = true;
  10615. }
  10616. }
  10617. void HqlCppTranslator::doBuildAggregateMergeFunc(BuildCtx & ctx, IHqlExpression * expr)
  10618. {
  10619. IHqlExpression * tgtRecord = expr->queryChild(1);
  10620. IHqlExpression * transform = expr->queryChild(2);
  10621. OwnedHqlExpr selSeq = createDummySelectorSequence();
  10622. BuildCtx funcctx(ctx);
  10623. funcctx.addQuotedCompound("virtual size32_t mergeAggregate(ARowBuilder & crSelf, const void * _right)");
  10624. //ensureRowAllocated(funcctx, "crSelf"); must be non null
  10625. funcctx.addQuoted("unsigned char * right = (unsigned char *) _right;");
  10626. OwnedHqlExpr resultDataset = createDataset(no_anon, LINK(tgtRecord));
  10627. BoundRow * selfRow = bindSelf(funcctx, resultDataset, "crSelf");
  10628. BoundRow * leftCursor = bindTableCursor(funcctx, resultDataset, "left", no_left, selSeq);
  10629. BoundRow * rightCursor = bindTableCursor(funcctx, resultDataset, "right", no_right, selSeq);
  10630. unsigned numAggregates = transform->numChildren();
  10631. unsigned idx;
  10632. IHqlExpression * left = leftCursor->querySelector();
  10633. IHqlExpression * right = rightCursor->querySelector();
  10634. OwnedHqlExpr self = getSelf(tgtRecord);
  10635. for (idx = 0; idx < numAggregates; idx++)
  10636. {
  10637. IHqlExpression * cur = transform->queryChild(idx);
  10638. if (cur->isAttribute())
  10639. continue;
  10640. OwnedHqlExpr target = selfRow->bindToRow(cur->queryChild(0), self);
  10641. OwnedHqlExpr src = replaceSelector(cur->queryChild(0), self, right);
  10642. IHqlExpression * op = cur->queryChild(1);
  10643. //MORE: How bind cursors...
  10644. switch (op->getOperator())
  10645. {
  10646. case no_countgroup:
  10647. case no_sumgroup:
  10648. {
  10649. buildIncrementAssign(funcctx, target, src);
  10650. }
  10651. break;
  10652. case no_maxgroup:
  10653. {
  10654. OwnedHqlExpr compare = createBoolExpr (no_gt, LINK(src), LINK(target));
  10655. BuildCtx filteredctx(funcctx);
  10656. buildFilter(filteredctx, compare);
  10657. buildAssign(filteredctx, target, src);
  10658. }
  10659. break;
  10660. case no_mingroup:
  10661. {
  10662. OwnedHqlExpr compare = createBoolExpr (no_lt, LINK(src), LINK(target));
  10663. BuildCtx filteredctx(funcctx);
  10664. buildFilter(filteredctx, compare);
  10665. buildAssign(filteredctx, target, src);
  10666. }
  10667. break;
  10668. case no_existsgroup:
  10669. {
  10670. BuildCtx condctx(funcctx);
  10671. buildFilter(condctx, src);
  10672. buildAssign(condctx, target, queryBoolExpr(true));
  10673. break;
  10674. }
  10675. case no_notexistsgroup:
  10676. {
  10677. BuildCtx condctx(funcctx);
  10678. buildFilter(condctx, target);
  10679. buildAssign(condctx, target, src);
  10680. }
  10681. break;
  10682. default:
  10683. //already filled in and wouldn't be legal to have an expression in this case anyway...
  10684. break;
  10685. }
  10686. }
  10687. buildReturnRecordSize(funcctx, selfRow);
  10688. }
  10689. //--------------------------------------------------------------------------------------
  10690. // User aggregate helpers
  10691. //--------------------------------------------------------------------------------------
  10692. void HqlCppTranslator::doBuildUserAggregateProcessTransform(BuildCtx & ctx, BoundRow * selfRow, IHqlExpression * expr, IHqlExpression * transform, IHqlExpression * alreadyDoneExpr)
  10693. {
  10694. bool alwaysFirstRow = matchesBoolean(alreadyDoneExpr, false);
  10695. bool alwaysNextRow = matchesBoolean(alreadyDoneExpr, true);
  10696. OwnedHqlExpr right = createSelector(no_right, expr, querySelSeq(expr));
  10697. bool usesRight = exprReferencesDataset(transform, right);
  10698. BuildCtx condctx(ctx);
  10699. if (!isKnownTransform(transform))
  10700. {
  10701. if (!alwaysNextRow)
  10702. {
  10703. IHqlStmt * ifStmt = NULL;
  10704. if (!alwaysFirstRow)
  10705. ifStmt = condctx.addFilter(alreadyDoneExpr);
  10706. if (usesRight)
  10707. {
  10708. CHqlBoundExpr boundNullRow;
  10709. buildDefaultRow(condctx, transform, boundNullRow);
  10710. bindTableCursor(condctx, transform->queryRecord(), boundNullRow.expr, no_right, querySelSeq(expr));
  10711. }
  10712. doUserTransform(condctx, transform, selfRow);
  10713. if (ifStmt)
  10714. condctx.selectElse(ifStmt);
  10715. }
  10716. if (!alwaysFirstRow)
  10717. {
  10718. if (usesRight)
  10719. bindTableCursor(condctx, transform->queryRecord(), selfRow->queryBound(), no_right, querySelSeq(expr));
  10720. doUserTransform(condctx, transform, selfRow);
  10721. }
  10722. }
  10723. else
  10724. {
  10725. if (usesRight)
  10726. {
  10727. BoundRow * rightCursor;
  10728. if (alwaysNextRow)
  10729. rightCursor = bindTableCursor(condctx, transform->queryRecord(), selfRow->queryBound(), no_right, querySelSeq(expr));
  10730. else
  10731. {
  10732. CHqlBoundExpr boundNullRow;
  10733. buildDefaultRow(condctx, transform, boundNullRow);
  10734. if (alwaysFirstRow)
  10735. {
  10736. //MORE: Only do this (and create default row) if transform refers to RIGHT...
  10737. rightCursor = bindTableCursor(condctx, transform->queryRecord(), boundNullRow.expr, no_right, querySelSeq(expr));
  10738. }
  10739. else
  10740. {
  10741. //create a temporary
  10742. Owned<ITypeInfo> rowType = makeReferenceModifier(expr->getType());
  10743. OwnedHqlExpr rowExpr = ctx.getTempDeclare(rowType, NULL);
  10744. OwnedHqlExpr defaultRowPtr = getPointer(boundNullRow.expr);
  10745. OwnedHqlExpr condRow = createValue(no_if, LINK(rowType), LINK(alreadyDoneExpr), LINK(selfRow->queryBound()), LINK(defaultRowPtr));
  10746. condctx.addAssign(rowExpr, condRow);
  10747. rightCursor = bindTableCursor(condctx, transform->queryRecord(), rowExpr, no_right, querySelSeq(expr));
  10748. }
  10749. }
  10750. if (alwaysFirstRow)
  10751. doTransform(condctx, transform, selfRow);
  10752. else
  10753. doUpdateTransform(condctx, transform, selfRow, rightCursor, alwaysNextRow);
  10754. }
  10755. else
  10756. doTransform(condctx, transform, selfRow);
  10757. }
  10758. }
  10759. //------------------------------------------------------------------------------------------------
  10760. static bool matchesSelect(IHqlExpression * expr, IHqlExpression * selector, IHqlExpression * field)
  10761. {
  10762. if (isCast(expr))
  10763. {
  10764. ITypeInfo * afterType = expr->queryType();
  10765. ITypeInfo * beforeType = expr->queryChild(0)->queryType();
  10766. if (preservesValue(afterType, beforeType))
  10767. expr = expr->queryChild(0);
  10768. else if (beforeType->isInteger() && afterType->isInteger() && beforeType->getSize() == afterType->getSize())
  10769. expr = expr->queryChild(0);
  10770. }
  10771. if (expr->getOperator() != no_select)
  10772. return false;
  10773. if (expr->queryChild(0) != selector)
  10774. return false;
  10775. if (expr->queryChild(1) != field)
  10776. return false;
  10777. return true;
  10778. }
  10779. //MORE: Derive a merge transform by walking and spotting self.x := self.x := right.x op a or a op right.x
  10780. class MergeTransformCreator
  10781. {
  10782. public:
  10783. MergeTransformCreator(IHqlExpression * expr)
  10784. {
  10785. IHqlExpression * dataset = expr->queryChild(0);
  10786. IHqlExpression * selSeq = querySelSeq(expr);
  10787. IHqlExpression * rowsid = expr->queryProperty(_rowsid_Atom);
  10788. self.setown(getSelf(expr));
  10789. left.setown(createSelector(no_left, dataset, selSeq));
  10790. right.setown(createSelector(no_right, expr, selSeq));
  10791. mergeLeft.set(right);
  10792. OwnedHqlExpr rows = createDataset(no_rows, LINK(right), LINK(rowsid));
  10793. mergeRight.setown(createRow(no_selectnth, LINK(rows), createConstant(2)));
  10794. requiresOrderedMerge = false;
  10795. }
  10796. IHqlExpression * transform(IHqlExpression * expr);
  10797. inline bool isOrdered() { return requiresOrderedMerge; }
  10798. protected:
  10799. IHqlExpression * transformAssign(IHqlExpression * expr);
  10800. protected:
  10801. OwnedHqlExpr self;
  10802. OwnedHqlExpr left;
  10803. OwnedHqlExpr right;
  10804. OwnedHqlExpr mergeLeft;
  10805. OwnedHqlExpr mergeRight;
  10806. bool requiresOrderedMerge;
  10807. };
  10808. //self.x := right.x append|concat f() -> self.x := right.x <op> rows(right)[2].x
  10809. //self.x := f() append|concat right.x -> self.x := rows(right)[2].x <op> right.x
  10810. //self.x := right.x [+|band|bor|max|min|xor] f() -> self.x := right.x <op> rows(right)[2].x
  10811. //self.x := f() [+|band|bor|max|min|xor] right.x -> self.x := right.x <op> rows(right)[2].x
  10812. IHqlExpression * MergeTransformCreator::transformAssign(IHqlExpression * expr)
  10813. {
  10814. IHqlExpression * lhs = expr->queryChild(0);
  10815. IHqlExpression * rhs = expr->queryChild(1);
  10816. IHqlExpression * field = lhs->queryChild(1);
  10817. bool commutative = true;
  10818. node_operator op = rhs->getOperator();
  10819. HqlExprArray args;
  10820. switch (op)
  10821. {
  10822. case no_concat:
  10823. case no_addfiles:
  10824. commutative = false;
  10825. rhs->unwindList(args, op);
  10826. break;
  10827. case no_add:
  10828. case no_band:
  10829. case no_bor:
  10830. case no_bxor:
  10831. case no_mul:
  10832. rhs->unwindList(args, op);
  10833. break;
  10834. case no_sumlist:
  10835. case no_maxlist:
  10836. case no_minlist:
  10837. {
  10838. IHqlExpression * list = rhs->queryChild(0);
  10839. if (list->getOperator() != no_list)
  10840. return false;
  10841. unwindChildren(args, list);
  10842. break;
  10843. }
  10844. default:
  10845. //ok, if it is only assigned once.
  10846. if (exprReferencesDataset(rhs, right))
  10847. return false;
  10848. //need to preserve the value if at a variable offset... Should be stripped if unnecessary
  10849. OwnedHqlExpr newRhs = createSelectExpr(LINK(mergeLeft), LINK(field));
  10850. return createAssign(LINK(lhs), newRhs.getClear());
  10851. }
  10852. unsigned matchPos = NotFound;
  10853. ForEachItemIn(i, args)
  10854. {
  10855. IHqlExpression & cur = args.item(i);
  10856. if (exprReferencesDataset(&cur, right))
  10857. {
  10858. if ((matchPos == NotFound) && matchesSelect(&cur, right, field))
  10859. matchPos = i;
  10860. else
  10861. return NULL;
  10862. }
  10863. }
  10864. if (matchPos == NotFound)
  10865. return NULL;
  10866. if (!commutative)
  10867. requiresOrderedMerge = true;
  10868. HqlExprArray newArgs;
  10869. if (commutative || (matchPos == 0))
  10870. {
  10871. newArgs.append(*createSelectExpr(LINK(mergeLeft), LINK(field)));
  10872. newArgs.append(*createSelectExpr(LINK(mergeRight), LINK(field)));
  10873. }
  10874. else if (matchPos == args.ordinality() - 1)
  10875. {
  10876. newArgs.append(*createSelectExpr(LINK(mergeRight), LINK(field)));
  10877. newArgs.append(*createSelectExpr(LINK(mergeLeft), LINK(field)));
  10878. }
  10879. if (newArgs.ordinality())
  10880. {
  10881. switch (op)
  10882. {
  10883. case no_sumlist:
  10884. case no_minlist:
  10885. case no_maxlist:
  10886. {
  10887. OwnedHqlExpr list = rhs->queryChild(0)->clone(newArgs);
  10888. newArgs.kill();
  10889. newArgs.append(*list.getClear());
  10890. break;
  10891. }
  10892. }
  10893. return createAssign(LINK(lhs), rhs->clone(newArgs));
  10894. }
  10895. return NULL;
  10896. }
  10897. IHqlExpression * MergeTransformCreator::transform(IHqlExpression * expr)
  10898. {
  10899. HqlExprArray children;
  10900. ForEachChild(i, expr)
  10901. {
  10902. IHqlExpression * cur = expr->queryChild(i);
  10903. OwnedHqlExpr mapped;
  10904. switch (cur->getOperator())
  10905. {
  10906. case no_assign:
  10907. mapped.setown(transformAssign(cur));
  10908. if (!mapped)
  10909. return NULL;
  10910. break;
  10911. case no_assignall:
  10912. mapped.setown(transform(cur));
  10913. if (!mapped)
  10914. return NULL;
  10915. break;
  10916. case no_alias_scope:
  10917. //Not so sure - there shouldn't be any at this point... Maybe we should allow...
  10918. return NULL;
  10919. case no_assert:
  10920. case no_skip:
  10921. return NULL;
  10922. case no_attr:
  10923. case no_attr_link:
  10924. case no_attr_expr:
  10925. break;
  10926. default:
  10927. UNIMPLEMENTED;
  10928. }
  10929. if (mapped)
  10930. children.append(*mapped.getClear());
  10931. }
  10932. return expr->clone(children);
  10933. }
  10934. //------------------------------------------------------------------------------------------------
  10935. // Replace SELF.x := F(LEFT) with SELF.x := RIGHT.x.
  10936. //i) To ensure the first value is always used and
  10937. //ii) to allow subsequent optimizations to remove the assignment
  10938. class NextTransformCreator
  10939. {
  10940. public:
  10941. NextTransformCreator(IHqlExpression * expr)
  10942. {
  10943. IHqlExpression * dataset = expr->queryChild(0);
  10944. IHqlExpression * selSeq = querySelSeq(expr);
  10945. IHqlExpression * rowsid = expr->queryProperty(_rowsid_Atom);
  10946. self.setown(getSelf(expr));
  10947. left.setown(createSelector(no_left, dataset, selSeq));
  10948. right.setown(createSelector(no_right, expr, selSeq));
  10949. }
  10950. IHqlExpression * transform(IHqlExpression * expr);
  10951. protected:
  10952. IHqlExpression * transformAssign(IHqlExpression * expr);
  10953. protected:
  10954. OwnedHqlExpr self;
  10955. OwnedHqlExpr left;
  10956. OwnedHqlExpr right;
  10957. };
  10958. IHqlExpression * NextTransformCreator::transformAssign(IHqlExpression * expr)
  10959. {
  10960. IHqlExpression * lhs = expr->queryChild(0);
  10961. IHqlExpression * rhs = expr->queryChild(1);
  10962. IHqlExpression * field = lhs->queryChild(1);
  10963. if (!rhs->isConstant() && !exprReferencesDataset(rhs, right))
  10964. {
  10965. OwnedHqlExpr newRhs = createSelectExpr(LINK(right), LINK(field));
  10966. return createAssign(LINK(lhs), newRhs.getClear());
  10967. }
  10968. return LINK(expr);
  10969. }
  10970. IHqlExpression * NextTransformCreator::transform(IHqlExpression * expr)
  10971. {
  10972. HqlExprArray children;
  10973. ForEachChild(i, expr)
  10974. {
  10975. IHqlExpression * cur = expr->queryChild(i);
  10976. OwnedHqlExpr mapped = LINK(cur);
  10977. switch (cur->getOperator())
  10978. {
  10979. case no_assign:
  10980. mapped.setown(transformAssign(cur));
  10981. break;
  10982. case no_assignall:
  10983. mapped.setown(transform(cur));
  10984. break;
  10985. }
  10986. children.append(*mapped.getClear());
  10987. }
  10988. return expr->clone(children);
  10989. }
  10990. //------------------------------------------------------------------------------------------------
  10991. void HqlCppTranslator::processUserAggregateTransform(IHqlExpression * expr, IHqlExpression * transform, OwnedHqlExpr & firstTransform, OwnedHqlExpr & nextTransform)
  10992. {
  10993. if (isKnownTransform(transform))
  10994. {
  10995. OwnedHqlExpr right = createSelector(no_right, expr, querySelSeq(expr));
  10996. OwnedHqlExpr nullRow = createRow(no_newrow, createRow(no_null, LINK(expr->queryRecord())));
  10997. firstTransform.setown(replaceSelector(transform, right, nullRow));
  10998. firstTransform.setown(foldHqlExpression(firstTransform));
  10999. }
  11000. else
  11001. firstTransform.set(transform);
  11002. if (isKnownTransform(transform))
  11003. {
  11004. {
  11005. NextTransformCreator builder(expr);
  11006. nextTransform.setown(builder.transform(transform));
  11007. }
  11008. }
  11009. else
  11010. nextTransform.set(transform);
  11011. }
  11012. IHqlExpression * HqlCppTranslator::getUserAggregateMergeTransform(IHqlExpression * expr, bool & requiresOrderedMerge)
  11013. {
  11014. IHqlExpression * mergeTransform = queryPropertyChild(expr, mergeAtom, 0);
  11015. if (mergeTransform)
  11016. {
  11017. requiresOrderedMerge = true;
  11018. return LINK(mergeTransform);
  11019. }
  11020. IHqlExpression * transform = expr->queryChild(2);
  11021. if (!isKnownTransform(transform))
  11022. return NULL;
  11023. MergeTransformCreator builder(expr);
  11024. OwnedHqlExpr ret = builder.transform(transform);
  11025. requiresOrderedMerge = builder.isOrdered();
  11026. return ret.getClear();
  11027. }
  11028. void HqlCppTranslator::doBuildUserMergeAggregateFunc(BuildCtx & ctx, IHqlExpression * expr, IHqlExpression * mergeTransform)
  11029. {
  11030. IHqlExpression * dataset = expr->queryChild(0);
  11031. IHqlExpression * tgtRecord = expr->queryChild(1);
  11032. IHqlExpression * transform = expr->queryChild(2);
  11033. IHqlExpression * selSeq = querySelSeq(expr);
  11034. if (!mergeTransform)
  11035. throwError(HQLERR_AggregateNeedMergeTransform);
  11036. IHqlExpression * rowsid = expr->queryProperty(_rowsid_Atom);
  11037. BuildCtx funcctx(ctx);
  11038. funcctx.addQuotedCompound("virtual size32_t mergeAggregate(ARowBuilder & upRight1, const void * _right2)");
  11039. funcctx.addQuoted("unsigned char * right2 = (unsigned char *) _right2;");
  11040. BoundRow * rightCursor = bindTableCursor(funcctx, expr, "upRight1.row()", no_right, selSeq);
  11041. BoundRow * leftCursor = bindTableCursor(funcctx, expr, "right2", no_left, selSeq);
  11042. BoundRow * selfCursor = bindSelf(funcctx, expr, "upRight1");
  11043. //Ugly!! RIGHT:2 is represented as ROWS(RIGHT)[2]. Change it to no_left since it will make like cleaner
  11044. //If we fail to replace all references then fall back to the worse case generation
  11045. OwnedHqlExpr rows = createDataset(no_rows, LINK(rightCursor->querySelector()), LINK(rowsid));
  11046. OwnedHqlExpr complexRight1 = createRow(no_selectnth, LINK(rows), createConstant(1));
  11047. OwnedHqlExpr complexRight2 = createRow(no_selectnth, LINK(rows), createConstant(2));
  11048. OwnedHqlExpr mappedTransform1 = removeAnnotations(mergeTransform, rows);
  11049. OwnedHqlExpr mappedTransform2 = replaceExpression(mappedTransform1, complexRight1, rightCursor->querySelector());
  11050. OwnedHqlExpr mappedTransform3 = replaceExpression(mappedTransform2, complexRight2, leftCursor->querySelector());
  11051. if (containsExpression(mappedTransform3, rows))
  11052. {
  11053. //Worse case fallback behaviour. Shouldn't occur in practice.
  11054. //If ROWS() really is needed due to some weird expression then we need to clone the target/RIGHT1 row because
  11055. //any modification may cause it to relocate and so invalidate the pointer passed in.
  11056. IHqlExpression * leftBound = leftCursor->queryBound();
  11057. ITypeInfo * rowType = leftBound->queryType();
  11058. CHqlBoundExpr boundRow1;
  11059. buildTempExpr(funcctx, rightCursor->querySelector(), boundRow1);
  11060. OwnedHqlExpr rows = createVariable("rows", makeArrayType(LINK(rowType), 2));
  11061. OwnedHqlExpr initializer = createValue(no_list, makeSetType(LINK(rowType)), LINK(boundRow1.expr), LINK(leftBound));
  11062. funcctx.addDeclare(rows, initializer);
  11063. bindRows(funcctx, no_right, selSeq, rowsid, expr, "2", "rows", options.mainRowsAreLinkCounted);
  11064. }
  11065. doUpdateTransform(funcctx, mappedTransform3, selfCursor, rightCursor, true);
  11066. buildReturnRecordSize(funcctx, selfCursor);
  11067. }
  11068. void HqlCppTranslator::doBuildUserAggregateFuncs(BuildCtx & ctx, IHqlExpression * expr, bool & requiresOrderedMerge)
  11069. {
  11070. IHqlExpression * dataset = expr->queryChild(0);
  11071. IHqlExpression * tgtRecord = expr->queryChild(1);
  11072. IHqlExpression * transform = expr->queryChild(2);
  11073. IHqlExpression * selSeq = querySelSeq(expr);
  11074. LinkedHqlExpr firstTransform;
  11075. LinkedHqlExpr nextTransform;
  11076. processUserAggregateTransform(expr, transform, firstTransform, nextTransform);
  11077. {
  11078. BuildCtx funcctx(ctx);
  11079. funcctx.addQuotedCompound("virtual size32_t processFirst(ARowBuilder & crSelf, const void * _src)");
  11080. ensureRowAllocated(funcctx, "crSelf");
  11081. funcctx.addQuoted("unsigned char * src = (unsigned char *) _src;");
  11082. BoundRow * selfRow = bindSelf(funcctx, expr, "crSelf");
  11083. bindTableCursor(funcctx, dataset, "src", options.mainRowsAreLinkCounted, no_left, selSeq);
  11084. doBuildUserAggregateProcessTransform(funcctx, selfRow, expr, firstTransform, queryBoolExpr(false));
  11085. buildReturnRecordSize(funcctx, selfRow);
  11086. }
  11087. {
  11088. BuildCtx funcctx(ctx);
  11089. funcctx.addQuotedCompound("virtual size32_t processNext(ARowBuilder & crSelf, const void * _src)");
  11090. ensureRowAllocated(funcctx, "crSelf");
  11091. funcctx.addQuoted("unsigned char * src = (unsigned char *) _src;");
  11092. BoundRow * leftCursor = bindTableCursor(funcctx, dataset, "src", options.mainRowsAreLinkCounted, no_left, selSeq);
  11093. BoundRow * selfCursor = bindSelf(funcctx, expr, "crSelf");
  11094. BoundRow * rightCursor = bindTableCursor(funcctx, expr, "crSelf.row()", no_right, querySelSeq(expr));
  11095. doBuildUserAggregateProcessTransform(funcctx, selfCursor, expr, nextTransform, queryBoolExpr(true));
  11096. buildReturnRecordSize(funcctx, selfCursor);
  11097. }
  11098. if (targetThor() && !isGrouped(dataset) && !expr->hasProperty(localAtom))
  11099. {
  11100. OwnedHqlExpr mergeTransform = getUserAggregateMergeTransform(expr, requiresOrderedMerge);
  11101. doBuildUserMergeAggregateFunc(ctx, expr, mergeTransform);
  11102. }
  11103. }
  11104. //--------------------------------------------------------------------------------------
  11105. void getMappedFields(HqlExprArray & aggregateFields, IHqlExpression * transform, HqlExprArray & recordFields, IHqlExpression * newSelector)
  11106. {
  11107. unsigned numFields = transform->numChildren();
  11108. OwnedHqlExpr self = getSelf(transform);
  11109. ForEachItemIn(idx, recordFields)
  11110. {
  11111. IHqlExpression & cur = recordFields.item(idx);
  11112. unsigned fieldIdx;
  11113. for (fieldIdx = 0; fieldIdx < numFields; fieldIdx++)
  11114. {
  11115. IHqlExpression * curAssign = transform->queryChild(fieldIdx);
  11116. IHqlExpression * rhs = curAssign->queryChild(1);
  11117. if (rhs->getOperator() == no_activerow)
  11118. rhs = rhs->queryChild(0);
  11119. if (rhs == &cur)
  11120. {
  11121. aggregateFields.append(*replaceSelector(curAssign->queryChild(0), self, newSelector));
  11122. break;
  11123. }
  11124. }
  11125. if (fieldIdx == numFields)
  11126. {
  11127. StringBuffer s;
  11128. getExprECL(&cur, s);
  11129. throwError1(HQLERR_AggregateMissingGroupingFields, s.str());
  11130. }
  11131. }
  11132. }
  11133. ABoundActivity * HqlCppTranslator::doBuildActivityAggregate(BuildCtx & ctx, IHqlExpression * expr)
  11134. {
  11135. IHqlExpression * dataset = expr->queryChild(0);
  11136. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  11137. node_operator op = expr->getOperator();
  11138. IHqlExpression * transform = expr->queryChild(2);
  11139. LinkedHqlExpr grouping = queryRealChild(expr, 3);
  11140. bool passThrough = (op == no_throughaggregate);
  11141. if (passThrough)
  11142. grouping.clear();
  11143. unsigned tableType = dataset->queryType()->getTypeCode();
  11144. const char *activity;
  11145. ThorActivityKind kind = TAKaggregate;
  11146. node_operator specialOp = no_none;
  11147. if (passThrough)
  11148. {
  11149. activity = "ThroughAggregate";
  11150. kind = TAKthroughaggregate;
  11151. }
  11152. else if (grouping)
  11153. {
  11154. activity = "HashAggregate";
  11155. kind = TAKhashaggregate;
  11156. }
  11157. else if (targetThor() && (tableType == type_groupedtable))
  11158. activity = "GroupAggregate";
  11159. else
  11160. {
  11161. activity = "Aggregate";
  11162. specialOp = querySimpleAggregate(expr, false, false);
  11163. if (specialOp == no_existsgroup)
  11164. {
  11165. kind = TAKexistsaggregate;
  11166. activity = "ExistsAggregate";
  11167. }
  11168. else if (specialOp == no_countgroup)
  11169. {
  11170. kind = TAKcountaggregate;
  11171. activity = "CountAggregate";
  11172. }
  11173. else
  11174. specialOp = no_none;
  11175. }
  11176. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, kind, expr, activity);
  11177. if (passThrough)
  11178. {
  11179. StringBuffer graphLabel;
  11180. graphLabel.append("Through Aggregate");
  11181. ForEachChild(idx, expr)
  11182. {
  11183. IHqlExpression * cur = expr->queryChild(idx);
  11184. if (cur->getOperator() == no_setresult || cur->getOperator() == no_extractresult)
  11185. {
  11186. IHqlExpression * sequence = queryPropertyChild(cur, sequenceAtom, 0);
  11187. IHqlExpression * name = queryPropertyChild(cur, namedAtom, 0);
  11188. getStoredDescription(graphLabel.append("\n"), sequence, name, false);
  11189. }
  11190. }
  11191. instance->graphLabel.set(graphLabel.str());
  11192. }
  11193. if (isKeyedCountAggregate(expr))
  11194. throwError(HQLERR_KeyedCountNonKeyable);
  11195. buildActivityFramework(instance);
  11196. buildInstancePrefix(instance);
  11197. StringBuffer flags;
  11198. if (specialOp == no_none)
  11199. {
  11200. doBuildAggregateClearFunc(instance->startctx, expr);
  11201. if (op == no_aggregate)
  11202. {
  11203. bool requiresOrderedMerge = false;
  11204. doBuildUserAggregateFuncs(instance->startctx, expr, requiresOrderedMerge);
  11205. if (requiresOrderedMerge)
  11206. flags.append("|TAForderedmerge");
  11207. }
  11208. else
  11209. {
  11210. doBuildAggregateFirstFunc(instance->startctx, expr);
  11211. doBuildAggregateNextFunc(instance->startctx, expr);
  11212. if (targetThor() && !isGrouped(dataset) && !expr->hasProperty(localAtom))
  11213. doBuildAggregateMergeFunc(instance->startctx, expr);
  11214. }
  11215. }
  11216. if (flags.length())
  11217. doBuildUnsignedFunction(instance->classctx, "getAggregateFlags", flags.str()+1);
  11218. if (grouping)
  11219. {
  11220. HqlExprArray recordFields, aggregateFields;
  11221. grouping->unwindList(recordFields, no_sortlist);
  11222. getMappedFields(aggregateFields, transform, recordFields, queryActiveTableSelector());
  11223. DatasetReference inputRef(dataset, (op == no_aggregate) ? no_left : no_none, querySelSeq(expr));
  11224. DatasetReference outRef(expr, no_activetable, NULL);
  11225. OwnedHqlExpr allRecordFields = createValueSafe(no_sortlist, makeSortListType(NULL), recordFields);
  11226. OwnedHqlExpr allAggregateFields = createValueSafe(no_sortlist, makeSortListType(NULL), aggregateFields);
  11227. buildCompareMember(instance->nestedctx, "CompareElements", allAggregateFields, outRef); // compare transformed elements
  11228. doCompareLeftRight(instance->nestedctx, "CompareRowElement", inputRef, outRef, recordFields, aggregateFields);
  11229. buildHashOfExprsClass(instance->nestedctx, "Hash", allRecordFields, inputRef, true);
  11230. buildHashOfExprsClass(instance->nestedctx, "HashElement", allAggregateFields, outRef, true);
  11231. }
  11232. if (passThrough)
  11233. {
  11234. BuildCtx sendctx(instance->startctx);
  11235. sendctx.addQuotedCompound("virtual void sendResult(const void * _self)");
  11236. sendctx.addQuoted("const unsigned char * self = (const unsigned char *)_self;");
  11237. OwnedHqlExpr resultDataset = createDataset(no_anon, LINK(expr->queryChild(1)), NULL);
  11238. bindTableCursor(sendctx, resultDataset, "self");
  11239. bindTableCursor(sendctx, dataset, "self");
  11240. ForEachChild(idx, expr)
  11241. {
  11242. IHqlExpression * cur = expr->queryChild(idx);
  11243. if (cur->getOperator() == no_setresult)
  11244. {
  11245. IHqlExpression * value = cur->queryChild(0);
  11246. IHqlExpression * sequence = queryPropertyChild(cur, sequenceAtom, 0);
  11247. IHqlExpression * name = queryPropertyChild(cur, namedAtom, 0);
  11248. buildSetResultInfo(sendctx, cur, value, NULL, false, false);
  11249. associateRemoteResult(*instance, sequence, name);
  11250. }
  11251. else if (cur->getOperator() == no_extractresult)
  11252. {
  11253. IHqlExpression * value = cur->queryChild(1);
  11254. IHqlExpression * sequence = queryPropertyChild(cur, sequenceAtom, 0);
  11255. IHqlExpression * name = queryPropertyChild(cur, namedAtom, 0);
  11256. buildSetResultInfo(sendctx, cur, value, NULL, false, false);
  11257. associateRemoteResult(*instance, sequence, name);
  11258. }
  11259. }
  11260. buildMetaMember(instance->classctx, resultDataset, "queryAggregateRecordSize");
  11261. }
  11262. buildInstanceSuffix(instance);
  11263. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  11264. return instance->getBoundActivity();
  11265. }
  11266. //---------------------------------------------------------------------------
  11267. ABoundActivity * HqlCppTranslator::doBuildActivityChildDataset(BuildCtx & ctx, IHqlExpression * expr)
  11268. {
  11269. if ((options.mainRowsAreLinkCounted && options.supportsLinkedChildRows && options.useLinkedRawIterator) || isGrouped(expr))
  11270. return doBuildActivityRawChildDataset(ctx, expr);
  11271. StringBuffer s;
  11272. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKchilditerator, expr, "ChildIterator");
  11273. buildActivityFramework(instance);
  11274. buildInstancePrefix(instance);
  11275. //Dummy implementation that just clears the record
  11276. OwnedHqlExpr value = LINK(expr);
  11277. switch (value->getOperator())
  11278. {
  11279. case no_alias:
  11280. case no_left:
  11281. case no_right:
  11282. case no_top:
  11283. case no_id2blob:
  11284. case no_rows:
  11285. case no_getgraphresult:
  11286. case no_activetable:
  11287. break;
  11288. case no_select:
  11289. if (!isNewSelector(expr))
  11290. break;
  11291. //fall through
  11292. default:
  11293. {
  11294. CHqlBoundExpr bound;
  11295. doBuildAliasValue(instance->onstartctx, value, bound);
  11296. value.setown(bound.getTranslatedExpr());
  11297. break;
  11298. }
  11299. }
  11300. Owned<IHqlCppDatasetCursor> iter = createDatasetSelector(instance->onstartctx, value);
  11301. iter->buildIterateMembers(instance->startctx, instance->onstartctx);
  11302. //virtual size32_t transform(ARowBuilder & crSelf) = 0;
  11303. BuildCtx transformCtx(instance->startctx);
  11304. transformCtx.addQuotedCompound("size32_t transform(ARowBuilder & crSelf)");
  11305. ensureRowAllocated(transformCtx, "crSelf");
  11306. BoundRow * selfCursor = bindSelf(transformCtx, expr, "crSelf");
  11307. OwnedHqlExpr active = ensureActiveRow(value);
  11308. buildAssign(transformCtx, selfCursor->querySelector(), active);
  11309. buildReturnRecordSize(transformCtx, selfCursor);
  11310. buildInstanceSuffix(instance);
  11311. return instance->getBoundActivity();
  11312. }
  11313. //---------------------------------------------------------------------------
  11314. ABoundActivity * HqlCppTranslator::doBuildActivityStreamedCall(BuildCtx & ctx, IHqlExpression * expr)
  11315. {
  11316. StringBuffer s;
  11317. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKstreamediterator, expr, "StreamedIterator");
  11318. buildActivityFramework(instance);
  11319. buildInstancePrefix(instance);
  11320. BuildCtx createctx(instance->startctx);
  11321. createctx.addQuotedCompound("virtual IRowStream * createInput()");
  11322. CHqlBoundExpr bound;
  11323. doBuildExprCall(createctx, expr, bound);
  11324. createctx.addReturn(bound.expr);
  11325. buildInstanceSuffix(instance);
  11326. return instance->getBoundActivity();
  11327. }
  11328. //---------------------------------------------------------------------------
  11329. ABoundActivity * HqlCppTranslator::doBuildActivityLinkedRawChildDataset(BuildCtx & ctx, IHqlExpression * expr)
  11330. {
  11331. StringBuffer s;
  11332. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKlinkedrawiterator, expr, "LinkedRawIterator");
  11333. buildActivityFramework(instance);
  11334. buildInstancePrefix(instance);
  11335. OwnedHqlExpr value = expr->isDatarow() ? createDatasetFromRow(LINK(expr)) : LINK(expr);
  11336. BuildCtx * declarectx;
  11337. BuildCtx * callctx;
  11338. instance->evalContext->getInvariantMemberContext(NULL, &declarectx, &callctx, false, true); // possibly should sometimes generate in onCreate(), if can evaluate in parent
  11339. CHqlBoundTarget invariantDs;
  11340. buildTempExpr(*callctx, *declarectx, invariantDs, value, FormatLinkedDataset, false);
  11341. CHqlBoundExpr boundDs;
  11342. boundDs.setFromTarget(invariantDs);
  11343. CHqlBoundTarget boundActiveIndex;
  11344. OwnedHqlExpr zero = getSizetConstant(0);
  11345. buildTempExpr(*callctx, *declarectx, boundActiveIndex, zero, FormatNatural, false);
  11346. //virtual byte * next() = 0;
  11347. BuildCtx funcctx(instance->startctx);
  11348. funcctx.addQuotedCompound("virtual byte * next()");
  11349. OwnedHqlExpr count = getBoundCount(boundDs);
  11350. OwnedHqlExpr test = createValue(no_lt, makeBoolType(), LINK(boundActiveIndex.expr), LINK(count));
  11351. BuildCtx subctx(funcctx);
  11352. subctx.addFilter(test);
  11353. OwnedHqlExpr ret = createValue(no_index, expr->getType(), LINK(boundDs.expr), createValue(no_postinc, LINK(sizetType), LINK(boundActiveIndex.expr)));
  11354. subctx.addReturn(ret);
  11355. funcctx.addReturn(queryQuotedNullExpr());
  11356. buildInstanceSuffix(instance);
  11357. return instance->getBoundActivity();
  11358. }
  11359. ABoundActivity * HqlCppTranslator::doBuildActivityRawChildDataset(BuildCtx & ctx, IHqlExpression * expr)
  11360. {
  11361. //If it is possible to create a linked child rows we need to use a linked raw child dataset iterator -
  11362. //otherwise the dataset may not be in the right format (and I can't work out how to efficiently force it otherwise.)
  11363. //If main rows are link counted then it is always going to be as good or better to generate a link counted raw iterator.
  11364. if (options.implicitLinkedChildRows || options.tempDatasetsUseLinkedRows)
  11365. return doBuildActivityLinkedRawChildDataset(ctx, expr);
  11366. StringBuffer s;
  11367. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKrawiterator, expr, "RawIterator");
  11368. buildActivityFramework(instance);
  11369. buildInstancePrefix(instance);
  11370. OwnedHqlExpr value = LINK(expr);
  11371. switch (value->getOperator())
  11372. {
  11373. case no_alias:
  11374. case no_left:
  11375. case no_right:
  11376. case no_id2blob:
  11377. case no_rows:
  11378. break;
  11379. case no_select:
  11380. if (!isNewSelector(expr))
  11381. break;
  11382. //fall through
  11383. default:
  11384. {
  11385. CHqlBoundExpr bound;
  11386. doBuildAliasValue(instance->onstartctx, value, bound);
  11387. value.setown(bound.getTranslatedExpr());
  11388. break;
  11389. }
  11390. }
  11391. //virtual void queryDataset(size_t & len, const void * & data)
  11392. BuildCtx funcctx(instance->startctx);
  11393. funcctx.addQuotedCompound("virtual void queryDataset(size32_t & len, const void * & data)");
  11394. CHqlBoundExpr boundValue;
  11395. buildExpr(funcctx, value, boundValue);
  11396. OwnedHqlExpr len = getBoundLength(boundValue);
  11397. generateExprCpp(s.clear().append("len = "), len).append(";");
  11398. funcctx.addQuoted(s);
  11399. OwnedHqlExpr addr = getPointer(boundValue.expr);
  11400. generateExprCpp(s.clear().append("data = "), addr).append(";");
  11401. funcctx.addQuoted(s);
  11402. buildInstanceSuffix(instance);
  11403. return instance->getBoundActivity();
  11404. }
  11405. //---------------------------------------------------------------------------
  11406. static void gatherDedupCompareExpr(HqlExprArray & equalities, HqlExprArray & comparisons, HqlExprArray & conds, IHqlExpression * left, IHqlExpression * right, IHqlExpression * dataset)
  11407. {
  11408. ForEachItemIn(idx, conds)
  11409. {
  11410. IHqlExpression * cond = &conds.item(idx);
  11411. if (cond->getOperator() == no_and)
  11412. {
  11413. HqlExprArray expanded;
  11414. cond->unwindList(expanded, no_and);
  11415. gatherDedupCompareExpr(equalities, comparisons, expanded, left, right, dataset);
  11416. }
  11417. else if (containsSelector(cond, left) || containsSelector(cond, right))
  11418. comparisons.append(*LINK(cond));
  11419. else if (!cond->isConstant())
  11420. equalities.append(*LINK(cond));
  11421. }
  11422. }
  11423. void optimizeGroupOrder(HqlExprArray & optimized, IHqlExpression * dataset, HqlExprArray & exprs)
  11424. {
  11425. RecordSelectIterator iter(dataset->queryRecord(), dataset->queryNormalizedSelector());
  11426. ForEach(iter)
  11427. {
  11428. IHqlExpression * select = iter.query();
  11429. unsigned match = exprs.find(*select);
  11430. if (match != NotFound)
  11431. {
  11432. optimized.append(*LINK(select));
  11433. //Remove this item, and all other matches (unusual, but if it does occur it is wasteful.)
  11434. do
  11435. {
  11436. exprs.remove(match);
  11437. match = exprs.find(*select);
  11438. } while (match != NotFound);
  11439. if (exprs.empty())
  11440. break;
  11441. }
  11442. }
  11443. }
  11444. IHqlExpression * createOrderFromCompareArray(HqlExprArray & exprs, IHqlExpression * dataset, IHqlExpression * left, IHqlExpression * right)
  11445. {
  11446. OwnedHqlExpr equalExpr = createValue(no_sortlist, makeSortListType(NULL), exprs);
  11447. OwnedHqlExpr lhs = replaceSelector(equalExpr, dataset, left);
  11448. OwnedHqlExpr rhs = replaceSelector(equalExpr, dataset, right);
  11449. return createValue(no_order, LINK(signedType), lhs.getClear(), rhs.getClear());
  11450. }
  11451. void HqlCppTranslator::buildDedupFilterFunction(BuildCtx & ctx, HqlExprArray & equalities, HqlExprArray & conds, IHqlExpression * dataset, IHqlExpression * selSeq)
  11452. {
  11453. HqlExprArray comparisons;
  11454. HqlExprArray allEqualities;
  11455. //MORE: The equalities really shouldn't be here - the activities should do them via the primaryCompare
  11456. appendArray(allEqualities, equalities);
  11457. OwnedHqlExpr left = createSelector(no_left, dataset, selSeq);
  11458. OwnedHqlExpr right = createSelector(no_right, dataset, selSeq);
  11459. gatherDedupCompareExpr(allEqualities, comparisons, conds, left, right, dataset);
  11460. BuildCtx funcctx(ctx);
  11461. IHqlStmt * functionStmt = funcctx.addQuotedCompound("virtual bool matches(const void * _left, const void * _right)");
  11462. funcctx.addQuoted("const unsigned char * left = (const unsigned char *) _left;");
  11463. funcctx.addQuoted("const unsigned char * right = (const unsigned char *) _right;");
  11464. BuildCtx filterctx(funcctx);
  11465. BoundRow * lRow = bindTableCursor(filterctx, dataset, "left", no_left, selSeq);
  11466. BoundRow * rRow = bindTableCursor(filterctx, dataset, "right", no_right, selSeq);
  11467. //convert any LEFT.x = RIGHT.y to an equality
  11468. unsigned numComparisons = comparisons.ordinality();
  11469. for (unsigned i1=0; i1 < numComparisons; )
  11470. {
  11471. IHqlExpression * cur = &comparisons.item(i1);
  11472. if (cur->getOperator() == no_eq)
  11473. {
  11474. IHqlExpression * lhs = cur->queryChild(0);
  11475. OwnedHqlExpr newLhs = replaceSelector(lhs, lRow->querySelector(), rRow->querySelector());
  11476. if (newLhs == cur->queryChild(1))
  11477. {
  11478. allEqualities.append(*replaceSelector(lhs, lRow->querySelector(), dataset));
  11479. comparisons.remove(i1);
  11480. numComparisons--;
  11481. }
  11482. //could check for right.x = left.x, but pretty unlikely.
  11483. else
  11484. i1++;
  11485. }
  11486. else
  11487. i1++;
  11488. }
  11489. ForEachItemIn(i, comparisons)
  11490. {
  11491. IHqlExpression * cur = &comparisons.item(i);
  11492. //if no equalities to follow, generate a return for the last non-equality
  11493. if (allEqualities.empty() && (i+1 == numComparisons))
  11494. {
  11495. buildReturn(filterctx, cur);
  11496. }
  11497. else
  11498. {
  11499. OwnedHqlExpr inverse = getInverse(cur);
  11500. buildFilteredReturn(filterctx, inverse, queryBoolExpr(false));
  11501. }
  11502. }
  11503. if (allEqualities.ordinality())
  11504. {
  11505. HqlExprArray optimized;
  11506. //Even better... sort the equality list by the field order...
  11507. if (options.optimizeGrouping && (allEqualities.ordinality() > 1))
  11508. optimizeGroupOrder(optimized, dataset, allEqualities);
  11509. appendArray(optimized, allEqualities);
  11510. OwnedHqlExpr order = createOrderFromCompareArray(optimized, dataset, lRow->querySelector(), rRow->querySelector());
  11511. doBuildReturnCompare(filterctx, order, no_eq, true);
  11512. }
  11513. else if (comparisons.ordinality() == 0)
  11514. functionStmt->setIncluded(false); // Use the implementation in the base class
  11515. }
  11516. void HqlCppTranslator::buildDedupSerializeFunction(BuildCtx & ctx, const char * funcName, IHqlExpression * srcDataset, IHqlExpression * tgtDataset, HqlExprArray & srcValues, HqlExprArray & tgtValues, IHqlExpression * selSeq)
  11517. {
  11518. StringBuffer s;
  11519. BuildCtx r2kctx(ctx);
  11520. s.append("virtual unsigned ").append(funcName).append("(ARowBuilder & crSelf, const void * _src)");
  11521. r2kctx.addQuotedCompound(s);
  11522. ensureRowAllocated(r2kctx, "crSelf");
  11523. r2kctx.addQuoted("const unsigned char * src = (const unsigned char *) _src;");
  11524. BoundRow * tgtCursor = bindSelf(ctx, tgtDataset, "crSelf");
  11525. BoundRow * srcCursor = bindTableCursor(ctx, srcDataset, "src", no_left, selSeq);
  11526. ForEachItemIn(idx2, srcValues)
  11527. {
  11528. Owned<IHqlExpression> self = tgtCursor->bindToRow(&tgtValues.item(idx2), queryActiveTableSelector());
  11529. Owned<IHqlExpression> left = srcCursor->bindToRow(&srcValues.item(idx2), srcDataset);
  11530. buildAssign(r2kctx, self, left);
  11531. }
  11532. buildReturnRecordSize(r2kctx, tgtCursor);
  11533. }
  11534. ABoundActivity * HqlCppTranslator::doBuildActivityDedup(BuildCtx & ctx, IHqlExpression * expr)
  11535. {
  11536. StringBuffer s;
  11537. DedupInfoExtractor info(expr);
  11538. IHqlExpression * dataset = expr->queryChild(0);
  11539. IHqlExpression * selSeq = querySelSeq(expr);
  11540. bool isGrouped = (dataset->queryType()->queryGroupInfo() != NULL);
  11541. bool isLocal = isLocalActivity(expr);
  11542. bool useHash = expr->hasProperty(hashAtom);
  11543. if (targetThor() && !isGrouped && !isLocal)
  11544. {
  11545. //Should really be done via an attribute on the dedup.
  11546. if (info.compareAllRows && !info.conds.ordinality())
  11547. useHash = true;
  11548. }
  11549. if (useHash && info.conds.ordinality())
  11550. throwError(HQLERR_GlobalDedupFuzzy);
  11551. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  11552. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, useHash ? TAKhashdedup : TAKdedup, expr, useHash ? "HashDedup" : "Dedup");
  11553. buildActivityFramework(instance);
  11554. buildInstancePrefix(instance);
  11555. if (!useHash)
  11556. {
  11557. if (info.compareAllRows)
  11558. doBuildBoolFunction(instance->classctx, "compareAll", info.compareAllRows);
  11559. if (!info.keepLeft)
  11560. doBuildBoolFunction(instance->classctx, "keepLeft", info.keepLeft);
  11561. if (!matchesConstantValue(info.numToKeep, 1))
  11562. doBuildUnsignedFunction(instance->startctx, "numToKeep", info.numToKeep);
  11563. //MORE: If input is grouped (pretty likely), then no need to include fields in the filter function that are already included.
  11564. if (instance->isGrouped)
  11565. {
  11566. HqlExprArray normalizedEqualities;
  11567. ForEachItemIn(i1, info.equalities)
  11568. normalizedEqualities.append(*replaceSelector(info.equalities.item(i1).queryBody(), dataset, queryActiveTableSelector()));
  11569. IHqlExpression * grouping = static_cast<IHqlExpression *>(dataset->queryType()->queryGroupInfo());
  11570. ForEachChild(i, grouping)
  11571. {
  11572. IHqlExpression * curGroup = grouping->queryChild(i);
  11573. unsigned match = normalizedEqualities.find(*curGroup);
  11574. if (match != NotFound)
  11575. {
  11576. normalizedEqualities.remove(match);
  11577. info.equalities.remove(match);
  11578. }
  11579. }
  11580. }
  11581. HqlExprArray noEqualities;
  11582. HqlExprArray * equalities = &info.equalities;
  11583. if (info.compareAllRows)
  11584. {
  11585. if (info.equalities.ordinality() && !instance->isGrouped)
  11586. {
  11587. OwnedHqlExpr order = createValue(no_sortlist, makeSortListType(NULL), info.equalities);
  11588. buildCompareMember(instance->nestedctx, "ComparePrimary", order, DatasetReference(dataset));
  11589. if (!targetThor())
  11590. equalities = &noEqualities;
  11591. }
  11592. }
  11593. buildDedupFilterFunction(instance->startctx, *equalities, info.conds, dataset, selSeq);
  11594. }
  11595. else
  11596. {
  11597. if (info.equalities.ordinality() == 0)
  11598. throwError(HQLERR_GlobalDedupNoEquality);
  11599. if (!matchesConstantValue(info.numToKeep, 1))
  11600. throwError1(HQLERR_HashDedupNotSupportX, "KEEP");
  11601. if (!info.keepLeft)
  11602. throwError1(HQLERR_HashDedupNotSupportX, "RIGHT");
  11603. OwnedHqlExpr order = createValueSafe(no_sortlist, makeSortListType(NULL), info.equalities);
  11604. buildCompareMember(instance->nestedctx, "Compare", order, DatasetReference(dataset));
  11605. buildHashOfExprsClass(instance->nestedctx, "Hash", order, DatasetReference(dataset), true);
  11606. HqlExprArray fields, selects;
  11607. ForEachItemIn(idx, info.equalities)
  11608. {
  11609. IHqlExpression & cur = info.equalities.item(idx);
  11610. IHqlExpression * field;
  11611. if ((cur.getOperator() == no_select) && (cur.queryChild(0) == dataset->queryNormalizedSelector()))
  11612. field = LINK(cur.queryChild(1));
  11613. else
  11614. {
  11615. StringBuffer name;
  11616. name.append("_expression_").append(idx);
  11617. field = createField(createIdentifierAtom(name.str()), cur.getType(), NULL);
  11618. }
  11619. fields.append(*field);
  11620. selects.append(*createSelectExpr(getActiveTableSelector(), LINK(field)));
  11621. }
  11622. OwnedHqlExpr keyDataset = createDataset(no_anon, createRecordInheritMaxLength(fields, dataset));
  11623. //virtual IOutputMetaData * queryKeySize()
  11624. buildMetaMember(instance->classctx, keyDataset, "queryKeySize");
  11625. //virtual unsigned recordToKey(void * _key, const void * _record)
  11626. buildDedupSerializeFunction(instance->startctx, "recordToKey", dataset, keyDataset, info.equalities, selects, selSeq);
  11627. //virtual ICompare * queryKeyCompare()
  11628. bool reuseCompare = false;
  11629. OwnedHqlExpr keyOrder = createValueSafe(no_sortlist, makeSortListType(NULL), selects);
  11630. if (recordTypesMatch(dataset, keyDataset))
  11631. {
  11632. OwnedHqlExpr globalOrder = replaceSelector(order, dataset, queryActiveTableSelector());
  11633. if (keyOrder == globalOrder)
  11634. reuseCompare = true;
  11635. }
  11636. if (!reuseCompare)
  11637. buildCompareMember(instance->nestedctx, "KeyCompare", keyOrder, DatasetReference(keyDataset, no_activetable, NULL));
  11638. else
  11639. instance->nestedctx.addQuoted("virtual ICompare * queryKeyCompare() { return &Compare; }");
  11640. }
  11641. buildInstanceSuffix(instance);
  11642. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  11643. return instance->getBoundActivity();
  11644. }
  11645. //---------------------------------------------------------------------------
  11646. // no_distribute
  11647. ABoundActivity * HqlCppTranslator::doBuildActivityDistribute(BuildCtx & ctx, IHqlExpression * expr)
  11648. {
  11649. if (!targetThor())
  11650. return buildCachedActivity(ctx, expr->queryChild(0));
  11651. StringBuffer s;
  11652. IHqlExpression * dataset = expr->queryChild(0);
  11653. if (isUngroup(dataset))
  11654. dataset = dataset->queryChild(0);
  11655. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  11656. IHqlExpression * cond = expr->queryChild(1);
  11657. IHqlExpression * mergeOrder = queryPropertyChild(expr, mergeAtom, 0);
  11658. if (cond->getOperator() == no_sortpartition)
  11659. {
  11660. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKpartition, expr, "Merge");
  11661. buildActivityFramework(instance);
  11662. buildInstancePrefix(instance);
  11663. HqlExprArray sorts;
  11664. unwindChildren(sorts, cond);
  11665. OwnedHqlExpr sortOrder = createValueSafe(no_sortlist, makeSortListType(NULL), sorts);
  11666. instance->startctx.addQuoted("virtual ICompare * queryCompare() { return &compare; }");
  11667. DatasetReference dsRef(dataset);
  11668. buildCompareClass(instance->nestedctx, "compare", sortOrder, dsRef);
  11669. if (!instance->isLocal)
  11670. generateSerializeKey(instance->nestedctx, no_none, dsRef, sorts, true, true, false);
  11671. buildInstanceSuffix(instance);
  11672. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  11673. return instance->getBoundActivity();
  11674. }
  11675. else
  11676. {
  11677. //Now generate the instance definition...
  11678. ThorActivityKind tak = (expr->getOperator() == no_distribute) ?
  11679. (mergeOrder ? TAKhashdistributemerge : TAKhashdistribute) : TAKdistributed;
  11680. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, tak, expr, "HashDistribute");
  11681. if (expr->hasProperty(skewAtom))
  11682. instance->graphLabel.set("Skew Distribute");
  11683. buildActivityFramework(instance);
  11684. buildInstancePrefix(instance);
  11685. if (!expr->hasProperty(skewAtom))
  11686. buildHashClass(instance->nestedctx, "Hash", cond, DatasetReference(dataset));
  11687. doBuildBoolFunction(instance->classctx, "isPulled", expr->hasProperty(pulledAtom));
  11688. buildSkewThresholdMembers(instance->classctx, expr);
  11689. if (mergeOrder)
  11690. buildCompareMember(instance->nestedctx, "MergeCompare", mergeOrder, DatasetReference(dataset));
  11691. buildInstanceSuffix(instance);
  11692. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  11693. return instance->getBoundActivity();
  11694. }
  11695. }
  11696. //---------------------------------------------------------------------------
  11697. // no_rollup
  11698. ABoundActivity * HqlCppTranslator::doBuildActivityRollup(BuildCtx & ctx, IHqlExpression * expr)
  11699. {
  11700. StringBuffer s;
  11701. IHqlExpression * dataset = expr->queryChild(0);
  11702. IHqlExpression * cond = expr->queryChild(1);
  11703. IHqlExpression * transform = expr->queryChild(2);
  11704. IHqlExpression * selSeq = querySelSeq(expr);
  11705. HqlExprArray equalities;
  11706. HqlExprArray conds;
  11707. if (cond->getOperator() == no_sortlist)
  11708. unwindChildren(conds, cond);
  11709. else
  11710. conds.append(*LINK(cond));
  11711. //build child table....
  11712. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  11713. //Now generate the instance definition...
  11714. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKrollup, expr, "Rollup");
  11715. buildActivityFramework(instance);
  11716. buildInstancePrefix(instance);
  11717. buildDedupFilterFunction(instance->startctx, equalities, conds, dataset, selSeq);
  11718. buildRollupTransformFunction(instance->startctx, dataset, transform, selSeq);
  11719. buildInstanceSuffix(instance);
  11720. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  11721. return instance->getBoundActivity();
  11722. }
  11723. //---------------------------------------------------------------------------
  11724. // no_denormalize
  11725. ABoundActivity * HqlCppTranslator::doBuildActivityDenormalize(BuildCtx & ctx, IHqlExpression * expr)
  11726. {
  11727. if (isKeyedJoin(expr))
  11728. return doBuildActivityKeyedJoinOrDenormalize(ctx, expr);
  11729. return doBuildActivityJoinOrDenormalize(ctx, expr);
  11730. }
  11731. //---------------------------------------------------------------------------
  11732. ABoundActivity * HqlCppTranslator::doBuildActivityFirstN(BuildCtx & ctx, IHqlExpression * expr)
  11733. {
  11734. IHqlExpression * dataset = expr->queryChild(0);
  11735. IHqlExpression * limit = expr->queryChild(1);
  11736. //choosen(x,ALL) does nothing, but is a way of getting round the implicit limitation.
  11737. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  11738. if (isChooseNAllLimit(limit) && !expr->queryChild(2))
  11739. return boundDataset.getClear();
  11740. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKfirstn, expr, "FirstN");
  11741. buildActivityFramework(instance);
  11742. buildInstancePrefix(instance);
  11743. BuildCtx funcctx(instance->startctx);
  11744. funcctx.addQuotedCompound("virtual __int64 getLimit()");
  11745. OwnedHqlExpr newLimit = ensurePositiveOrZeroInt64(limit);
  11746. if (options.spotCSE)
  11747. newLimit.setown(spotScalarCSE(newLimit));
  11748. buildReturn(funcctx, newLimit);
  11749. if (queryRealChild(expr, 2))
  11750. {
  11751. OwnedHqlExpr adjusted = adjustValue(expr->queryChild(2), -1);
  11752. OwnedHqlExpr newAdjusted = ensurePositiveOrZeroInt64(adjusted);
  11753. BuildCtx funcctx(instance->startctx);
  11754. funcctx.addQuotedCompound("virtual __int64 numToSkip()");
  11755. buildReturn(funcctx, newAdjusted);
  11756. }
  11757. if (expr->hasProperty(groupedAtom))
  11758. doBuildBoolFunction(instance->classctx, "preserveGrouping", true);
  11759. buildInstanceSuffix(instance);
  11760. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  11761. return instance->getBoundActivity();
  11762. }
  11763. //---------------------------------------------------------------------------
  11764. ABoundActivity * HqlCppTranslator::doBuildActivityChooseSetsEx(BuildCtx & ctx, IHqlExpression * expr)
  11765. {
  11766. IHqlExpression * dataset = expr->queryChild(0);
  11767. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  11768. bool isEnth = expr->hasProperty(enthAtom);
  11769. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, isEnth ? TAKchoosesetsenth : TAKchoosesetslast, expr, "ChooseSetsEx");
  11770. buildActivityFramework(instance);
  11771. buildInstancePrefix(instance);
  11772. unsigned numArgs = expr->numChildren();
  11773. bool keepExtras = false;
  11774. unsigned numConditions = 0;
  11775. for (unsigned idx1 = 1; idx1 < numArgs; idx1++)
  11776. {
  11777. switch (expr->queryChild(idx1)->getOperator())
  11778. {
  11779. case no_mapto:
  11780. numConditions++;
  11781. break;
  11782. case no_attr:
  11783. break;
  11784. default:
  11785. keepExtras = true;
  11786. }
  11787. }
  11788. unsigned numCategories = numConditions + (keepExtras ? 1 : 0);
  11789. //virtual unsigned getNumSets()
  11790. BuildCtx funcctx(instance->classctx);
  11791. funcctx.addQuotedCompound("virtual unsigned getNumSets()");
  11792. OwnedHqlExpr numExpr = createConstant((int)numCategories);
  11793. buildReturn(funcctx, numExpr, unsignedType);
  11794. //virtual unsigned getRecordCategory(const void * _self) = 0;
  11795. BuildCtx categoryctx(instance->startctx);
  11796. categoryctx.addQuotedCompound("virtual unsigned getCategory(const void * _self)");
  11797. categoryctx.addQuoted("const unsigned char * self = (const unsigned char *)_self;");
  11798. bindTableCursor(categoryctx, dataset, "self");
  11799. HqlExprArray args;
  11800. for (unsigned idx3 = 1; idx3 <= numConditions; idx3++)
  11801. {
  11802. IHqlExpression * cur = expr->queryChild(idx3);
  11803. args.append(*createValue(no_mapto, makeVoidType(), LINK(cur->queryChild(0)), createConstant(unsignedType->castFrom(false, (__int64)idx3))));
  11804. }
  11805. if (keepExtras)
  11806. args.append(*createConstant((__int64)numConditions+1));
  11807. else
  11808. args.append(*createConstant((__int64)0));
  11809. OwnedHqlExpr map = createValue(no_map, LINK(unsignedType), args);
  11810. buildReturn(categoryctx, map);
  11811. //virtual void getLimits(unsigned * counts) = 0;
  11812. BuildCtx limitctx(instance->startctx);
  11813. StringBuffer s;
  11814. limitctx.addQuotedCompound("virtual void getLimits(__int64 * counts)");
  11815. for (unsigned idx2 = 1; idx2 <= numCategories; idx2++)
  11816. {
  11817. IHqlExpression * cur = expr->queryChild(idx2);
  11818. s.clear().append("counts[").append(idx2-1).append("]");
  11819. OwnedHqlExpr target = createVariable(s.str(), LINK(defaultIntegralType));
  11820. switch (cur->getOperator())
  11821. {
  11822. case no_mapto:
  11823. buildAssignToTemp(limitctx, target, cur->queryChild(1));
  11824. break;
  11825. default:
  11826. buildAssignToTemp(limitctx, target, cur);
  11827. break;
  11828. }
  11829. }
  11830. buildInstanceSuffix(instance);
  11831. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  11832. return instance->getBoundActivity();
  11833. }
  11834. ABoundActivity * HqlCppTranslator::doBuildActivityChooseSets(BuildCtx & ctx, IHqlExpression * expr)
  11835. {
  11836. if (expr->hasProperty(enthAtom) || expr->hasProperty(lastAtom))
  11837. return doBuildActivityChooseSetsEx(ctx, expr);
  11838. IHqlExpression * dataset = expr->queryChild(0);
  11839. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  11840. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKchoosesets, expr, "ChooseSets");
  11841. buildActivityFramework(instance);
  11842. buildInstancePrefix(instance);
  11843. unsigned numArgs = expr->numChildren();
  11844. bool allowSpill = !expr->hasProperty(exclusiveAtom);
  11845. bool keepExtras = false;
  11846. unsigned numConditions = 0;
  11847. for (unsigned idx1 = 1; idx1 < numArgs; idx1++)
  11848. {
  11849. switch (expr->queryChild(idx1)->getOperator())
  11850. {
  11851. case no_mapto:
  11852. numConditions++;
  11853. break;
  11854. case no_attr:
  11855. case no_attr_expr:
  11856. case no_attr_link:
  11857. break;
  11858. default:
  11859. keepExtras = true;
  11860. }
  11861. }
  11862. unsigned numCategories = numConditions + (keepExtras ? 1 : 0);
  11863. BuildCtx funcctx(instance->classctx);
  11864. funcctx.addQuotedCompound("virtual unsigned getNumSets()");
  11865. OwnedHqlExpr numExpr = createConstant((int)numCategories);
  11866. buildReturn(funcctx, numExpr, unsignedType);
  11867. StringBuffer s;
  11868. BuildCtx limitctx(instance->classctx);
  11869. instance->startctx.addQuoted("unsigned * counts;");
  11870. instance->startctx.addQuoted("unsigned numFull;");
  11871. limitctx.addQuotedCompound("virtual bool setCounts(unsigned * data)");
  11872. limitctx.addQuoted("counts = data;");
  11873. limitctx.addQuoted("numFull = 0;");
  11874. OwnedHqlExpr tally = createVariable("counts", makeIntType(4, false));
  11875. BuildCtx validctx(instance->startctx);
  11876. validctx.addQuotedCompound("virtual unsigned getRecordAction(const void * _self)");
  11877. validctx.addQuoted("const unsigned char * self = (const unsigned char *)_self;");
  11878. bindTableCursor(validctx, dataset, "self");
  11879. OwnedHqlExpr one = createConstant((int)1);
  11880. for (unsigned idx = 0; idx < numCategories; idx++)
  11881. {
  11882. OwnedHqlExpr indexExpr = getSizetConstant(idx);
  11883. OwnedHqlExpr bucketExpr = createValue(no_index, LINK(unsignedType), tally.getLink(), indexExpr.getLink());
  11884. OwnedHqlExpr transBucketExpr = createTranslated(bucketExpr);
  11885. IHqlExpression * arg = expr->queryChild(idx+1);
  11886. IHqlExpression * count = (arg->getOperator() == no_mapto ? arg->queryChild(1) : arg);
  11887. OwnedHqlExpr cond2 = createBoolExpr(no_lt, transBucketExpr.getLink(), ensureExprType(count, unsignedType));
  11888. OwnedHqlExpr condDone = createBoolExpr(no_eq, transBucketExpr.getLink(), ensureExprType(count, unsignedType));
  11889. BuildCtx condctx(validctx);
  11890. if (arg->getOperator() == no_mapto)
  11891. {
  11892. IHqlExpression * filter = arg->queryChild(0);
  11893. if (allowSpill)
  11894. {
  11895. OwnedHqlExpr cond = createBoolExpr(no_and, LINK(filter), cond2.getLink());
  11896. CHqlBoundExpr bound;
  11897. buildExpr(condctx, cond, bound);
  11898. condctx.addFilter(bound.expr);
  11899. }
  11900. else
  11901. {
  11902. buildFilter(condctx, filter);
  11903. BuildCtx failctx(condctx);
  11904. buildFilter(condctx, cond2);
  11905. buildReturn(failctx, queryZero());
  11906. }
  11907. }
  11908. else
  11909. {
  11910. buildFilter(condctx, cond2);
  11911. }
  11912. OwnedHqlExpr inc = createValue(no_postinc, bucketExpr.getLink());
  11913. condctx.addExpr(inc);
  11914. BuildCtx doneCtx(condctx);
  11915. buildFilter(doneCtx, condDone);
  11916. doneCtx.addQuoted(s.clear().append("if (++numFull == ").append(numCategories).append(") return 2;"));
  11917. buildReturn(condctx, one, unsignedType);
  11918. BuildCtx limitCondCtx(limitctx);
  11919. buildFilter(limitCondCtx, condDone);
  11920. limitCondCtx.addQuoted("numFull++;");
  11921. }
  11922. buildReturn(validctx, queryZero());
  11923. limitctx.addQuoted(s.clear().append("return numFull == ").append(numCategories).append(";"));
  11924. buildInstanceSuffix(instance);
  11925. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  11926. return instance->getBoundActivity();
  11927. }
  11928. //---------------------------------------------------------------------------
  11929. ABoundActivity * HqlCppTranslator::doBuildActivityNormalizeGroup(BuildCtx & ctx, IHqlExpression * expr)
  11930. {
  11931. throwUnexpected();
  11932. IHqlExpression * dataset = expr->queryChild(0);
  11933. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  11934. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKnormalize, expr,"Normalize");
  11935. buildActivityFramework(instance);
  11936. buildInstancePrefix(instance);
  11937. buildInstanceSuffix(instance);
  11938. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  11939. return instance->getBoundActivity();
  11940. }
  11941. //---------------------------------------------------------------------------
  11942. ABoundActivity * HqlCppTranslator::doBuildActivityNormalize(BuildCtx & ctx, IHqlExpression * expr)
  11943. {
  11944. IHqlExpression * numRows = expr->queryChild(1);
  11945. if (!numRows->queryType()->isScalar())
  11946. return doBuildActivityNormalizeChild(ctx, expr);
  11947. IHqlExpression * dataset = expr->queryChild(0);
  11948. IHqlExpression * transform = expr->queryChild(2);
  11949. IHqlExpression * selSeq = querySelSeq(expr);
  11950. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  11951. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKnormalize, expr,"Normalize");
  11952. buildActivityFramework(instance);
  11953. buildInstancePrefix(instance);
  11954. BuildCtx funcctx(instance->startctx);
  11955. funcctx.addQuotedCompound("virtual unsigned numExpandedRows(const void * _left)");
  11956. funcctx.addQuoted("unsigned char * left = (unsigned char *) _left;");
  11957. bindTableCursor(funcctx, dataset, "left", no_left, selSeq);
  11958. bindTableCursor(funcctx, dataset, "left");
  11959. buildReturn(funcctx, numRows);
  11960. BuildCtx transformctx(instance->startctx);
  11961. transformctx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf, const void * _left, unsigned counter)");
  11962. ensureRowAllocated(transformctx, "crSelf");
  11963. IHqlExpression * counter = queryPropertyChild(expr, _countProject_Atom, 0);
  11964. associateCounter(transformctx, counter, "counter");
  11965. buildTransformBody(transformctx, transform, dataset, NULL, instance->dataset, selSeq);
  11966. buildInstanceSuffix(instance);
  11967. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  11968. return instance->getBoundActivity();
  11969. }
  11970. //---------------------------------------------------------------------------
  11971. ABoundActivity * HqlCppTranslator::doBuildActivityNormalizeChild(BuildCtx & ctx, IHqlExpression * expr)
  11972. {
  11973. IHqlExpression * dataset = expr->queryChild(0);
  11974. LinkedHqlExpr childDataset = expr->queryChild(1);
  11975. IHqlExpression * transform = expr->queryChild(2);
  11976. IHqlExpression * selSeq = querySelSeq(expr);
  11977. if (options.useLinkedNormalize)
  11978. {
  11979. if (transformReturnsSide(expr, no_right, 1))
  11980. return doBuildActivityNormalizeLinkedChild(ctx, expr);
  11981. }
  11982. StringBuffer s;
  11983. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  11984. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKnormalizechild, expr,"NormalizeChild");
  11985. buildActivityFramework(instance);
  11986. buildInstancePrefix(instance);
  11987. //Generate queryChildRecordSize();
  11988. buildMetaMember(instance->classctx, childDataset, "queryChildRecordSize");
  11989. // INormalizeChildIterator * queryIterator();
  11990. {
  11991. bool outOfLine = options.tempDatasetsUseLinkedRows;
  11992. if (childDataset->isDatarow())
  11993. childDataset.setown(createDatasetFromRow(childDataset.getClear()));
  11994. if (childDataset->getOperator() == no_select)
  11995. outOfLine = isArrayRowset(childDataset->queryType());
  11996. if (hasLinkCountedModifier(childDataset))
  11997. outOfLine = true;
  11998. BuildCtx iterclassctx(instance->nestedctx);
  11999. StringBuffer memberName, className;
  12000. getUniqueId(memberName.append("m"));
  12001. getMemberClassName(className, memberName.str());
  12002. ExpressionFormat format;
  12003. if (outOfLine)
  12004. {
  12005. beginNestedClass(iterclassctx, memberName, "CNormalizeLinkedChildIterator");
  12006. format = FormatLinkedDataset;
  12007. }
  12008. else
  12009. {
  12010. beginNestedClass(iterclassctx, memberName, "CNormalizeChildIterator");
  12011. format = FormatBlockedDataset;
  12012. MetaInstance childmeta(*this, childDataset);
  12013. buildMetaInfo(childmeta);
  12014. s.clear().append(className).append("() : CNormalizeChildIterator(").append(childmeta.queryInstanceObject()).append(") {}");
  12015. iterclassctx.addQuoted(s);
  12016. }
  12017. bool callFromActivity = false;
  12018. BuildCtx activityinitctx(instance->startctx);
  12019. BuildCtx funcctx(iterclassctx);
  12020. funcctx.addQuotedCompound("virtual void init(const void * _left)");
  12021. funcctx.addQuoted("const byte * left = (const byte *)_left;");
  12022. CHqlBoundExpr bound;
  12023. if (childDataset->getOperator() != no_select)
  12024. {
  12025. //Ugly......
  12026. //If this is a complex expression, then ensure the temporary variable is a member of the activity class, and
  12027. //evaluate it in the function defined inside the activity (so the member variables don't need mangling)
  12028. funcctx.addQuoted("activity->init(left);");
  12029. BuildCtx * declarectx = NULL;
  12030. instance->evalContext->getInvariantMemberContext(NULL, &declarectx, NULL, false, true);
  12031. queryEvalContext(iterclassctx)->ensureHelpersExist();
  12032. assertex(declarectx);
  12033. activityinitctx.addQuotedCompound("void init(const byte * left)");
  12034. bindTableCursor(activityinitctx, dataset, "left", no_left, selSeq);
  12035. CHqlBoundTarget tempTarget;
  12036. buildTempExpr(activityinitctx, *declarectx, tempTarget, childDataset, format, false);
  12037. bound.setFromTarget(tempTarget);
  12038. callFromActivity = true;
  12039. }
  12040. else
  12041. {
  12042. bindTableCursor(funcctx, dataset, "left", no_left, selSeq);
  12043. buildDataset(funcctx, childDataset, bound, format);
  12044. }
  12045. s.clear();
  12046. if (callFromActivity)
  12047. s.append(memberName).append(".");
  12048. s.append("setDataset(");
  12049. if (outOfLine)
  12050. {
  12051. generateExprCpp(s, bound.count).append(",");
  12052. generateExprCpp(s, bound.expr).append(");");
  12053. }
  12054. else
  12055. {
  12056. OwnedHqlExpr length = getBoundLength(bound);
  12057. generateExprCpp(s, length).append(",");
  12058. generateExprCpp(s, bound.expr).append(");");
  12059. }
  12060. if (callFromActivity)
  12061. activityinitctx.addQuoted(s);
  12062. else
  12063. funcctx.addQuoted(s);
  12064. endNestedClass();
  12065. s.clear().append("INormalizeChildIterator * queryIterator() { return &").append(memberName).append("; }");
  12066. instance->startctx.addQuoted(s);
  12067. }
  12068. BuildCtx transformctx(instance->startctx);
  12069. transformctx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf, const void * _left, const void * _right, unsigned counter)");
  12070. ensureRowAllocated(transformctx, "crSelf");
  12071. IHqlExpression * counter = queryPropertyChild(expr, _countProject_Atom, 0);
  12072. associateCounter(transformctx, counter, "counter");
  12073. buildTransformBody(transformctx, transform, dataset, childDataset, instance->dataset, selSeq);
  12074. buildInstanceSuffix(instance);
  12075. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  12076. return instance->getBoundActivity();
  12077. }
  12078. //---------------------------------------------------------------------------
  12079. ABoundActivity * HqlCppTranslator::doBuildActivityNormalizeLinkedChild(BuildCtx & ctx, IHqlExpression * expr)
  12080. {
  12081. OwnedHqlExpr dataset;
  12082. OwnedHqlExpr childDataset;
  12083. node_operator selectorOp = no_none;
  12084. IHqlExpression * selSeq = NULL;
  12085. switch (expr->getOperator())
  12086. {
  12087. case no_select:
  12088. {
  12089. bool isNew;
  12090. dataset.set(querySelectorDataset(expr, isNew));
  12091. //Ensure input is a dataset so cleanly bound as a cursor later
  12092. if (dataset->isDatarow())
  12093. dataset.setown(createDatasetFromRow(dataset.getClear()));
  12094. assertex(isNew);
  12095. OwnedHqlExpr active = ensureActiveRow(dataset);
  12096. childDataset.setown(replaceSelectorDataset(expr, active));
  12097. break;
  12098. }
  12099. case no_normalize:
  12100. {
  12101. dataset.set(expr->queryChild(0));
  12102. childDataset.set(expr->queryChild(1));
  12103. selectorOp = no_left;
  12104. selSeq = querySelSeq(expr);
  12105. break;
  12106. }
  12107. default:
  12108. throwUnexpectedOp(expr->getOperator());
  12109. }
  12110. StringBuffer s;
  12111. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  12112. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKnormalizelinkedchild, expr,"NormalizeLinkedChild");
  12113. buildActivityFramework(instance);
  12114. buildInstancePrefix(instance);
  12115. OwnedHqlExpr value = childDataset->isDatarow() ? createDatasetFromRow(LINK(childDataset)) : LINK(childDataset);
  12116. BuildCtx * declarectx = NULL;
  12117. instance->evalContext->getInvariantMemberContext(NULL, &declarectx, NULL, false, true);
  12118. assertex(declarectx);
  12119. CHqlBoundTarget childTarget;
  12120. CHqlBoundExpr boundChild;
  12121. CHqlBoundTarget boundActive;
  12122. createTempFor(*declarectx, value->queryType(), childTarget, typemod_none, FormatLinkedDataset);
  12123. boundChild.setFromTarget(childTarget);
  12124. assertex(boundChild.count);
  12125. OwnedHqlExpr test;
  12126. OwnedHqlExpr ret;
  12127. //virtual byte * first(const void * parentRecord) = 0;
  12128. {
  12129. BuildCtx firstctx(instance->startctx);
  12130. firstctx.addQuotedCompound("virtual byte * first(const void * parentRecord)");
  12131. firstctx.addQuoted("const byte * left = (const byte *)parentRecord;");
  12132. bindTableCursor(firstctx, dataset, "left", selectorOp, selSeq);
  12133. buildDatasetAssign(firstctx, childTarget, value);
  12134. OwnedHqlExpr zero = getSizetConstant(0);
  12135. buildTempExpr(firstctx, *declarectx, boundActive, zero, FormatNatural, false);
  12136. test.setown(createValue(no_lt, makeBoolType(), LINK(boundActive.expr), LINK(boundChild.count)));
  12137. ret.setown(createValue(no_index, expr->getType(), LINK(boundChild.expr), LINK(boundActive.expr)));
  12138. BuildCtx subctx(firstctx);
  12139. subctx.addFilter(test);
  12140. subctx.addReturn(ret);
  12141. firstctx.addReturn(queryQuotedNullExpr());
  12142. }
  12143. {
  12144. //virtual byte * next() = 0;
  12145. BuildCtx nextctx(instance->startctx);
  12146. nextctx.addQuotedCompound("virtual byte * next()");
  12147. OwnedHqlExpr inc = createValue(no_postinc, LINK(sizetType), LINK(boundActive.expr));
  12148. nextctx.addExpr(inc);
  12149. BuildCtx subctx(nextctx);
  12150. subctx.addFilter(test);
  12151. subctx.addReturn(ret);
  12152. nextctx.addReturn(queryQuotedNullExpr());
  12153. }
  12154. buildInstanceSuffix(instance);
  12155. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  12156. return instance->getBoundActivity();
  12157. }
  12158. //---------------------------------------------------------------------------
  12159. ABoundActivity * HqlCppTranslator::doBuildActivitySelectNew(BuildCtx & ctx, IHqlExpression * expr)
  12160. {
  12161. if (options.useLinkedNormalize)
  12162. return doBuildActivityNormalizeLinkedChild(ctx, expr);
  12163. IHqlExpression * ds = expr->queryChild(0);
  12164. // if (!isContextDependent(ds) && !containsActiveDataset(ds))
  12165. // return doBuildActivityChildDataset(ctx, expr);
  12166. if (canEvaluateInline(&ctx, ds))
  12167. return doBuildActivityChildDataset(ctx, expr);
  12168. //Convert the ds.x to normalize(ds, left.x, transform(right));
  12169. IHqlExpression * field = expr->queryChild(1);
  12170. OwnedHqlExpr selSeq = createDummySelectorSequence();
  12171. OwnedHqlExpr left = createSelector(no_left, ds, selSeq);
  12172. OwnedHqlExpr selector = createSelectExpr(LINK(left), LINK(field));
  12173. OwnedHqlExpr transformedExpr;
  12174. if (field->isDatarow())
  12175. {
  12176. OwnedHqlExpr transform = createTransformFromRow(selector);
  12177. transformedExpr.setown(createDatasetF(no_hqlproject, LINK(ds), LINK(transform), LINK(selSeq), NULL)); // MORE: UID
  12178. }
  12179. else
  12180. {
  12181. OwnedHqlExpr right = createSelector(no_right, expr, selSeq);
  12182. OwnedHqlExpr transform = createTransformFromRow(right);
  12183. transformedExpr.setown(createDatasetF(no_normalize, LINK(ds), LINK(selector), LINK(transform), LINK(selSeq), NULL)); // MORE: UID
  12184. }
  12185. return buildActivity(ctx, transformedExpr, false);
  12186. }
  12187. //---------------------------------------------------------------------------
  12188. ABoundActivity * HqlCppTranslator::doBuildActivityPrefetchProject(BuildCtx & ctx, IHqlExpression * expr)
  12189. {
  12190. IHqlExpression * dataset = expr->queryChild(0);
  12191. IHqlExpression * transform = queryNewColumnProvider(expr);
  12192. IHqlExpression * counter = queryPropertyChild(expr, _countProject_Atom, 0);
  12193. IHqlExpression * selSeq = querySelSeq(expr);
  12194. IHqlExpression * prefetch = expr->queryProperty(prefetchAtom);
  12195. IHqlExpression * lookahead = queryPropertyChild(expr, prefetchAtom, 0);
  12196. IHqlExpression * record = expr->queryRecord();
  12197. #ifdef _DEBUG
  12198. assertex((counter != NULL) == transformContainsCounter(transform, counter));
  12199. #endif
  12200. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  12201. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, counter ? TAKprefetchcountproject : TAKprefetchproject, expr, "PrefetchProject");
  12202. buildActivityFramework(instance);
  12203. buildInstancePrefix(instance);
  12204. StringBuffer flags;
  12205. if (prefetch && prefetch->hasProperty(parallelAtom)) flags.append("|PPFparallel");
  12206. if (flags.length())
  12207. doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);
  12208. if (transformContainsSkip(transform))
  12209. doBuildBoolFunction(instance->classctx, "canFilter", true);
  12210. if (lookahead)
  12211. doBuildUnsignedFunction(instance->startctx, "getLookahead", lookahead);
  12212. //Similar code to project below. First generate the post processing function (which all aliases etc. will get generated into)
  12213. BuildCtx postctx(instance->startctx);
  12214. postctx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf, const void * _left, IEclGraphResults * results, unsigned __int64 _counter)");
  12215. ensureRowAllocated(postctx, "crSelf");
  12216. postctx.addQuoted("const unsigned char * left = (const unsigned char *) _left;");
  12217. if (expr->getOperator() == no_hqlproject)
  12218. bindTableCursor(postctx, dataset, "left", no_left, selSeq);
  12219. else
  12220. bindTableCursor(postctx, dataset, "left");
  12221. BoundRow * selfCursor = bindSelf(postctx, expr, "crSelf");
  12222. associateSkipReturnMarker(postctx, queryZero(), selfCursor);
  12223. if (counter)
  12224. associateCounter(postctx, counter, "counter");
  12225. //Now process the transform
  12226. HqlExprArray assigns;
  12227. //Introduce a scope to ensure that mapper and builder have the minimum lifetime.
  12228. {
  12229. //Possibly cleaner if this was implemented inside a class derived from TransformBuilder
  12230. TransformBuilder builder(*this, postctx, record, selfCursor, assigns);
  12231. filterExpandAssignments(postctx, &builder, assigns, transform);
  12232. builder.buildTransformChildren(postctx, record, selfCursor->querySelector());
  12233. if (builder.hasGraphPending())
  12234. {
  12235. //Generate the extract preparation function
  12236. BuildCtx prectx(instance->startctx);
  12237. IHqlStmt * preStmt = prectx.addQuotedCompound("virtual bool preTransform(rtlRowBuilder & builder, const void * _left, unsigned __int64 _counter)");
  12238. associateSkipReturnMarker(prectx, queryBoolExpr(false), NULL);
  12239. prectx.addQuoted("const unsigned char * left = (const unsigned char *) _left;");
  12240. if (expr->getOperator() == no_hqlproject)
  12241. bindTableCursor(prectx, dataset, "left", no_left, selSeq);
  12242. else
  12243. bindTableCursor(prectx, dataset, "left");
  12244. if (counter)
  12245. associateCounter(prectx, counter, "counter");
  12246. OwnedHqlExpr graph, results;
  12247. builder.generatePrefetch(prectx, &graph, &results);
  12248. prectx.addReturn(queryBoolExpr(true));
  12249. BuildCtx childctx(instance->startctx);
  12250. childctx.addQuotedCompound("virtual IThorChildGraph *queryChild()");
  12251. childctx.addReturn(graph);
  12252. //hack! I need to change the way results are bound so they aren't nesc. the same as the query name
  12253. StringBuffer s;
  12254. s.append("IEclGraphResults * ");
  12255. generateExprCpp(s, results);
  12256. s.append(" = results;");
  12257. postctx.addQuoted(s);
  12258. //Add an association for the results into the transform function.
  12259. postctx.associateExpr(results, results);
  12260. }
  12261. builder.flush(postctx);
  12262. }
  12263. buildReturnRecordSize(postctx, selfCursor);
  12264. buildInstanceSuffix(instance);
  12265. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  12266. return instance->getBoundActivity();
  12267. }
  12268. ABoundActivity * HqlCppTranslator::doBuildActivityProject(BuildCtx & ctx, IHqlExpression * expr)
  12269. {
  12270. if (expr->hasProperty(prefetchAtom) || options.usePrefetchForAllProjects)
  12271. return doBuildActivityPrefetchProject(ctx, expr);
  12272. const node_operator op = expr->getOperator();
  12273. IHqlExpression * dataset = expr->queryChild(0);
  12274. IHqlExpression * normalized = dataset->queryNormalizedSelector();
  12275. IHqlExpression * counter = queryPropertyChild(expr, _countProject_Atom, 0);
  12276. IHqlExpression * selSeq = querySelSeq(expr);
  12277. LinkedHqlExpr transform = queryNewColumnProvider(expr);
  12278. LinkedHqlExpr invariantCondition;
  12279. HqlExprArray filterConditions;
  12280. OwnedHqlExpr leftSelector = (op != no_newusertable) ? createSelector(no_left, normalized, selSeq) : NULL;
  12281. //Spot any filters preceding this project/user table, and fold them into the project
  12282. //It doesn't work with count projects though - because it would alter the counter passed to the transform.
  12283. //MORE: It would be possible to spot a skip condition for a count project which was input invariant, but thor
  12284. //would need a new implementation if the input wasn't grouped because the count would need to be global.
  12285. if (!counter)
  12286. {
  12287. bool done = false;
  12288. do
  12289. {
  12290. switch (dataset->getOperator())
  12291. {
  12292. case no_filter:
  12293. {
  12294. LinkedHqlExpr invariant;
  12295. OwnedHqlExpr cond = extractFilterConditions(invariant, dataset, normalized, false);
  12296. //A dataset invariant filter is only worth combining if the engine supports a filtered project operation.
  12297. if (!options.supportFilterProject && invariant)
  12298. {
  12299. done = true;
  12300. break;
  12301. }
  12302. //Don't merge the condition if it would create an ambiguity on LEFT - highly unlikely to occur in practice
  12303. if (leftSelector && cond)
  12304. {
  12305. if (containsSelectorAnywhere(cond, leftSelector))
  12306. {
  12307. done = true;
  12308. break;
  12309. }
  12310. cond.setown(replaceSelector(cond, normalized, leftSelector));
  12311. }
  12312. extendConditionOwn(invariantCondition, no_and, invariant.getClear());
  12313. if (cond)
  12314. cond->unwindList(filterConditions, no_and);
  12315. dataset = dataset->queryChild(0);
  12316. break;
  12317. }
  12318. case no_sorted:
  12319. dataset = dataset->queryChild(0);
  12320. break;
  12321. default:
  12322. done = true;
  12323. break;
  12324. }
  12325. } while (!done);
  12326. }
  12327. bool isFilterProject = invariantCondition != NULL;
  12328. bool containsCounter = expr->hasProperty(_countProject_Atom);
  12329. #ifdef _DEBUG
  12330. assertex(containsCounter == transformContainsCounter(transform, counter));
  12331. #endif
  12332. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  12333. Owned<ActivityInstance> instance = isFilterProject ? new ActivityInstance(*this, ctx, TAKfilterproject, expr, "FilterProject") :
  12334. containsCounter ? new ActivityInstance(*this, ctx, TAKcountproject, expr, "CountProject")
  12335. : new ActivityInstance(*this, ctx, TAKproject, expr, "Project");
  12336. if (filterConditions.ordinality())
  12337. {
  12338. if (isGroupedActivity(expr))
  12339. instance->graphLabel.set("Grouped Filtered Project");
  12340. else
  12341. instance->graphLabel.set("Filtered Project");
  12342. }
  12343. buildActivityFramework(instance);
  12344. buildInstancePrefix(instance);
  12345. BuildCtx funcctx(instance->startctx);
  12346. if (isFilterProject || containsCounter)
  12347. {
  12348. funcctx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf, const void * _left, unsigned __int64 counter)");
  12349. if (containsCounter)
  12350. associateCounter(funcctx, counter, "counter");
  12351. }
  12352. else
  12353. funcctx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf, const void * _left)");
  12354. ensureRowAllocated(funcctx, "crSelf");
  12355. if (filterConditions.ordinality())
  12356. {
  12357. HqlExprArray args;
  12358. ForEachItemIn(i, filterConditions)
  12359. {
  12360. OwnedHqlExpr test = createValue(no_skip, makeVoidType(), getInverse(&filterConditions.item(i)));
  12361. args.append(*LINK(test));
  12362. }
  12363. unwindChildren(args, transform);
  12364. transform.setown(transform->clone(args));
  12365. }
  12366. if (op == no_newusertable)
  12367. {
  12368. funcctx.addQuoted("const unsigned char * left = (const unsigned char *) _left;");
  12369. BoundRow * selfCursor = bindSelf(funcctx, expr, "crSelf");
  12370. associateSkipReturnMarker(funcctx, queryZero(), selfCursor);
  12371. bindTableCursor(funcctx, dataset, "left");
  12372. doTransform(funcctx, transform, selfCursor);
  12373. buildReturnRecordSize(funcctx, selfCursor);
  12374. }
  12375. else
  12376. buildTransformBody(funcctx, transform, dataset, NULL, instance->dataset, selSeq);
  12377. if (filterConditions.ordinality() || transformContainsSkip(transform))
  12378. doBuildBoolFunction(instance->classctx, "canFilter", true);
  12379. if (invariantCondition)
  12380. doBuildBoolFunction(instance->startctx, "canMatchAny", invariantCondition);
  12381. buildInstanceSuffix(instance);
  12382. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  12383. return instance->getBoundActivity();
  12384. }
  12385. //---------------------------------------------------------------------------
  12386. ABoundActivity * HqlCppTranslator::doBuildActivitySerialize(BuildCtx & ctx, IHqlExpression * expr)
  12387. {
  12388. IHqlExpression * dataset = expr->queryChild(0);
  12389. bool serialize = (expr->getOperator() == no_serialize);
  12390. assertex(serialize);//Needs changing once interface has changed.
  12391. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  12392. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKproject, expr, "Project");
  12393. buildActivityFramework(instance);
  12394. buildInstancePrefix(instance);
  12395. BuildCtx funcctx(instance->startctx);
  12396. funcctx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf, const void * _left)");
  12397. assertex(!options.limitMaxLength); //need more thought
  12398. // Bind left to "left" and right to RIGHT
  12399. BoundRow * leftCursor = bindTableCursor(funcctx, dataset, "_left");
  12400. BoundRow * selfCursor = bindSelf(funcctx, expr, "crSelf");
  12401. //MORE: I don't have any examples that trigger this code as far as I know...
  12402. _ATOM func = serialize ? rtlSerializeToBuilderAtom : rtlDeserializeToBuilderAtom;
  12403. _ATOM kind = serialize ? serializerAtom : deserializerAtom;
  12404. IHqlExpression * record = expr->queryRecord();
  12405. HqlExprArray args;
  12406. args.append(*createRowSerializer(ctx, record, kind));
  12407. args.append(*ensureActiveRow(dataset));
  12408. Owned<ITypeInfo> type = makeTransformType(record->getType());
  12409. OwnedHqlExpr call = bindFunctionCall(func, args, type);
  12410. doTransform(funcctx, call, selfCursor);
  12411. buildReturnRecordSize(funcctx, selfCursor);
  12412. buildInstanceSuffix(instance);
  12413. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  12414. return instance->getBoundActivity();
  12415. }
  12416. //---------------------------------------------------------------------------
  12417. ABoundActivity * HqlCppTranslator::doBuildActivityDefineSideEffect(BuildCtx & ctx, IHqlExpression * expr)
  12418. {
  12419. Owned<ABoundActivity> parentActivity = buildCachedActivity(ctx, expr->queryChild(0));
  12420. OwnedHqlExpr attr = createAttribute(_sideEffect_Atom, LINK(expr->queryProperty(_uid_Atom)));
  12421. OwnedHqlExpr unknown = createUnknown(no_attr, NULL, NULL, LINK(parentActivity));
  12422. activeGraphCtx->associateExpr(attr, unknown);
  12423. return parentActivity.getClear();
  12424. }
  12425. //---------------------------------------------------------------------------
  12426. ABoundActivity * HqlCppTranslator::doBuildActivityCallSideEffect(BuildCtx & ctx, IHqlExpression * expr)
  12427. {
  12428. OwnedHqlExpr attr = createAttribute(_sideEffect_Atom, LINK(expr->queryProperty(_uid_Atom)));
  12429. HqlExprAssociation * match = activeGraphCtx->queryMatchExpr(attr);
  12430. if (!match)
  12431. throwUnexpected();
  12432. ABoundActivity * activity = static_cast<ABoundActivity *>(match->queryExpr()->queryUnknownExtra());
  12433. return LINK(activity);
  12434. }
  12435. //---------------------------------------------------------------------------
  12436. ABoundActivity * HqlCppTranslator::doBuildActivityExecuteWhen(BuildCtx & ctx, IHqlExpression * expr)
  12437. {
  12438. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, expr->queryChild(0));
  12439. Owned<ABoundActivity> associatedActivity = buildCachedActivity(ctx, expr->queryChild(1));
  12440. if (!associatedActivity)
  12441. return boundDataset.getClear();
  12442. const char * label;
  12443. int when;
  12444. if (expr->hasProperty(successAtom))
  12445. {
  12446. label = "Success";
  12447. when = WhenSuccessId;
  12448. }
  12449. else if (expr->hasProperty(failureAtom))
  12450. {
  12451. label = "Failure";
  12452. when = WhenFailureId;
  12453. }
  12454. else if (expr->hasProperty(parallelAtom))
  12455. {
  12456. label = "Parallel";
  12457. when = WhenParallelId;
  12458. }
  12459. else
  12460. {
  12461. label = "Before";
  12462. when = WhenDefaultId;
  12463. }
  12464. bool useImplementationClass = options.minimizeActivityClasses;
  12465. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKwhen, expr, "WhenAction");
  12466. if (useImplementationClass)
  12467. instance->setImplementationClass(newWhenActionArgAtom);
  12468. buildActivityFramework(instance);
  12469. buildInstancePrefix(instance);
  12470. buildInstanceSuffix(instance);
  12471. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  12472. addDependency(ctx, associatedActivity, instance->queryBoundActivity(), dependencyAtom, label, when);
  12473. return instance->getBoundActivity();
  12474. }
  12475. //---------------------------------------------------------------------------
  12476. IHqlExpression * extractFilterConditions(HqlExprAttr & invariant, IHqlExpression * expr, IHqlExpression * dataset, bool spotCSE)
  12477. {
  12478. unsigned num = expr->numChildren();
  12479. assertex(num > 1);
  12480. HqlExprAttr cond = queryRealChild(expr, 1);
  12481. bool changed = false;
  12482. unsigned idx;
  12483. for (idx = 2; idx < num; idx++)
  12484. {
  12485. IHqlExpression * cur = expr->queryChild(idx);
  12486. if (!cur->isAttribute())
  12487. extendConditionOwn(cond, no_and, LINK(cur));
  12488. }
  12489. if (!cond)
  12490. return NULL;
  12491. if (spotCSE)
  12492. cond.setown(spotScalarCSE(cond));
  12493. HqlExprArray tests;
  12494. cond->unwindList(tests, no_and);
  12495. ForEachItemInRev(i, tests)
  12496. {
  12497. IHqlExpression & cur = tests.item(i);
  12498. if (!exprReferencesDataset(&cur, dataset))
  12499. {
  12500. changed = true;
  12501. if (!matchesBoolean(&cur, true))
  12502. invariant.setown(extendConditionOwn(no_and, LINK(&cur), invariant.getClear()));
  12503. tests.remove(i);
  12504. }
  12505. }
  12506. if (changed)
  12507. cond.setown(createBalanced(no_and, queryBoolType(), tests));
  12508. return cond.getClear();
  12509. }
  12510. ABoundActivity * HqlCppTranslator::doBuildActivityFilter(BuildCtx & ctx, IHqlExpression * expr)
  12511. {
  12512. IHqlExpression * dataset = expr->queryChild(0);
  12513. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  12514. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKfilter, expr,"Filter");
  12515. buildActivityFramework(instance);
  12516. buildInstancePrefix(instance);
  12517. HqlExprAttr invariant;
  12518. OwnedHqlExpr cond = extractFilterConditions(invariant, expr, dataset, options.spotCSE);
  12519. //Base class returns true, so only generate if no non-invariant conditions
  12520. if (cond)
  12521. {
  12522. BuildCtx funcctx(instance->startctx);
  12523. funcctx.addQuotedCompound("virtual bool isValid(const void * _self)");
  12524. funcctx.addQuoted("unsigned char * self = (unsigned char *) _self;");
  12525. bindTableCursor(funcctx, dataset, "self");
  12526. buildReturn(funcctx, cond);
  12527. }
  12528. if (invariant)
  12529. doBuildBoolFunction(instance->startctx, "canMatchAny", invariant);
  12530. buildInstanceSuffix(instance);
  12531. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  12532. return instance->getBoundActivity();
  12533. }
  12534. ABoundActivity * HqlCppTranslator::doBuildActivityFilterGroup(BuildCtx & ctx, IHqlExpression * expr)
  12535. {
  12536. IHqlExpression * dataset = expr->queryChild(0);
  12537. IHqlExpression * selSeq = querySelSeq(expr);
  12538. IHqlExpression * rowsid = expr->queryProperty(_rowsid_Atom);
  12539. if (targetThor() && !isGrouped(dataset))
  12540. throwError(HQLERR_ThorHavingMustBeGrouped);
  12541. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  12542. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKfiltergroup, expr,"FilterGroup");
  12543. buildActivityFramework(instance);
  12544. buildInstancePrefix(instance);
  12545. HqlExprAttr invariant;
  12546. OwnedHqlExpr left = createSelector(no_left, dataset, selSeq);
  12547. OwnedHqlExpr cond = extractFilterConditions(invariant, expr, left, options.spotCSE);
  12548. //Base class returns true, so only generate if no non-invariant conditions
  12549. if (cond)
  12550. {
  12551. BuildCtx funcctx(instance->startctx);
  12552. funcctx.addQuotedCompound("virtual bool isValid(unsigned numRows, const void * * _rows)");
  12553. funcctx.addQuoted("const unsigned char * left = (const unsigned char *) _rows[0];");
  12554. funcctx.addQuoted("unsigned char * * rows = (unsigned char * *) _rows;");
  12555. bindTableCursor(funcctx, dataset, "left", no_left, selSeq);
  12556. bindRows(funcctx, no_left, selSeq, rowsid, dataset, "numRows", "rows", options.mainRowsAreLinkCounted);
  12557. buildReturn(funcctx, cond);
  12558. }
  12559. if (invariant)
  12560. doBuildBoolFunction(instance->startctx, "canMatchAny", invariant);
  12561. buildInstanceSuffix(instance);
  12562. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  12563. return instance->getBoundActivity();
  12564. }
  12565. //---------------------------------------------------------------------------
  12566. ABoundActivity * HqlCppTranslator::doBuildActivityCombine(BuildCtx & ctx, IHqlExpression * expr)
  12567. {
  12568. //MORE: Need to expand nested combines so they have multiple inputs.
  12569. //But will need to assign aliases to the inputs + do a reasonable amount of processing.
  12570. IHqlExpression * left = expr->queryChild(0);
  12571. IHqlExpression * right = expr->queryChild(1);
  12572. IHqlExpression * transform = expr->queryChild(2);
  12573. IHqlExpression * selSeq = querySelSeq(expr);
  12574. if (targetThor() && !expr->hasProperty(localAtom))
  12575. ERRORAT(queryLocation(expr), HQLERR_ThorCombineOnlyLocal);
  12576. CIArray bound;
  12577. bound.append(*buildCachedActivity(ctx, left));
  12578. bound.append(*buildCachedActivity(ctx, right));
  12579. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKcombine, expr, "Combine");
  12580. buildActivityFramework(instance);
  12581. buildInstancePrefix(instance);
  12582. BuildCtx funcctx(instance->startctx);
  12583. funcctx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf, unsigned _num, const void * * _rows)");
  12584. if (transform->getOperator() != no_skip)
  12585. {
  12586. funcctx.addQuoted("const unsigned char * left = (const unsigned char *) _rows[0];");
  12587. funcctx.addQuoted("const unsigned char * right = (const unsigned char *) _rows[1];");
  12588. ensureRowAllocated(funcctx, "crSelf");
  12589. bindTableCursor(funcctx, left, "left", no_left, selSeq);
  12590. bindTableCursor(funcctx, right, "right", no_right, selSeq);
  12591. BoundRow * selfCursor = bindSelf(funcctx, expr, "crSelf");
  12592. associateSkipReturnMarker(funcctx, queryZero(), selfCursor);
  12593. doTransform(funcctx, transform, selfCursor);
  12594. buildReturnRecordSize(funcctx, selfCursor);
  12595. }
  12596. else
  12597. funcctx.addReturn(queryZero());
  12598. if (transformContainsSkip(transform))
  12599. doBuildBoolFunction(instance->classctx, "canFilter", true);
  12600. buildInstanceSuffix(instance);
  12601. ForEachItemIn(idx2, bound)
  12602. buildConnectInputOutput(ctx, instance, (ABoundActivity *)&bound.item(idx2), 0, idx2);
  12603. return instance->getBoundActivity();
  12604. }
  12605. //---------------------------------------------------------------------------
  12606. void HqlCppTranslator::bindRows(BuildCtx & ctx, node_operator side, IHqlExpression * selSeq, IHqlExpression * rowsid, IHqlExpression * dataset, const char * numText, const char * rowsText, bool rowsAreLinkCounted)
  12607. {
  12608. OwnedHqlExpr selector = createSelector(side, dataset, selSeq);
  12609. OwnedHqlExpr rowsExpr = createDataset(no_rows, LINK(selector), LINK(rowsid));
  12610. ITypeInfo * rowType = makeReferenceModifier(LINK(rowsExpr->queryType()->queryChildType()));
  12611. if (rowsAreLinkCounted)
  12612. rowType = makeAttributeModifier(rowType, getLinkCountedAttr());
  12613. //Rows may be link counted, but rows() is not a linkable rowset
  12614. OwnedITypeInfo rowsType = makeReferenceModifier(makeTableType(rowType, NULL, NULL, NULL));
  12615. rowsType.setown(makeOutOfLineModifier(LINK(rowsType)));
  12616. CHqlBoundExpr boundRows;
  12617. boundRows.count.setown(createQuoted(numText, LINK(unsignedType)));
  12618. boundRows.expr.setown(createQuoted(rowsText, LINK(rowsType)));
  12619. ctx.associateExpr(rowsExpr, boundRows);
  12620. }
  12621. //---------------------------------------------------------------------------
  12622. ABoundActivity * HqlCppTranslator::doBuildActivityCombineGroup(BuildCtx & ctx, IHqlExpression * expr)
  12623. {
  12624. //MORE: Need to expand nested combines so they have multiple inputs.
  12625. //But will need to assign aliases to the inputs + do a reasonable amount of processing.
  12626. IHqlExpression * left = expr->queryChild(0);
  12627. IHqlExpression * right = expr->queryChild(1);
  12628. IHqlExpression * selSeq = querySelSeq(expr);
  12629. IHqlExpression * transform = expr->queryChild(2);
  12630. IHqlExpression * rowsid = expr->queryProperty(_rowsid_Atom);
  12631. CIArray bound;
  12632. bound.append(*buildCachedActivity(ctx, left));
  12633. bound.append(*buildCachedActivity(ctx, right));
  12634. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKcombinegroup, expr, "CombineGroup");
  12635. buildActivityFramework(instance);
  12636. buildInstancePrefix(instance);
  12637. BuildCtx funcctx(instance->startctx);
  12638. funcctx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf, const void * _left, unsigned numRows, const void * * _rows)");
  12639. if (transform->getOperator() != no_skip)
  12640. {
  12641. funcctx.addQuoted("const unsigned char * left = (const unsigned char *)_left;");
  12642. funcctx.addQuoted("const unsigned char * right = (const unsigned char *) _rows[0];");
  12643. funcctx.addQuoted("unsigned char * * rows = (unsigned char * *) _rows;");
  12644. ensureRowAllocated(funcctx, "crSelf");
  12645. bindTableCursor(funcctx, left, "left", no_left, selSeq);
  12646. bindTableCursor(funcctx, right, "right", no_right, selSeq);
  12647. bindRows(funcctx, no_right, selSeq, rowsid, right, "numRows", "rows", options.mainRowsAreLinkCounted);
  12648. BoundRow * selfCursor = bindSelf(funcctx, expr, "crSelf");
  12649. associateSkipReturnMarker(funcctx, queryZero(), selfCursor);
  12650. doTransform(funcctx, transform, selfCursor);
  12651. buildReturnRecordSize(funcctx, selfCursor);
  12652. }
  12653. else
  12654. funcctx.addReturn(queryZero());
  12655. if (transformContainsSkip(transform))
  12656. doBuildBoolFunction(instance->classctx, "canFilter", true);
  12657. buildInstanceSuffix(instance);
  12658. ForEachItemIn(idx2, bound)
  12659. buildConnectInputOutput(ctx, instance, (ABoundActivity *)&bound.item(idx2), 0, idx2);
  12660. return instance->getBoundActivity();
  12661. }
  12662. //---------------------------------------------------------------------------
  12663. ABoundActivity * HqlCppTranslator::doBuildActivityRollupGroup(BuildCtx & ctx, IHqlExpression * expr)
  12664. {
  12665. //MORE: Need to expand nested combines so they have multiple inputs.
  12666. IHqlExpression * dataset = expr->queryChild(0);
  12667. IHqlExpression * transform = expr->queryChild(1);
  12668. IHqlExpression * selSeq = querySelSeq(expr);
  12669. IHqlExpression * rowsid = expr->queryProperty(_rowsid_Atom);
  12670. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  12671. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKrollupgroup, expr, "RollupGroup");
  12672. instance->graphLabel.set("Rollup Group"); // Grouped Rollup Group looks silly
  12673. buildActivityFramework(instance);
  12674. buildInstancePrefix(instance);
  12675. BuildCtx funcctx(instance->startctx);
  12676. funcctx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf, unsigned numRows, const void * * _rows)");
  12677. if (transform->getOperator() != no_skip)
  12678. {
  12679. funcctx.addQuoted("const unsigned char * left = (const unsigned char *) _rows[0];");
  12680. funcctx.addQuoted("unsigned char * * rows = (unsigned char * *) _rows;");
  12681. ensureRowAllocated(funcctx, "crSelf");
  12682. bindTableCursor(funcctx, dataset, "left", no_left, selSeq);
  12683. bindRows(funcctx, no_left, selSeq, rowsid, dataset, "numRows", "rows", options.mainRowsAreLinkCounted);
  12684. BoundRow * selfCursor = bindSelf(funcctx, expr, "crSelf");
  12685. associateSkipReturnMarker(funcctx, queryZero(), selfCursor);
  12686. doTransform(funcctx, transform, selfCursor);
  12687. buildReturnRecordSize(funcctx, selfCursor);
  12688. }
  12689. else
  12690. funcctx.addReturn(queryZero());
  12691. if (transformContainsSkip(transform))
  12692. doBuildBoolFunction(instance->classctx, "canFilter", true);
  12693. buildInstanceSuffix(instance);
  12694. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  12695. return instance->getBoundActivity();
  12696. }
  12697. //---------------------------------------------------------------------------
  12698. ABoundActivity * HqlCppTranslator::doBuildActivityAssert(BuildCtx & ctx, IHqlExpression * expr)
  12699. {
  12700. HqlExprArray args;
  12701. expr->unwindList(args, expr->getOperator());
  12702. IHqlExpression * dataset = &args.item(0);
  12703. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  12704. //MORE: Change this when ThroughApply activities are supported in engines.
  12705. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKfilter, expr,"Filter");
  12706. buildActivityFramework(instance);
  12707. buildInstancePrefix(instance);
  12708. unsigned num = args.ordinality();
  12709. BuildCtx funcctx(instance->startctx);
  12710. funcctx.addQuotedCompound("virtual bool isValid(const void * _self)");
  12711. funcctx.addQuoted("unsigned char * self = (unsigned char *) _self;");
  12712. bindTableCursor(funcctx, dataset, "self");
  12713. for (unsigned i=1; i < num; i++)
  12714. {
  12715. IHqlExpression & cur = args.item(i);
  12716. if (!cur.isAttribute())
  12717. buildStmt(funcctx, &cur);
  12718. }
  12719. funcctx.addReturn(queryBoolExpr(true));
  12720. buildInstanceSuffix(instance);
  12721. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  12722. return instance->getBoundActivity();
  12723. }
  12724. //---------------------------------------------------------------------------
  12725. void HqlCppTranslator::buildLimitHelpers(BuildCtx & ctx, IHqlExpression * rowLimit, IHqlExpression * failAction, bool isSkip, IHqlExpression * filename, unique_id_t id)
  12726. {
  12727. doBuildUnsigned64Function(ctx, "getRowLimit", rowLimit);
  12728. if (isZero(rowLimit))
  12729. WARNING(HQLWRN_LimitIsZero);
  12730. if (!isSkip)
  12731. {
  12732. LinkedHqlExpr fail = failAction;
  12733. if (!fail || fail->isAttribute())
  12734. {
  12735. if (!id)
  12736. id = queryCurrentActivityId(ctx);
  12737. fail.setown(createFailAction("Limit exceeded", rowLimit, filename, id));
  12738. }
  12739. BuildCtx ctx2(ctx);
  12740. ctx2.addQuotedCompound("virtual void onLimitExceeded()");
  12741. buildStmt(ctx2, fail);
  12742. }
  12743. }
  12744. void HqlCppTranslator::buildLimitHelpers(BuildCtx & ctx, IHqlExpression * expr, IHqlExpression * filename, unique_id_t id)
  12745. {
  12746. buildLimitHelpers(ctx, expr->queryChild(1), queryRealChild(expr, 2), expr->hasProperty(skipAtom), filename, id);
  12747. IHqlExpression * transform = queryPropertyChild(expr, onFailAtom, 0);
  12748. if (transform)
  12749. {
  12750. BuildCtx transformctx(ctx);
  12751. transformctx.addQuotedCompound("virtual size32_t transformOnLimitExceeded(ARowBuilder & crSelf)");
  12752. ensureRowAllocated(transformctx, "crSelf");
  12753. buildTransformBody(transformctx, transform, NULL, NULL, expr, NULL);
  12754. }
  12755. }
  12756. ABoundActivity * HqlCppTranslator::doBuildActivityLimit(BuildCtx & ctx, IHqlExpression * expr)
  12757. {
  12758. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, expr->queryChild(0));
  12759. IHqlExpression * transform = queryPropertyChild(expr, onFailAtom, 0);
  12760. ThorActivityKind kind = TAKlimit;
  12761. const char * helper = "Limit";
  12762. if (transform)
  12763. {
  12764. kind = TAKcreaterowlimit;
  12765. helper = "CreateRowLimit";
  12766. }
  12767. else if (expr->hasProperty(skipAtom))
  12768. kind = TAKskiplimit;
  12769. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, kind, expr, helper);
  12770. buildActivityFramework(instance);
  12771. buildInstancePrefix(instance);
  12772. buildLimitHelpers(instance->startctx, expr, NULL, instance->activityId);
  12773. buildInstanceSuffix(instance);
  12774. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  12775. return instance->getBoundActivity();
  12776. }
  12777. ABoundActivity * HqlCppTranslator::doBuildActivityCatch(BuildCtx & ctx, IHqlExpression * expr)
  12778. {
  12779. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, expr->queryChild(0));
  12780. IHqlExpression * arg1 = queryRealChild(expr, 1);
  12781. IHqlExpression * filter = NULL;
  12782. IHqlExpression * action = NULL;
  12783. if (arg1 && arg1->isBoolean())
  12784. {
  12785. filter = arg1;
  12786. action = queryRealChild(expr, 2);
  12787. }
  12788. else
  12789. action = arg1;
  12790. IHqlExpression * transform = queryPropertyChild(expr, onFailAtom, 0);
  12791. bool isSkip = expr->hasProperty(skipAtom);
  12792. ThorActivityKind kind = TAKcatch;
  12793. const char * helper = "Catch";
  12794. if (transform)
  12795. kind = TAKcreaterowcatch;
  12796. else if (isSkip)
  12797. kind = TAKskipcatch;
  12798. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, kind, expr, helper);
  12799. buildActivityFramework(instance);
  12800. buildInstancePrefix(instance);
  12801. if (filter)
  12802. {
  12803. BuildCtx isMatchCtx(instance->startctx);
  12804. isMatchCtx.addQuotedCompound("virtual bool isMatch(IException * except)");
  12805. associateLocalFailure(isMatchCtx, "except");
  12806. OwnedHqlExpr cseFilter = spotScalarCSE(filter);
  12807. buildReturn(isMatchCtx, cseFilter, queryBoolType());
  12808. }
  12809. if (transform)
  12810. {
  12811. BuildCtx onFailCtx(instance->startctx);
  12812. onFailCtx.addQuotedCompound("virtual unsigned transformOnExceptionCaught(ARowBuilder & crSelf, IException * except)");
  12813. ensureRowAllocated(onFailCtx, "crSelf");
  12814. associateLocalFailure(onFailCtx, "except");
  12815. buildTransformBody(onFailCtx, transform, NULL, NULL, expr, NULL);
  12816. }
  12817. else if (!isSkip)
  12818. {
  12819. LinkedHqlExpr fail = action;
  12820. if (!fail)
  12821. fail.setown(createFailAction("Missing failure", NULL, NULL, instance->activityId));
  12822. BuildCtx throwctx(instance->startctx);
  12823. throwctx.addQuotedCompound("virtual void onExceptionCaught()");
  12824. buildStmt(throwctx, fail);
  12825. }
  12826. buildInstanceSuffix(instance);
  12827. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  12828. return instance->getBoundActivity();
  12829. }
  12830. ABoundActivity * HqlCppTranslator::doBuildActivitySection(BuildCtx & ctx, IHqlExpression * expr)
  12831. {
  12832. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, expr->queryChild(0));
  12833. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKsection, expr, "Section");
  12834. StringBuffer label;
  12835. getStringValue(label, expr->queryChild(1));
  12836. instance->graphLabel.set(label.str());
  12837. buildActivityFramework(instance);
  12838. buildInstancePrefix(instance);
  12839. StringBuffer flags;
  12840. IHqlExpression * description = NULL;
  12841. ForEachChildFrom(i, expr, 2)
  12842. {
  12843. IHqlExpression * cur = expr->queryChild(i);
  12844. if (cur->isAttribute())
  12845. {
  12846. _ATOM name= cur->queryName();
  12847. if (name == privateAtom)
  12848. flags.append("|TSFprivate");
  12849. }
  12850. else if (isStringType(cur->queryType()))
  12851. {
  12852. description = cur;
  12853. flags.append("|TSFdynamicDescription");
  12854. }
  12855. }
  12856. if (description)
  12857. doBuildStringFunction(instance->startctx, "getDescription", description);
  12858. if (flags.length())
  12859. doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);
  12860. buildInstanceSuffix(instance);
  12861. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  12862. return instance->getBoundActivity();
  12863. }
  12864. ABoundActivity * HqlCppTranslator::doBuildActivitySectionInput(BuildCtx & ctx, IHqlExpression * expr)
  12865. {
  12866. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, expr->queryChild(0));
  12867. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKsectioninput, expr, "SectionInput");
  12868. StringBuffer label;
  12869. expr->queryChild(1)->queryValue()->getStringValue(label);
  12870. instance->graphLabel.set(label.str());
  12871. instance->graphEclText.append("<>");
  12872. buildActivityFramework(instance);
  12873. buildInstancePrefix(instance);
  12874. StringBuffer flags;
  12875. if (expr->hasProperty(privateAtom))
  12876. flags.append("|TSFprivate");
  12877. if (flags.length())
  12878. doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);
  12879. buildInstanceSuffix(instance);
  12880. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  12881. return instance->getBoundActivity();
  12882. }
  12883. //---------------------------------------------------------------------------
  12884. ABoundActivity * HqlCppTranslator::doBuildActivityMetaActivity(BuildCtx & ctx, IHqlExpression * expr)
  12885. {
  12886. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, expr->queryChild(0));
  12887. if (!targetThor())
  12888. return boundDataset.getClear();
  12889. assertex(expr->hasProperty(pullAtom));
  12890. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKpull, expr, "Pull");
  12891. buildActivityFramework(instance);
  12892. buildInstancePrefix(instance);
  12893. buildInstanceSuffix(instance);
  12894. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  12895. return instance->getBoundActivity();
  12896. }
  12897. //---------------------------------------------------------------------------
  12898. //-- no_sub --
  12899. ABoundActivity * HqlCppTranslator::doBuildActivitySub(BuildCtx & ctx, IHqlExpression * expr)
  12900. {
  12901. assertex(!"MORE!");
  12902. return NULL;
  12903. }
  12904. //---------------------------------------------------------------------------
  12905. //-- no_sample [GROUP] --
  12906. ABoundActivity * HqlCppTranslator::doBuildActivityEnth(BuildCtx & ctx, IHqlExpression * expr)
  12907. {
  12908. IHqlExpression * dataset = expr->queryChild(0);
  12909. IHqlExpression * numerator = expr->queryChild(1);
  12910. IHqlExpression * denominator = expr->queryChild(2);
  12911. IHqlExpression * sample = expr->queryChild(3);
  12912. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  12913. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKenth, expr, "Enth");
  12914. buildActivityFramework(instance);
  12915. buildInstancePrefix(instance);
  12916. BuildCtx funcctx(instance->startctx);
  12917. funcctx.addQuotedCompound("virtual unsigned __int64 getProportionNumerator()");
  12918. buildReturn(funcctx, numerator);
  12919. BuildCtx funcctx2(instance->startctx);
  12920. funcctx2.addQuotedCompound("virtual unsigned __int64 getProportionDenominator()");
  12921. if (denominator && !denominator->isAttribute())
  12922. buildReturn(funcctx2, denominator);
  12923. else
  12924. {
  12925. OwnedHqlExpr notProvided = createConstant(counterType->castFrom(true, I64C(-1)));
  12926. buildReturn(funcctx2, notProvided);
  12927. }
  12928. BuildCtx funcctx3(instance->startctx);
  12929. funcctx3.addQuotedCompound("virtual unsigned getSampleNumber()");
  12930. if (sample && !sample->isAttribute())
  12931. buildReturn(funcctx3, sample);
  12932. else
  12933. funcctx3.addQuoted("return 1;");
  12934. buildInstanceSuffix(instance);
  12935. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  12936. return instance->getBoundActivity();
  12937. }
  12938. //---------------------------------------------------------------------------
  12939. //-- no_sample [GROUP] --
  12940. ABoundActivity * HqlCppTranslator::doBuildActivitySample(BuildCtx & ctx, IHqlExpression * expr)
  12941. {
  12942. StringBuffer s;
  12943. IHqlExpression * dataset = expr->queryChild(0);
  12944. IHqlExpression * sampleExpr = queryRealChild(expr, 2);
  12945. unsigned sample = (unsigned)getIntValue(sampleExpr, 1);
  12946. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  12947. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKsample, expr,"Sample");
  12948. buildActivityFramework(instance);
  12949. buildInstancePrefix(instance);
  12950. doBuildUnsignedFunction(instance->startctx, "getProportion", expr->queryChild(1));
  12951. BuildCtx funcctx2(instance->startctx);
  12952. funcctx2.addQuotedCompound("virtual unsigned getSampleNumber()");
  12953. s.clear().append("return ").append(sample).append(";");
  12954. funcctx2.addQuoted(s);
  12955. buildInstanceSuffix(instance);
  12956. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  12957. return instance->getBoundActivity();
  12958. }
  12959. //---------------------------------------------------------------------------
  12960. //-- no_group [GROUP] --
  12961. /* In parms: NOT linked. Return: linked */
  12962. IHqlExpression * HqlCppTranslator::createOrderFromSortList(const DatasetReference & dataset, IHqlExpression * sortList, IHqlExpression * leftSelect, IHqlExpression * rightSelect)
  12963. {
  12964. HqlExprArray leftList, rightList;
  12965. unsigned max = sortList->numChildren();
  12966. unsigned idx;
  12967. for (idx = 0; idx < max; idx++)
  12968. {
  12969. IHqlExpression * next = sortList->queryChild(idx);
  12970. //optimize order on (string)qstring to order on qstring for example. Can make quite a difference.
  12971. if (isCast(next))
  12972. {
  12973. IHqlExpression * uncast = next->queryChild(0);
  12974. ITypeInfo * castType = next->queryType();
  12975. ITypeInfo * uncastType = uncast->queryType();
  12976. if (preservesValue(castType, uncastType) && preservesOrder(castType, uncastType))
  12977. next = uncast;
  12978. }
  12979. bool invert = false;
  12980. if (next->getOperator() == no_negate)
  12981. {
  12982. invert = true;
  12983. next = next->queryChild(0);
  12984. }
  12985. IHqlExpression * leftResolved = dataset.mapScalar(next, leftSelect);
  12986. IHqlExpression * rightResolved = dataset.mapScalar(next, rightSelect);
  12987. if (invert)
  12988. {
  12989. leftList.append(*rightResolved);
  12990. rightList.append(*leftResolved);
  12991. }
  12992. else
  12993. {
  12994. leftList.append(*leftResolved);
  12995. rightList.append(*rightResolved);
  12996. }
  12997. }
  12998. return createValue(no_order, LINK(signedType), createValue(no_sortlist, makeSortListType(NULL), leftList), createValue(no_sortlist, makeSortListType(NULL), rightList));
  12999. }
  13000. IHqlExpression * HqlCppTranslator::doCompare(BuildCtx & ctx, IHqlExpression *sortList, const DatasetReference & dataset)
  13001. {
  13002. OwnedHqlExpr selSeq = createDummySelectorSequence();
  13003. OwnedHqlExpr leftSelect = dataset.getSelector(no_left, selSeq);
  13004. OwnedHqlExpr rightSelect = dataset.getSelector(no_right, selSeq);
  13005. OwnedHqlExpr order = createOrderFromSortList(dataset, sortList, leftSelect, rightSelect);
  13006. bindTableCursor(ctx, dataset.queryDataset(), "left", no_left, selSeq);
  13007. bindTableCursor(ctx, dataset.queryDataset(), "right", no_right, selSeq);
  13008. CHqlBoundExpr bound;
  13009. buildCachedExpr(ctx, order, bound);
  13010. return bound.expr.getClear();
  13011. }
  13012. void HqlCppTranslator::buildReturnOrder(BuildCtx & ctx, IHqlExpression *sortList, const DatasetReference & dataset)
  13013. {
  13014. OwnedHqlExpr selSeq = createDummySelectorSequence();
  13015. OwnedHqlExpr leftSelect = dataset.getSelector(no_left, selSeq);
  13016. OwnedHqlExpr rightSelect = dataset.getSelector(no_right, selSeq);
  13017. OwnedHqlExpr order = createOrderFromSortList(dataset, sortList, leftSelect, rightSelect);
  13018. bindTableCursor(ctx, dataset.queryDataset(), "left", no_left, selSeq);
  13019. bindTableCursor(ctx, dataset.queryDataset(), "right", no_right, selSeq);
  13020. doBuildReturnCompare(ctx, order, no_order, false);
  13021. }
  13022. ABoundActivity * HqlCppTranslator::doBuildActivityGroup(BuildCtx & ctx, IHqlExpression * expr)
  13023. {
  13024. IHqlExpression * dataset = expr->queryChild(0);
  13025. IHqlExpression * child = dataset;
  13026. while (child->getOperator() == no_group)
  13027. child = child->queryChild(0);
  13028. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, child);
  13029. if (expr->queryType()->queryGroupInfo() == child->queryType()->queryGroupInfo())
  13030. return boundDataset.getClear();
  13031. IHqlExpression * sortlist = queryRealChild(expr, 1);
  13032. if (!sortlist || ((sortlist->numChildren() == 0) && (sortlist->getOperator() != no_activetable)))
  13033. {
  13034. bool useImplementationClass = options.minimizeActivityClasses;
  13035. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKdegroup, expr,"Degroup");
  13036. if (useImplementationClass)
  13037. instance->setImplementationClass(newDegroupArgAtom);
  13038. buildActivityFramework(instance);
  13039. buildInstancePrefix(instance);
  13040. buildInstanceSuffix(instance);
  13041. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  13042. return instance->getBoundActivity();
  13043. }
  13044. else
  13045. {
  13046. ThorActivityKind tak = (expr->getOperator() == no_group) ? TAKgroup: TAKgrouped;
  13047. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, tak, expr, "Group");
  13048. buildActivityFramework(instance);
  13049. buildInstancePrefix(instance);
  13050. //virtual bool isSameGroup(const void *left, const void *right);
  13051. BuildCtx funcctx(instance->startctx);
  13052. funcctx.addQuotedCompound("virtual bool isSameGroup(const void * _left, const void * _right)");
  13053. if (sortlist->getOperator() == no_activetable)
  13054. buildReturn(funcctx, queryBoolExpr(false));
  13055. else
  13056. {
  13057. funcctx.addQuoted("const unsigned char * left = (const unsigned char *) _left;");
  13058. funcctx.addQuoted("const unsigned char * right = (const unsigned char *) _right;");
  13059. OwnedHqlExpr selSeq = createSelectorSequence();
  13060. OwnedHqlExpr leftSelect = createSelector(no_left, dataset, selSeq);
  13061. OwnedHqlExpr rightSelect = createSelector(no_right, dataset, selSeq);
  13062. HqlExprArray args;
  13063. HqlExprArray leftValues, rightValues;
  13064. HqlExprArray compares;
  13065. unwindChildren(compares, sortlist);
  13066. //Optimize the grouping conditions by ordering them by the fields in the record (so the
  13067. //doBuildReturnCompare() can combine as many as possible), and remove duplicates
  13068. if (options.optimizeGrouping && (compares.ordinality() > 1))
  13069. {
  13070. HqlExprArray equalities;
  13071. optimizeGroupOrder(equalities, dataset, compares);
  13072. ForEachItemIn(i, equalities)
  13073. {
  13074. IHqlExpression * test = &equalities.item(i);
  13075. leftValues.append(*replaceSelector(test, dataset, leftSelect));
  13076. rightValues.append(*replaceSelector(test, dataset, rightSelect));
  13077. }
  13078. }
  13079. ForEachItemIn(idx, compares)
  13080. {
  13081. IHqlExpression * test = &compares.item(idx);
  13082. if (containsSelector(test, leftSelect) || containsSelector(test, rightSelect))
  13083. args.append(*LINK(test));
  13084. else
  13085. {
  13086. OwnedHqlExpr lhs = replaceSelector(test, dataset, leftSelect);
  13087. OwnedHqlExpr rhs = replaceSelector(test, dataset, rightSelect);
  13088. if (lhs != rhs)
  13089. {
  13090. leftValues.append(*lhs.getClear());
  13091. rightValues.append(*rhs.getClear());
  13092. }
  13093. }
  13094. }
  13095. OwnedHqlExpr result;
  13096. OwnedHqlExpr orderResult;
  13097. //Use the optimized equality code for more than one element - which often combines the comparisons.
  13098. if (leftValues.ordinality() != 0)
  13099. {
  13100. if (leftValues.ordinality() == 1)
  13101. args.append(*createValue(no_eq, makeBoolType(), LINK(&leftValues.item(0)), LINK(&rightValues.item(0))));
  13102. else
  13103. orderResult.setown(createValue(no_order, LINK(signedType), createValue(no_sortlist, makeSortListType(NULL), leftValues), createValue(no_sortlist, makeSortListType(NULL), rightValues)));
  13104. }
  13105. if (args.ordinality() == 1)
  13106. result.set(&args.item(0));
  13107. else if (args.ordinality() != 0)
  13108. result.setown(createValue(no_and, makeBoolType(), args));
  13109. bindTableCursor(funcctx, dataset, "left", no_left, selSeq);
  13110. bindTableCursor(funcctx, dataset, "right", no_right, selSeq);
  13111. IHqlExpression * trueExpr = queryBoolExpr(true);
  13112. if (result)
  13113. {
  13114. if (orderResult)
  13115. {
  13116. buildFilteredReturn(funcctx, result, trueExpr);
  13117. doBuildReturnCompare(funcctx, orderResult, no_eq, true);
  13118. }
  13119. else
  13120. {
  13121. buildReturn(funcctx, result);
  13122. }
  13123. }
  13124. else
  13125. {
  13126. if (orderResult)
  13127. doBuildReturnCompare(funcctx, orderResult, no_eq, true);
  13128. else
  13129. buildReturn(funcctx, trueExpr);
  13130. }
  13131. }
  13132. buildInstanceSuffix(instance);
  13133. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  13134. return instance->getBoundActivity();
  13135. }
  13136. }
  13137. //---------------------------------------------------------------------------
  13138. //-- no_if (dataset) --
  13139. ABoundActivity * queryAssociatedActivity(BuildCtx & ctx, IHqlExpression * expr)
  13140. {
  13141. ActivityAssociation * match = static_cast<ActivityAssociation *>(ctx.queryAssociation(expr, AssocActivity, NULL));
  13142. if (match)
  13143. return match->activity;
  13144. return NULL;
  13145. }
  13146. ABoundActivity * HqlCppTranslator::getConditionalActivity(BuildCtx & ctx, IHqlExpression * expr, bool isChild)
  13147. {
  13148. if (!expr)
  13149. return NULL;
  13150. return buildCachedActivity(ctx, expr);
  13151. }
  13152. ABoundActivity * HqlCppTranslator::doBuildActivityIf(BuildCtx & ctx, IHqlExpression * expr, bool isRoot)
  13153. {
  13154. IHqlExpression * cond = expr->queryChild(0);
  13155. IHqlExpression * trueBranch = expr->queryChild(1);
  13156. IHqlExpression * falseBranch = queryRealChild(expr, 2);
  13157. if (falseBranch && (falseBranch->getOperator() == no_null))
  13158. falseBranch = NULL;
  13159. OwnedHqlExpr cseCond = options.spotCSE ? spotScalarCSE(cond) : LINK(cond);
  13160. bool isChild = (insideChildGraph(ctx) || insideRemoteGraph(ctx) || insideLibrary());
  13161. IHqlExpression * activeGraph = queryActiveSubGraph(ctx)->graphTag;
  13162. if (isChild)
  13163. {
  13164. Owned<ABoundActivity> boundTrue = buildCachedActivity(ctx, trueBranch);
  13165. Owned<ABoundActivity> boundFalse = falseBranch ? buildCachedActivity(ctx, falseBranch) : NULL;
  13166. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKchildif, expr, "If");
  13167. buildActivityFramework(instance, isRoot);
  13168. buildInstancePrefix(instance);
  13169. {
  13170. MemberFunction getcond(*this, instance->startctx, "virtual bool getCondition()", MFsingle);
  13171. buildReturn(getcond.ctx, cseCond);
  13172. }
  13173. if (isGraphIndependent(cseCond, activeGraph) && !instance->hasChildActivity)
  13174. instance->addAttributeBool("_graphIndependent", true);
  13175. buildConnectInputOutput(ctx, instance, boundTrue, 0, 0, "True");
  13176. if (boundFalse)
  13177. buildConnectInputOutput(ctx, instance, boundFalse, 0, 1, "False");
  13178. buildInstanceSuffix(instance);
  13179. return instance->getBoundActivity();
  13180. }
  13181. else
  13182. {
  13183. Owned<ABoundActivity> boundTrue = getConditionalActivity(ctx, trueBranch, isChild);
  13184. Owned<ABoundActivity> boundFalse = getConditionalActivity(ctx, falseBranch, isChild);
  13185. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, expr->isAction() ? TAKifaction : TAKif, expr,"If");
  13186. buildActivityFramework(instance, isRoot);
  13187. buildInstancePrefix(instance);
  13188. {
  13189. MemberFunction getcond(*this, instance->startctx, "virtual bool getCondition()", MFsingle);
  13190. buildReturn(getcond.ctx, cseCond);
  13191. }
  13192. if (isGraphIndependent(cseCond, activeGraph) && !instance->hasChildActivity)
  13193. instance->addAttributeBool("_graphIndependent", true);
  13194. if (expr->isAction())
  13195. {
  13196. if (boundTrue)
  13197. addDependency(ctx, boundTrue, instance->queryBoundActivity(), dependencyAtom, "True", 1);
  13198. if (boundFalse)
  13199. addDependency(ctx, boundFalse, instance->queryBoundActivity(), dependencyAtom, "False", 2);
  13200. }
  13201. else
  13202. {
  13203. if (boundTrue)
  13204. buildConnectInputOutput(ctx, instance, boundTrue, 0, 0, "True");
  13205. if (boundFalse)
  13206. buildConnectInputOutput(ctx, instance, boundFalse, 0, 1, "False");
  13207. }
  13208. buildInstanceSuffix(instance);
  13209. return instance->getBoundActivity();
  13210. }
  13211. }
  13212. //---------------------------------------------------------------------------
  13213. ABoundActivity * HqlCppTranslator::doBuildActivitySequentialParallel(BuildCtx & ctx, IHqlExpression * expr, bool isRoot)
  13214. {
  13215. Array boundActivities;
  13216. ForEachChild(i, expr)
  13217. {
  13218. IHqlExpression * cur = expr->queryChild(i);
  13219. if (!cur->isAttribute() && (cur->getOperator() != no_null))
  13220. {
  13221. ABoundActivity * activity = buildCachedActivity(ctx, cur);
  13222. if (activity)
  13223. boundActivities.append(*activity);
  13224. }
  13225. }
  13226. ThorActivityKind kind = (expr->getOperator() != no_parallel) ? TAKsequential : TAKparallel;
  13227. const char * helper = (expr->getOperator() != no_parallel) ? "Sequential" : "Parallel";
  13228. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, kind, expr, helper);
  13229. buildActivityFramework(instance, isRoot);
  13230. buildInstancePrefix(instance);
  13231. doBuildUnsignedFunction(instance->createctx, "numBranches", boundActivities.ordinality());
  13232. ForEachItemIn(j, boundActivities)
  13233. {
  13234. ABoundActivity & cur = (ABoundActivity&)boundActivities.item(j);
  13235. StringBuffer temp;
  13236. temp.append("Action #").append(j+1);
  13237. addDependency(ctx, &cur, instance->queryBoundActivity(), dependencyAtom, temp.str(), j+1);
  13238. }
  13239. buildInstanceSuffix(instance);
  13240. return instance->getBoundActivity();
  13241. }
  13242. ABoundActivity * HqlCppTranslator::doBuildActivityCase(BuildCtx & ctx, IHqlExpression * expr, bool isRoot)
  13243. {
  13244. node_operator op = expr->getOperator();
  13245. bool isChild = (insideChildGraph(ctx) || insideRemoteGraph(ctx));
  13246. unsigned first = 0;
  13247. unsigned max = expr->numChildren();
  13248. if (op == no_case)
  13249. first++;
  13250. CIArrayOf<ABoundActivity> inputs;
  13251. for (unsigned iinput = first; iinput < max-1; iinput++)
  13252. inputs.append(*getConditionalActivity(ctx, expr->queryChild(iinput)->queryChild(1), isChild));
  13253. Owned<ABoundActivity> boundDefault = getConditionalActivity(ctx, expr->queryChild(max-1), isChild);
  13254. ThorActivityKind tak = isChild ? TAKchildcase : TAKcase;
  13255. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, tak, expr, "Case");
  13256. buildActivityFramework(instance, isRoot);
  13257. buildInstancePrefix(instance);
  13258. BuildCtx funcctx(instance->startctx);
  13259. funcctx.addQuotedCompound("virtual unsigned getBranch()");
  13260. //MORE: If we created a map/case expression and then called buildReturn it would potentially generate better code.
  13261. // code is below, but not enabled because it doesn't necessarily improve things at the moment (see jholt39.xhql)
  13262. IHqlExpression * activeGraph = queryActiveSubGraph(ctx)->graphTag;
  13263. HqlExprArray args;
  13264. if (op == no_case)
  13265. args.append(*LINK(expr->queryChild(0)));
  13266. StringBuffer label;
  13267. for (unsigned idx = first; idx < max-1; idx++)
  13268. {
  13269. unsigned branchIdx = idx-first;
  13270. IHqlExpression * branch = expr->queryChild(idx);
  13271. IHqlExpression * branchCond = branch->queryChild(0);
  13272. OwnedHqlExpr ret = getSizetConstant(branchIdx);
  13273. args.append(*createValue(no_mapto, ret->getType(), LINK(branchCond), LINK(ret)));
  13274. ABoundActivity * boundBranch = &inputs.item(branchIdx);
  13275. getExprECL(branchCond, label.clear(), false, true);
  13276. if (label.length() > 20)
  13277. label.clear().append("Branch ").append(branchIdx+1);
  13278. if (expr->isAction())
  13279. addDependency(ctx, boundBranch, instance->queryBoundActivity(), dependencyAtom, label.str(), branchIdx+1);
  13280. else
  13281. buildConnectInputOutput(ctx, instance, boundBranch, 0, idx-first, label.str());
  13282. }
  13283. args.append(*createConstant(unsignedType->castFrom(false, (__int64)max-1)));
  13284. OwnedHqlExpr fullCond = createValue(op, LINK(unsignedType), args);
  13285. fullCond.setown(foldHqlExpression(fullCond));
  13286. if (options.spotCSE)
  13287. fullCond.setown(spotScalarCSE(fullCond));
  13288. buildReturn(funcctx, fullCond);
  13289. bool graphIndependent = isGraphIndependent(fullCond, activeGraph);
  13290. if (graphIndependent && !instance->hasChildActivity)
  13291. instance->addAttributeBool("_graphIndependent", true);
  13292. buildConnectInputOutput(ctx, instance, boundDefault, 0, max-1-first, "default");
  13293. buildInstanceSuffix(instance);
  13294. return instance->getBoundActivity();
  13295. }
  13296. //---------------------------------------------------------------------------
  13297. //-- no_sort [SORT] --
  13298. void HqlCppTranslator::buildSkewThresholdMembers(BuildCtx & ctx, IHqlExpression * expr)
  13299. {
  13300. StringBuffer s, temp;
  13301. if (getProperty(expr, thresholdAtom, temp.clear()))
  13302. {
  13303. s.clear().append("virtual unsigned __int64 getThreshold() { return ").append(temp).append("; }");
  13304. ctx.addQuoted(s);
  13305. }
  13306. IHqlExpression * skew = expr->queryProperty(skewAtom);
  13307. if (skew)
  13308. {
  13309. Owned<ITypeInfo> doubleType = makeRealType(8);
  13310. IHqlExpression * skewMax = skew->queryChild(0);
  13311. if (skewMax->getOperator() != no_null)
  13312. doBuildFunction(ctx, doubleType, "getSkew", skewMax);
  13313. IHqlExpression * skewTarget = queryRealChild(skew, 1);
  13314. doBuildFunction(ctx, doubleType, "getTargetSkew", skewTarget);
  13315. }
  13316. }
  13317. ABoundActivity * HqlCppTranslator::doBuildActivitySort(BuildCtx & ctx, IHqlExpression * expr)
  13318. {
  13319. IHqlExpression * dataset = expr->queryChild(0);
  13320. LinkedHqlExpr sortlist = expr->queryChild(1);
  13321. IHqlExpression * limit = NULL;
  13322. IHqlExpression * cosort = NULL;
  13323. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  13324. const char *helper;
  13325. ThorActivityKind actKind = TAKsort;
  13326. switch (expr->getOperator())
  13327. {
  13328. case no_topn:
  13329. {
  13330. actKind = TAKtopn;
  13331. limit = expr->queryChild(2);
  13332. helper = "TopN";
  13333. break;
  13334. }
  13335. case no_assertsorted:
  13336. {
  13337. actKind = TAKsorted;
  13338. helper = "Sort";
  13339. break;
  13340. }
  13341. default:
  13342. {
  13343. cosort = expr->queryChild(2);
  13344. if (cosort && cosort->isAttribute())
  13345. cosort = NULL;
  13346. helper = "Sort";
  13347. break;
  13348. }
  13349. }
  13350. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, actKind, expr, helper);
  13351. buildActivityFramework(instance);
  13352. StringBuffer s;
  13353. buildInstancePrefix(instance);
  13354. instance->classctx.addQuoted("virtual ICompare * queryCompare() { return &compare; }");
  13355. // sortlist.setown(spotScalarCSE(sortlist));
  13356. buildCompareClass(instance->nestedctx, "compare", sortlist, DatasetReference(dataset));
  13357. IHqlExpression * record = dataset->queryRecord();
  13358. OwnedHqlExpr serializedRecord = getSerializedForm(record);
  13359. if (!targetRoxie())
  13360. {
  13361. if (record != serializedRecord)
  13362. {
  13363. instance->classctx.addQuoted("virtual ICompare * queryCompareSerializedRow() { return &compareSR; }");
  13364. OwnedHqlExpr selSeq = createSelectorSequence();
  13365. OwnedHqlExpr leftSelector = createSelector(no_left, dataset, selSeq);
  13366. OwnedHqlExpr mappedSortlist = replaceSelector(sortlist, dataset, leftSelector);
  13367. OwnedHqlExpr serializedSortlist = replaceMemorySelectorWithSerializedSelector(mappedSortlist, record, no_left, selSeq);
  13368. OwnedHqlExpr serializedDataset = createDataset(no_null, LINK(serializedRecord));
  13369. DatasetReference serializedRef(serializedDataset, no_left, selSeq);
  13370. try
  13371. {
  13372. buildCompareClass(instance->nestedctx, "compareSR", serializedSortlist, serializedRef);
  13373. }
  13374. catch (IException * e)
  13375. {
  13376. e->Release();
  13377. ERRORAT(queryLocation(expr), HQLERR_CannotGenerateSerializedCompare);
  13378. }
  13379. }
  13380. }
  13381. HqlExprArray sorts;
  13382. sortlist->unwindList(sorts, no_sortlist);
  13383. bool tryToSerializeKey = (actKind == TAKsort) && !isGroupedActivity(expr) && !isLocalActivity(expr) && !instance->isChildActivity();
  13384. generateSerializeKey(instance->nestedctx, no_none, DatasetReference(dataset), sorts, tryToSerializeKey, false, false);
  13385. buildSkewThresholdMembers(instance->classctx, expr);
  13386. if (limit)
  13387. {
  13388. OwnedHqlExpr newLimit = ensurePositiveOrZeroInt64(limit);
  13389. BuildCtx funcctx(instance->startctx);
  13390. funcctx.addQuotedCompound("virtual __int64 getLimit()");
  13391. buildReturn(funcctx, newLimit, defaultIntegralType);
  13392. }
  13393. IHqlExpression * best = expr->queryProperty(bestAtom);
  13394. if (best)
  13395. {
  13396. doBuildBoolFunction(instance->classctx, "hasBest", true);
  13397. HqlExprArray sortValues, maxValues;
  13398. sortlist->unwindList(sortValues, no_sortlist);
  13399. unwindChildren(maxValues, best);
  13400. ForEachItemIn(i, sortValues)
  13401. {
  13402. IHqlExpression & cur = sortValues.item(i);
  13403. if (cur.getOperator() == no_negate)
  13404. {
  13405. sortValues.replace(OLINK(maxValues.item(i)), i);
  13406. maxValues.replace(*LINK(cur.queryChild(0)), i);
  13407. }
  13408. }
  13409. OwnedHqlExpr order = createValue(no_order, LINK(signedType), createValue(no_sortlist, makeSortListType(NULL), sortValues), createValue(no_sortlist, makeSortListType(NULL), maxValues));
  13410. BuildCtx funcctx(instance->startctx);
  13411. funcctx.addQuotedCompound("virtual int compareBest(const void * _self)");
  13412. funcctx.addQuoted("unsigned char * self = (unsigned char *) _self;");
  13413. bindTableCursor(funcctx, dataset, "self");
  13414. buildReturn(funcctx, order);
  13415. }
  13416. if (cosort)
  13417. {
  13418. //cosort has form joined(dataset),
  13419. IHqlExpression * cosortDataset = cosort->queryChild(0);
  13420. if (cosortDataset->getOperator() == no_compound_diskread)
  13421. cosortDataset = cosortDataset->queryChild(0);
  13422. if (cosortDataset->getOperator() == no_sorted)
  13423. {
  13424. IHqlExpression * source = cosortDataset->queryChild(0);
  13425. if (source->getOperator() != no_table)
  13426. throwError(HQLERR_JoinSortedMustBeDataset);
  13427. IHqlExpression * sourceType = source->queryChild(2);
  13428. if (!sourceType || sourceType->getOperator() != no_thor)
  13429. throwError(HQLERR_JoinSortedMustBeThor);
  13430. s.clear().append("virtual const char * getSortedFilename() { return ");
  13431. generateExprCpp(s, source->queryChild(0)).append("; }");
  13432. instance->startctx.addQuoted(s);
  13433. buildMetaMember(instance->classctx, cosortDataset, "querySortedRecordSize");
  13434. }
  13435. else
  13436. {
  13437. instance->startctx.addQuoted("virtual const char * getSortedFilename() { return NULL; }");
  13438. instance->classctx.addQuoted("virtual IOutputMetaData * querySortedRecordSize() { return NULL; }");
  13439. ABoundActivity * masterSort = queryAssociatedActivity(ctx, cosortDataset);
  13440. if (!masterSort)
  13441. throwError(HQLERR_SortAndCoSortConcurrent);
  13442. Owned<ABoundActivity> slave = instance->getBoundActivity();
  13443. buildConnectOrders(ctx, slave, masterSort);
  13444. }
  13445. HqlExprArray left, right;
  13446. cosortDataset->queryChild(1)->unwindList(left, no_sortlist);
  13447. sortlist->unwindList(right, no_sortlist);
  13448. doCompareLeftRight(instance->nestedctx, "CompareLeftRight", DatasetReference(cosortDataset), DatasetReference(dataset), left, right);
  13449. }
  13450. if (expr->hasProperty(manyAtom))
  13451. instance->classctx.addQuoted("virtual bool hasManyRecords() { return true; }");
  13452. IHqlExpression * stable = expr->queryProperty(stableAtom);
  13453. IHqlExpression * unstable = expr->queryProperty(unstableAtom);
  13454. IHqlExpression * method = NULL;
  13455. StringBuffer flags;
  13456. if (stable)
  13457. {
  13458. flags.append("|TAFstable");
  13459. method = stable->queryChild(0);
  13460. }
  13461. else if (unstable)
  13462. {
  13463. flags.append("|TAFunstable");
  13464. method = unstable->queryChild(0);
  13465. }
  13466. if (!method || method->isConstant())
  13467. flags.append("|TAFconstant");
  13468. if (method)
  13469. doBuildVarStringFunction(instance->startctx, "queryAlgorithm", method);
  13470. if (!streq(flags.str(), "|TAFconstant"))
  13471. instance->classctx.addQuotedF("virtual unsigned getAlgorithmFlags() { return %s; }", flags.str()+1);
  13472. buildInstanceSuffix(instance);
  13473. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  13474. return instance->getBoundActivity();
  13475. }
  13476. //---------------------------------------------------------------------------
  13477. void HqlCppTranslator::doBuildXmlReadMember(ActivityInstance & instance, IHqlExpression * expr, const char * functionName, bool & usesContents)
  13478. {
  13479. StringBuffer s, xmlInstanceName;
  13480. usesContents = false;
  13481. if (isValidXmlRecord(expr->queryRecord()))
  13482. {
  13483. StringBuffer xmlFactoryName;
  13484. getUniqueId(xmlInstanceName.append("xml"));
  13485. buildXmlReadTransform(expr, xmlFactoryName, usesContents);
  13486. instance.classctx.addQuoted(s.clear().append("Owned<IXmlToRowTransformer> ").append(xmlInstanceName).append(";"));
  13487. BuildCtx * callctx = NULL;
  13488. instance.evalContext->getInvariantMemberContext(NULL, NULL, &callctx, true, false);
  13489. callctx->addQuoted(s.clear().append(xmlInstanceName).append(".setown(").append(xmlFactoryName).append("(ctx,").append(instance.activityId).append("));"));
  13490. }
  13491. else
  13492. xmlInstanceName.append("NULL");
  13493. s.clear().append("virtual IXmlToRowTransformer * ").append(functionName).append("() { return ").append(xmlInstanceName).append("; }");
  13494. instance.classctx.addQuoted(s);
  13495. }
  13496. ABoundActivity * HqlCppTranslator::doBuildActivityWorkunitRead(BuildCtx & ctx, IHqlExpression * expr)
  13497. {
  13498. IHqlExpression * wuid = expr->queryProperty(wuidAtom);
  13499. IHqlExpression * sequence = queryPropertyChild(expr, sequenceAtom, 0);
  13500. IHqlExpression * name = queryPropertyChild(expr, nameAtom, 0);
  13501. __int64 sequenceValue = sequence->queryValue()->getIntValue();
  13502. bool isStored = (sequenceValue == ResultSequenceStored);
  13503. bool useImplementationClass = options.minimizeActivityClasses && !wuid && (sequenceValue == ResultSequenceInternal);
  13504. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKworkunitread, expr,"WorkunitRead");
  13505. if (useImplementationClass)
  13506. instance->setImplementationClass(newWorkUnitReadArgAtom);
  13507. noteResultAccessed(ctx, sequence, name);
  13508. StringBuffer graphLabel;
  13509. graphLabel.append(getActivityText(instance->kind)).append("\n");
  13510. getStoredDescription(graphLabel, sequence, name, true);
  13511. instance->graphLabel.set(graphLabel.str());
  13512. buildActivityFramework(instance);
  13513. buildInstancePrefix(instance);
  13514. if (!useImplementationClass)
  13515. {
  13516. doBuildVarStringFunction(instance->classctx, "queryName", name);
  13517. if (sequenceValue != ResultSequenceInternal)
  13518. {
  13519. BuildCtx func2ctx(instance->classctx);
  13520. func2ctx.addQuotedCompound("virtual int querySequence()");
  13521. buildReturn(func2ctx, sequence, signedType);
  13522. }
  13523. if (wuid)
  13524. doBuildVarStringFunction(instance->classctx, "queryWUID", wuid->queryChild(0));
  13525. bool usesContents = false;
  13526. if (isStored || (targetRoxie() && (sequenceValue >= 0)))
  13527. doBuildXmlReadMember(*instance, expr, "queryXmlTransformer", usesContents);
  13528. StringBuffer csvInstanceName;
  13529. if (isValidCsvRecord(expr->queryRecord()) && isStored && options.allowCsvWorkunitRead)
  13530. {
  13531. buildCsvReadTransformer(expr, csvInstanceName, NULL);
  13532. StringBuffer s;
  13533. s.append("virtual ICsvToRowTransformer * queryCsvTransformer() { return &").append(csvInstanceName).append("; }");
  13534. instance->classctx.addQuoted(s);
  13535. }
  13536. }
  13537. else
  13538. {
  13539. instance->addConstructorParameter(name);
  13540. }
  13541. queryAddResultDependancy(*instance->queryBoundActivity(), sequence, name);
  13542. buildInstanceSuffix(instance);
  13543. return instance->getBoundActivity();
  13544. }
  13545. //---------------------------------------------------------------------------
  13546. //-- xmlparse
  13547. void HqlCppTranslator::noteXpathUsed(const char * xpath)
  13548. {
  13549. if (strstr(xpath, XPATH_CONTENTS_TEXT))
  13550. xmlUsesContents = true;
  13551. }
  13552. void HqlCppTranslator::noteXpathUsed(IHqlExpression * expr)
  13553. {
  13554. IValue * value = expr->queryValue();
  13555. if (value)
  13556. {
  13557. StringBuffer temp;
  13558. value->getStringValue(temp);
  13559. noteXpathUsed(temp);
  13560. }
  13561. else
  13562. xmlUsesContents = true;
  13563. }
  13564. ABoundActivity * HqlCppTranslator::doBuildActivityXmlParse(BuildCtx & ctx, IHqlExpression * expr)
  13565. {
  13566. IHqlExpression * dataset = expr->queryChild(0);
  13567. IHqlExpression * selSeq = querySelSeq(expr);
  13568. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  13569. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKxmlparse, expr, "XmlParse");
  13570. buildActivityFramework(instance);
  13571. buildInstancePrefix(instance);
  13572. IHqlExpression * xmlAttr = expr->queryProperty(xmlAtom);
  13573. //MORE: What encoding is the search text in???
  13574. doBuildParseSearchText(instance->startctx, dataset, expr->queryChild(1), type_utf8, unknownStringType);
  13575. doBuildVarStringFunction(instance->classctx, "queryIteratorPath", xmlAttr ? queryRealChild(xmlAttr, 0) : NULL);
  13576. BuildCtx funcctx(instance->startctx);
  13577. funcctx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf, const void * _left, IColumnProvider * parsed)");
  13578. ensureRowAllocated(funcctx, "crSelf");
  13579. funcctx.addQuoted("const unsigned char * left = (const unsigned char *) _left;");
  13580. // Both left and the dataset are bound to left because it might be a new transform or a transform
  13581. IHqlExpression * transform = expr->queryChild(3);
  13582. BoundRow * selfCursor = bindSelf(funcctx, expr, "crSelf");
  13583. if (transform->getOperator() == no_newtransform)
  13584. bindTableCursor(funcctx, dataset, "left");
  13585. else
  13586. bindTableCursor(funcctx, dataset, "left", no_left, selSeq);
  13587. associateSkipReturnMarker(funcctx, queryZero(), selfCursor);
  13588. OwnedHqlExpr helperName = createQuoted("parsed", makeBoolType());
  13589. funcctx.associateExpr(xmlColumnProviderMarkerExpr, helperName);
  13590. xmlUsesContents = false;
  13591. doTransform(funcctx, transform, selfCursor);
  13592. buildReturnRecordSize(funcctx, selfCursor);
  13593. if (xmlUsesContents)
  13594. instance->classctx.addQuoted("virtual bool requiresContents() { return true; }");
  13595. buildInstanceSuffix(instance);
  13596. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  13597. return instance->getBoundActivity();
  13598. }
  13599. void HqlCppTranslator::doBuildExprXmlText(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt)
  13600. {
  13601. HqlExprAssociation * match = ctx.queryMatchExpr(xmlColumnProviderMarkerExpr);
  13602. if (!match)
  13603. throwError(HQLERR_XmlTextNotValid);
  13604. IHqlExpression * xpath = expr->queryChild(0);
  13605. noteXpathUsed(xpath);
  13606. HqlExprArray args;
  13607. args.append(*LINK(match->queryExpr()));
  13608. args.append(*LINK(xpath));
  13609. OwnedHqlExpr call = bindFunctionCall(columnGetStringXAtom, args);
  13610. buildCachedExpr(ctx, call, tgt);
  13611. }
  13612. void HqlCppTranslator::doBuildExprXmlUnicode(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt)
  13613. {
  13614. HqlExprAssociation * match = ctx.queryMatchExpr(xmlColumnProviderMarkerExpr);
  13615. if (!match)
  13616. throwError(HQLERR_XmlUnicodeNotValid);
  13617. IHqlExpression * xpath = expr->queryChild(0);
  13618. noteXpathUsed(xpath);
  13619. HqlExprArray args;
  13620. args.append(*LINK(match->queryExpr()));
  13621. args.append(*LINK(xpath));
  13622. OwnedHqlExpr call = bindFunctionCall(columnGetUnicodeXAtom, args);
  13623. buildCachedExpr(ctx, call, tgt);
  13624. }
  13625. void HqlCppTranslator::buildDatasetAssignXmlProject(BuildCtx & ctx, IHqlCppDatasetBuilder * target, IHqlExpression * expr)
  13626. {
  13627. HqlExprAssociation * match = ctx.queryMatchExpr(xmlColumnProviderMarkerExpr);
  13628. if (!match)
  13629. throwError(HQLERR_XmlTextNotValid);
  13630. StringBuffer iterTag;
  13631. IHqlExpression * xpath = expr->queryChild(0);
  13632. if (xpath->queryValue())
  13633. xpath->queryValue()->getStringValue(iterTag);
  13634. noteXpathUsed(xpath);
  13635. //Generate the code to process a child iterator
  13636. OwnedHqlExpr subRowExpr;
  13637. BuildCtx loopctx(ctx);
  13638. buildXmlReadChildrenIterator(loopctx, iterTag.str(), match->queryExpr(), subRowExpr);
  13639. loopctx.associateExpr(xmlColumnProviderMarkerExpr, subRowExpr);
  13640. BoundRow * targetRow = target->buildCreateRow(loopctx);
  13641. Owned<IReferenceSelector> targetRef = buildActiveRow(loopctx, targetRow->querySelector());
  13642. OwnedHqlExpr rowValue = createRow(no_createrow, LINK(expr->queryChild(1)));
  13643. buildRowAssign(loopctx, targetRef, rowValue);
  13644. target->finishRow(loopctx, targetRow);
  13645. }
  13646. //---------------------------------------------------------------------------
  13647. //-- no_temptable [DATASET] --
  13648. ABoundActivity * HqlCppTranslator::doBuildActivityTempTable(BuildCtx & ctx, IHqlExpression * expr)
  13649. {
  13650. StringBuffer s;
  13651. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKtemptable, expr, "TempTable");
  13652. OwnedHqlExpr values = normalizeListCasts(expr->queryChild(0));
  13653. IHqlExpression * record = expr->queryChild(1);
  13654. IHqlExpression * defaults = expr->queryChild(2);
  13655. assertex(values->getOperator() != no_recordlist); // should have been transformed by now.
  13656. //-----------------
  13657. buildActivityFramework(instance);
  13658. buildInstancePrefix(instance);
  13659. BuildCtx funcctx(instance->startctx);
  13660. funcctx.addQuotedCompound("virtual size32_t getRow(ARowBuilder & crSelf, unsigned row)");
  13661. ensureRowAllocated(funcctx, "crSelf");
  13662. BoundRow * selfCursor = bindSelf(funcctx, instance->dataset, "crSelf");
  13663. IHqlExpression * self = selfCursor->querySelector();
  13664. OwnedHqlExpr clearAction;
  13665. OwnedHqlExpr rowsExpr;
  13666. OwnedHqlExpr rowVar = createVariable("row", makeIntType(4, false));
  13667. if (expr->getOperator() == no_datasetfromrow)
  13668. {
  13669. BuildCtx subctx(funcctx);
  13670. subctx.addFilter(rowVar);
  13671. if (clearAction)
  13672. subctx.addExpr(clearAction);
  13673. subctx.addReturn(queryZero());
  13674. buildAssign(funcctx, self, values);
  13675. buildReturnRecordSize(funcctx, selfCursor);
  13676. rowsExpr.setown(getSizetConstant(1));
  13677. }
  13678. else if ((values->getOperator() == no_list) || (values->getOperator() == no_null))
  13679. // else if (((values->getOperator() == no_list) &&
  13680. // (!values->isConstant() || (values->queryType()->queryChildType()->getSize() == UNKNOWN_LENGTH))) || (values->getOperator() == no_null))
  13681. {
  13682. //
  13683. unsigned maxRows = values->numChildren();
  13684. if (maxRows)
  13685. {
  13686. unsigned dummyIdx = 0;
  13687. OwnedHqlExpr tgt = createSelectExpr(LINK(self), LINK(queryNextRecordField(record, dummyIdx)));
  13688. BuildCtx switchctx(funcctx);
  13689. switchctx.addQuotedCompound("switch (row)");
  13690. unsigned row;
  13691. for (row = 0; row < maxRows; row++)
  13692. {
  13693. BuildCtx casectx(switchctx);
  13694. casectx.addQuotedCompound(s.clear().append("case ").append(row).append(":"));
  13695. buildAssign(casectx, tgt, values->queryChild(row));
  13696. //casectx.setNextDestructor();
  13697. buildReturnRecordSize(casectx, selfCursor);
  13698. }
  13699. }
  13700. if (clearAction)
  13701. funcctx.addExpr(clearAction);
  13702. funcctx.addReturn(queryZero());
  13703. rowsExpr.setown(getSizetConstant(maxRows));
  13704. }
  13705. else
  13706. {
  13707. CHqlBoundExpr bound;
  13708. //MORE: This shouldn't be done this way...
  13709. OwnedHqlExpr normalized = normalizeListCasts(values);
  13710. if (normalized->getOperator() == no_alias)
  13711. buildExpr(instance->startctx, normalized, bound);
  13712. else
  13713. {
  13714. BuildCtx * declarectx;
  13715. BuildCtx * callctx;
  13716. instance->evalContext->getInvariantMemberContext(NULL, &declarectx, &callctx, false, isContextDependent(normalized, false, false) || !isIndependentOfScope(normalized));
  13717. // if (isContextDependent(normalized, false, false))
  13718. // buildTempExpr(instance->onstartctx, *declarectx, normalized, bound, FormatNatural);
  13719. // else
  13720. CHqlBoundTarget tempTarget;
  13721. buildTempExpr(*callctx, *declarectx, tempTarget, normalized, FormatNatural, !canSetBeAll(normalized));
  13722. bound.setFromTarget(tempTarget);
  13723. }
  13724. rowsExpr.setown(getBoundCount(bound));
  13725. rowsExpr.setown(createTranslated(rowsExpr));
  13726. OwnedHqlExpr compare = createValue(no_ge, makeBoolType(), LINK(rowVar), LINK(rowsExpr));
  13727. BuildCtx condctx(funcctx);
  13728. buildFilter(condctx, compare);
  13729. if (clearAction)
  13730. condctx.addExpr(clearAction);
  13731. buildReturn(condctx, queryZero());
  13732. HqlExprArray args;
  13733. args.append(*bound.getTranslatedExpr());
  13734. args.append(*adjustValue(rowVar, 1));
  13735. args.append(*createAttribute(noBoundCheckAtom));
  13736. args.append(*createAttribute(forceAllCheckAtom));
  13737. OwnedHqlExpr src = createValue(no_index, LINK(values->queryType()->queryChildType()), args);
  13738. OwnedHqlExpr tgt = createSelectExpr(LINK(self), LINK(queryOnlyField(record)));
  13739. buildAssign(funcctx, tgt, src);
  13740. buildReturnRecordSize(funcctx, selfCursor);
  13741. }
  13742. doBuildUnsignedFunction(instance->startctx, "numRows", rowsExpr);
  13743. if (!values->isConstant())
  13744. doBuildBoolFunction(instance->startctx, "isConstant", false);
  13745. buildInstanceSuffix(instance);
  13746. return instance->getBoundActivity();
  13747. }
  13748. //NB: Also used to create row no_null as an activity.
  13749. ABoundActivity * HqlCppTranslator::doBuildActivityCreateRow(BuildCtx & ctx, IHqlExpression * expr, bool isDataset)
  13750. {
  13751. bool valuesAreConstant = false;
  13752. StringBuffer valueText;
  13753. IValue * singleValue = NULL;
  13754. node_operator op = expr->getOperator();
  13755. if (op == no_createrow)
  13756. {
  13757. IHqlExpression * transform = expr->queryChild(0);
  13758. if (transform->isConstant())
  13759. {
  13760. IHqlExpression * assign = queryTransformSingleAssign(transform);
  13761. if (assign)
  13762. {
  13763. singleValue = assign->queryChild(1)->queryValue();
  13764. if (singleValue)
  13765. singleValue->generateECL(valueText);
  13766. }
  13767. valuesAreConstant = true;
  13768. }
  13769. if (!isDataset && containsSkip(transform))
  13770. reportError(queryLocation(transform), ECODETEXT(HQLERR_SkipInsideCreateRow));
  13771. }
  13772. else if (op == no_null)
  13773. {
  13774. valueText.append("Blank");
  13775. valuesAreConstant = true;
  13776. }
  13777. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKtemprow, expr,"TempRow");
  13778. if (valueText.length())
  13779. {
  13780. StringBuffer graphLabel;
  13781. if (valueText.length() > MAX_ROW_VALUE_TEXT_LEN)
  13782. {
  13783. valueText.setLength(MAX_ROW_VALUE_TEXT_LEN);
  13784. valueText.append("...");
  13785. }
  13786. graphLabel.append(getActivityText(instance->kind)).append("\n{").append(valueText).append("}");
  13787. instance->graphLabel.set(graphLabel.str());
  13788. }
  13789. //-----------------
  13790. buildActivityFramework(instance);
  13791. buildInstancePrefix(instance);
  13792. BuildCtx funcctx(instance->startctx);
  13793. funcctx.addQuotedCompound("virtual size32_t getRowSingle(ARowBuilder & crSelf)");
  13794. ensureRowAllocated(funcctx, "crSelf");
  13795. BoundRow * selfCursor = bindSelf(funcctx, instance->dataset, "crSelf");
  13796. IHqlExpression * self = selfCursor->querySelector();
  13797. if (isDataset)
  13798. associateSkipReturnMarker(funcctx, queryBoolExpr(false), selfCursor);
  13799. buildAssign(funcctx, self, expr);
  13800. buildReturnRecordSize(funcctx, selfCursor);
  13801. if (!valuesAreConstant)
  13802. doBuildBoolFunction(instance->startctx, "isConstant", false);
  13803. buildInstanceSuffix(instance);
  13804. return instance->getBoundActivity();
  13805. }
  13806. ABoundActivity * HqlCppTranslator::doBuildActivityInlineTable(BuildCtx & ctx, IHqlExpression * expr)
  13807. {
  13808. IHqlExpression * values = expr->queryChild(0);
  13809. if (values->numChildren() == 1)
  13810. {
  13811. OwnedHqlExpr rowValue = createRow(no_createrow, LINK(values->queryChild(0)));
  13812. OwnedHqlExpr row = expr->cloneAllAnnotations(rowValue);
  13813. return doBuildActivityCreateRow(ctx, row, true);
  13814. }
  13815. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKtemptable, expr,"TempTable");
  13816. //-----------------
  13817. buildActivityFramework(instance);
  13818. buildInstancePrefix(instance);
  13819. BuildCtx funcctx(instance->startctx);
  13820. funcctx.addQuotedCompound("virtual size32_t getRow(ARowBuilder & crSelf, unsigned row)");
  13821. ensureRowAllocated(funcctx, "crSelf");
  13822. BoundRow * selfCursor = bindSelf(funcctx, instance->dataset, "crSelf");
  13823. IHqlExpression * self = selfCursor->querySelector();
  13824. unsigned maxRows = values->numChildren();
  13825. bool done = false;
  13826. if (values->isConstant())
  13827. {
  13828. CHqlBoundExpr bound;
  13829. if (doBuildDatasetInlineTable(funcctx, expr, bound, FormatNatural))
  13830. {
  13831. OwnedHqlExpr whichRow = createVariable("row", LINK(unsignedType));
  13832. BuildCtx subctx(funcctx);
  13833. OwnedHqlExpr test = createValue(no_ge, LINK(whichRow), getSizetConstant(maxRows));
  13834. subctx.addFilter(test);
  13835. buildReturn(subctx, queryZero());
  13836. OwnedHqlExpr ds = bound.getTranslatedExpr();
  13837. OwnedHqlExpr thisRow = createRowF(no_selectnth, LINK(ds), adjustValue(whichRow, 1), createAttribute(noBoundCheckAtom), NULL);
  13838. buildAssign(funcctx, self, thisRow);
  13839. buildReturnRecordSize(funcctx, selfCursor);
  13840. done = true;
  13841. }
  13842. }
  13843. if (!done)
  13844. {
  13845. if (maxRows)
  13846. {
  13847. StringBuffer s;
  13848. BuildCtx switchctx(funcctx);
  13849. switchctx.addQuotedCompound("switch (row)");
  13850. unsigned row;
  13851. for (row = 0; row < maxRows; row++)
  13852. {
  13853. BuildCtx casectx(switchctx);
  13854. casectx.addQuotedCompound(s.clear().append("case ").append(row).append(":"));
  13855. IHqlExpression * cur = values->queryChild(row);
  13856. OwnedHqlExpr rowValue = createRow(no_createrow, LINK(cur));
  13857. buildAssign(casectx, self, rowValue);
  13858. buildReturnRecordSize(casectx, selfCursor);
  13859. }
  13860. }
  13861. funcctx.addReturn(queryZero());
  13862. }
  13863. OwnedHqlExpr rowsExpr = getSizetConstant(maxRows);
  13864. doBuildUnsignedFunction(instance->startctx, "numRows", rowsExpr);
  13865. if (!values->isConstant())
  13866. doBuildBoolFunction(instance->startctx, "isConstant", false);
  13867. buildInstanceSuffix(instance);
  13868. return instance->getBoundActivity();
  13869. }
  13870. //---------------------------------------------------------------------------
  13871. void HqlCppTranslator::buildHTTPtoXml(BuildCtx & ctx)
  13872. {
  13873. BuildCtx funcctx(ctx);
  13874. //virtual void toXML(const byte * self, StringBuffer & out) = 0;
  13875. funcctx.addQuotedCompound("virtual void toXML(const byte *, IXmlWriter & out)");
  13876. }
  13877. //---------------------------------------------------------------------------
  13878. void HqlCppTranslator::buildSOAPtoXml(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * transform, IHqlExpression * selSeq)
  13879. {
  13880. BuildCtx funcctx(ctx);
  13881. //virtual void toXML(const byte * self, StringBuffer & out) = 0;
  13882. if (dataset)
  13883. {
  13884. funcctx.addQuotedCompound("virtual void toXML(const byte * left, IXmlWriter & out)");
  13885. if (transform->getOperator() == no_newtransform)
  13886. bindTableCursor(funcctx, dataset, "left");
  13887. else
  13888. bindTableCursor(funcctx, dataset, "left", no_left, selSeq);
  13889. }
  13890. else
  13891. funcctx.addQuotedCompound("virtual void toXML(const byte *, IXmlWriter & out)");
  13892. // Bind left to "left" and right to RIGHT
  13893. HqlExprArray assigns;
  13894. filterExpandAssignments(funcctx, NULL, assigns, transform);
  13895. OwnedHqlExpr self = getSelf(transform);
  13896. buildXmlSerialize(funcctx, transform->queryRecord(), self, &assigns);
  13897. }
  13898. IHqlExpression * HqlCppTranslator::associateLocalFailure(BuildCtx & ctx, const char * exceptionName)
  13899. {
  13900. OwnedHqlExpr activeFailMarker = createAttribute(activeFailureAtom);
  13901. OwnedHqlExpr activeFailVariable = createVariable(exceptionName, makeBoolType());
  13902. ctx.associateExpr(activeFailMarker, activeFailVariable);
  13903. return activeFailVariable;
  13904. }
  13905. void HqlCppTranslator::validateExprScope(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * expr, const char * opName, const char * argName)
  13906. {
  13907. if (dataset && exprReferencesDataset(expr, dataset) && !resolveSelectorDataset(ctx, dataset))
  13908. throwError2(HQLERR_OpArgDependsDataset, opName, argName);
  13909. }
  13910. ABoundActivity * HqlCppTranslator::doBuildActivitySOAP(BuildCtx & ctx, IHqlExpression * expr, bool isSink, bool isRoot)
  13911. {
  13912. ThorActivityKind tak;
  13913. const char * helper;
  13914. unsigned firstArg = 0;
  13915. IHqlExpression * dataset = NULL;
  13916. Owned<ABoundActivity> boundDataset;
  13917. IHqlExpression * selSeq = querySelSeq(expr);
  13918. if (expr->getOperator() == no_newsoapcall)
  13919. {
  13920. if (isSink)
  13921. {
  13922. tak = TAKsoap_rowaction;
  13923. helper = "SoapAction";
  13924. }
  13925. else
  13926. {
  13927. tak = TAKsoap_rowdataset;
  13928. helper = "SoapCall";
  13929. }
  13930. }
  13931. else
  13932. {
  13933. if (isSink)
  13934. {
  13935. tak = TAKsoap_datasetaction;
  13936. helper = "SoapAction";
  13937. }
  13938. else
  13939. {
  13940. tak = TAKsoap_datasetdataset;
  13941. helper = "SoapCall";
  13942. }
  13943. dataset = expr->queryChild(0);
  13944. boundDataset.setown(buildCachedActivity(ctx, dataset));
  13945. firstArg = 1;
  13946. }
  13947. StringBuffer s;
  13948. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, tak, expr, helper);
  13949. //-----------------
  13950. buildActivityFramework(instance, isRoot);
  13951. buildInstancePrefix(instance);
  13952. IHqlExpression * hosts = expr->queryChild(firstArg);
  13953. IHqlExpression * service = expr->queryChild(firstArg+1);
  13954. //Because of scope handling in parser it is possible for these expressions to be unexpectedly dependent on the dataset
  13955. const char * opText = getOpString(expr->getOperator());
  13956. validateExprScope(instance->startctx, dataset, hosts, opText, "host url");
  13957. validateExprScope(instance->startctx, dataset, service, opText, "service");
  13958. //virtual const char * queryHosts() = 0;
  13959. doBuildVarStringFunction(instance->startctx, "queryHosts", hosts);
  13960. //virtual const char * queryService() = 0;
  13961. doBuildVarStringFunction(instance->startctx, "queryService", service);
  13962. //virtual void toXML(const byte * self, StringBuffer & out) = 0;
  13963. buildSOAPtoXml(instance->startctx, dataset, expr->queryChild(firstArg+3), selSeq);
  13964. //virtual const char * queryOutputIteratorPath()
  13965. IHqlExpression * separator = expr->queryProperty(separatorAtom);
  13966. if (separator)
  13967. doBuildVarStringFunction(instance->startctx, "queryOutputIteratorPath", separator->queryChild(0));
  13968. //virtual const char * queryHeader()
  13969. //virtual const char * queryFooter()
  13970. IHqlExpression * header = expr->queryProperty(headerAtom);
  13971. if (header)
  13972. {
  13973. doBuildVarStringFunction(instance->startctx, "queryHeader", header->queryChild(0));
  13974. doBuildVarStringFunction(instance->startctx, "queryFooter", header->queryChild(1));
  13975. }
  13976. IHqlExpression * action = expr->queryProperty(soapActionAtom);
  13977. if (action)
  13978. doBuildVarStringFunction(instance->startctx, "querySoapAction", action->queryChild(0));
  13979. IHqlExpression * httpHeader = expr->queryProperty(httpHeaderAtom);
  13980. if (httpHeader)
  13981. {
  13982. doBuildVarStringFunction(instance->startctx, "queryHttpHeaderName", httpHeader->queryChild(0));
  13983. doBuildVarStringFunction(instance->startctx, "queryHttpHeaderValue", httpHeader->queryChild(1));
  13984. }
  13985. IHqlExpression * proxyAddress = expr->queryProperty(proxyAddressAtom);
  13986. if (proxyAddress)
  13987. {
  13988. doBuildVarStringFunction(instance->startctx, "queryProxyAddress", proxyAddress->queryChild(0));
  13989. }
  13990. IHqlExpression * namespaceAttr = expr->queryProperty(namespaceAtom);
  13991. IHqlExpression * responseAttr = expr->queryProperty(responseAtom);
  13992. //virtual unsigned getFlags()
  13993. {
  13994. StringBuffer flags;
  13995. if (expr->hasProperty(groupAtom))
  13996. flags.append("|SOAPFgroup");
  13997. if (expr->hasProperty(onFailAtom))
  13998. flags.append("|SOAPFonfail");
  13999. if (expr->hasProperty(logAtom))
  14000. flags.append("|SOAPFlog");
  14001. if (expr->hasProperty(trimAtom))
  14002. flags.append("|SOAPFtrim");
  14003. if (expr->hasProperty(literalAtom))
  14004. flags.append("|SOAPFliteral");
  14005. if (namespaceAttr)
  14006. flags.append("|SOAPFnamespace");
  14007. if (expr->hasProperty(encodingAtom))
  14008. flags.append("|SOAPFencoding");
  14009. if (responseAttr && responseAttr->hasProperty(noTrimAtom))
  14010. flags.append("|SOAPFpreserveSpace");
  14011. if (flags.length())
  14012. doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);
  14013. }
  14014. //virtual unsigned numParallelThreads()
  14015. doBuildUnsignedFunction(instance->classctx, "numParallelThreads", queryPropertyChild(expr, parallelAtom, 0));
  14016. //virtual unsigned numRecordsPerBatch()
  14017. doBuildUnsignedFunction(instance->classctx, "numRecordsPerBatch", queryPropertyChild(expr, mergeAtom, 0));
  14018. //virtual int numRetries()
  14019. doBuildSignedFunction(instance->classctx, "numRetries", queryPropertyChild(expr, retryAtom, 0));
  14020. //virtual unsigned getTimeout()
  14021. doBuildUnsignedFunction(instance->classctx, "getTimeout", queryPropertyChild(expr, timeoutAtom, 0));
  14022. //virtual unsigned getTimeLimit()
  14023. doBuildUnsignedFunction(instance->classctx, "getTimeLimit", queryPropertyChild(expr, timeLimitAtom, 0));
  14024. if (namespaceAttr)
  14025. {
  14026. doBuildVarStringFunction(instance->startctx, "queryNamespaceName", namespaceAttr->queryChild(0));
  14027. if (namespaceAttr->queryChild(1))
  14028. doBuildVarStringFunction(instance->startctx, "queryNamespaceVar", namespaceAttr->queryChild(1));
  14029. }
  14030. if (!isSink)
  14031. {
  14032. //virtual IXmlToRowTransformer * queryTransformer()
  14033. bool usesContents = false;
  14034. doBuildXmlReadMember(*instance, expr, "queryInputTransformer", usesContents);
  14035. if (usesContents)
  14036. throwError(HQLERR_ContentsInSoapCall);
  14037. //virtual const char * queryInputIteratorPath()
  14038. IHqlExpression * xpath = expr->queryProperty(xpathAtom);
  14039. if (xpath)
  14040. doBuildVarStringFunction(instance->classctx, "queryInputIteratorPath", xpath->queryChild(0));
  14041. IHqlExpression * onFail = expr->queryProperty(onFailAtom);
  14042. if (onFail)
  14043. {
  14044. IHqlExpression * onFailTransform = onFail->queryChild(0);
  14045. if (onFailTransform->isTransform())
  14046. assertex(recordTypesMatch(expr, onFailTransform));
  14047. //virtual unsigned onFailTransform(ARowBuilder & crSelf, const void * _left, IException * e)
  14048. BuildCtx onFailCtx(instance->startctx);
  14049. onFailCtx.addQuotedCompound("virtual unsigned onFailTransform(ARowBuilder & crSelf, const void * _left, IException * except)");
  14050. ensureRowAllocated(onFailCtx, "crSelf");
  14051. associateLocalFailure(onFailCtx, "except");
  14052. buildTransformBody(onFailCtx, onFailTransform, dataset, NULL, expr, selSeq);
  14053. }
  14054. }
  14055. buildInstanceSuffix(instance);
  14056. if (boundDataset)
  14057. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  14058. if (isSink)
  14059. return NULL;
  14060. return instance->getBoundActivity();
  14061. }
  14062. //---------------------------------------------------------------------------
  14063. ABoundActivity * HqlCppTranslator::doBuildActivityHTTP(BuildCtx & ctx, IHqlExpression * expr, bool isSink, bool isRoot)
  14064. {
  14065. ThorActivityKind tak;
  14066. const char * helper = "HttpCall";
  14067. unsigned firstArg = 0;
  14068. IHqlExpression * dataset = NULL;
  14069. Owned<ABoundActivity> boundDataset;
  14070. IHqlExpression * selSeq = querySelSeq(expr);
  14071. assertex(!isSink);
  14072. tak = TAKhttp_rowdataset;
  14073. StringBuffer s;
  14074. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, tak, expr, helper);
  14075. //-----------------
  14076. buildActivityFramework(instance, isRoot);
  14077. buildInstancePrefix(instance);
  14078. //virtual const char * queryHosts() = 0;
  14079. doBuildVarStringFunction(instance->startctx, "queryHosts", expr->queryChild(firstArg));
  14080. //virtual const char * queryService() = 0;
  14081. doBuildVarStringFunction(instance->startctx, "queryService", expr->queryChild(firstArg+1));
  14082. //virtual const char * queryAcceptType() = 0;
  14083. doBuildVarStringFunction(instance->startctx, "queryAcceptType", expr->queryChild(firstArg+2));
  14084. //virtual void toXML(const byte * self, StringBuffer & out) = 0;
  14085. buildHTTPtoXml(instance->startctx);
  14086. //virtual const char * queryOutputIteratorPath()
  14087. IHqlExpression * separator = expr->queryProperty(separatorAtom);
  14088. if (separator)
  14089. doBuildVarStringFunction(instance->startctx, "queryOutputIteratorPath", separator->queryChild(0));
  14090. IHqlExpression * namespaceAttr = expr->queryProperty(namespaceAtom);
  14091. //virtual unsigned getFlags()
  14092. {
  14093. StringBuffer flags;
  14094. if (expr->hasProperty(groupAtom))
  14095. flags.append("|SOAPFgroup");
  14096. if (expr->hasProperty(onFailAtom))
  14097. flags.append("|SOAPFonfail");
  14098. if (expr->hasProperty(logAtom))
  14099. flags.append("|SOAPFlog");
  14100. if (expr->hasProperty(trimAtom))
  14101. flags.append("|SOAPFtrim");
  14102. if (expr->hasProperty(literalAtom))
  14103. flags.append("|SOAPFliteral");
  14104. if (namespaceAttr)
  14105. flags.append("|SOAPFnamespace");
  14106. if (flags.length())
  14107. doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);
  14108. }
  14109. //virtual unsigned numParallelThreads()
  14110. doBuildUnsignedFunction(instance->classctx, "numParallelThreads", queryPropertyChild(expr, parallelAtom, 0));
  14111. //virtual unsigned numRecordsPerBatch()
  14112. doBuildUnsignedFunction(instance->classctx, "numRecordsPerBatch", queryPropertyChild(expr, mergeAtom, 0));
  14113. //virtual int numRetries()
  14114. doBuildSignedFunction(instance->classctx, "numRetries", queryPropertyChild(expr, retryAtom, 0));
  14115. //virtual unsigned getTimeout()
  14116. doBuildUnsignedFunction(instance->classctx, "getTimeout", queryPropertyChild(expr, timeoutAtom, 0));
  14117. //virtual unsigned getTimeLimit()
  14118. doBuildUnsignedFunction(instance->classctx, "getTimeLimit", queryPropertyChild(expr, timeLimitAtom, 0));
  14119. if (namespaceAttr)
  14120. {
  14121. doBuildVarStringFunction(instance->startctx, "queryNamespaceName", namespaceAttr->queryChild(0));
  14122. if (namespaceAttr->queryChild(1))
  14123. doBuildVarStringFunction(instance->startctx, "queryNamespaceVar", namespaceAttr->queryChild(1));
  14124. }
  14125. if (!isSink)
  14126. {
  14127. //virtual IXmlToRowTransformer * queryTransformer()
  14128. bool usesContents = false;
  14129. doBuildXmlReadMember(*instance, expr, "queryInputTransformer", usesContents);
  14130. if (usesContents)
  14131. throwError(HQLERR_ContentsInSoapCall);
  14132. //virtual const char * queryInputIteratorPath()
  14133. IHqlExpression * xpath = expr->queryProperty(xpathAtom);
  14134. if (xpath)
  14135. doBuildVarStringFunction(instance->classctx, "queryInputIteratorPath", xpath->queryChild(0));
  14136. IHqlExpression * onFail = expr->queryProperty(onFailAtom);
  14137. if (onFail)
  14138. {
  14139. //virtual unsigned onFailTransform(ARowBuilder & crSelf, const void * _left, IException * e)
  14140. BuildCtx onFailCtx(instance->startctx);
  14141. onFailCtx.addQuotedCompound("virtual unsigned onFailTransform(ARowBuilder & crSelf, const void * _left, IException * except)");
  14142. ensureRowAllocated(onFailCtx, "crSelf");
  14143. associateLocalFailure(onFailCtx, "except");
  14144. buildTransformBody(onFailCtx, onFail->queryChild(0), dataset, NULL, expr, selSeq);
  14145. }
  14146. }
  14147. buildInstanceSuffix(instance);
  14148. if (boundDataset)
  14149. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  14150. if (isSink)
  14151. return NULL;
  14152. return instance->getBoundActivity();
  14153. }
  14154. //---------------------------------------------------------------------------
  14155. IHqlExpression * HqlCppTranslator::doBuildRegexCompileInstance(BuildCtx & ctx, IHqlExpression * pattern, bool isUnicode, bool isCaseSensitive)
  14156. {
  14157. OwnedHqlExpr searchKey = createAttribute(_regexInstance_Atom, LINK(pattern), createConstant(isUnicode), createConstant(isCaseSensitive));
  14158. HqlExprAssociation * match = ctx.queryMatchExpr(searchKey);
  14159. if (match)
  14160. return match->queryExpr();
  14161. BuildCtx * initCtx = &ctx;
  14162. BuildCtx * declareCtx = &ctx;
  14163. if (pattern->isConstant())
  14164. getInvariantMemberContext(ctx, &declareCtx, &initCtx, true, false);
  14165. StringBuffer tempName;
  14166. getUniqueId(tempName.append("regex"));
  14167. ITypeInfo * type = makeClassType(isUnicode ? "rtlCompiledUStrRegex" : "rtlCompiledStrRegex");
  14168. OwnedHqlExpr regexInstance = createVariable(tempName.str(), type);
  14169. declareCtx->addDeclare(regexInstance);
  14170. HqlExprArray args;
  14171. args.append(*LINK(regexInstance));
  14172. args.append(*LINK(pattern));
  14173. args.append(*createConstant(isCaseSensitive));
  14174. _ATOM func = isUnicode ? regexNewSetUStrPatternAtom : regexNewSetStrPatternAtom;
  14175. buildFunctionCall(*initCtx, func, args);
  14176. declareCtx->associateExpr(searchKey, regexInstance);
  14177. return regexInstance;
  14178. }
  14179. IHqlExpression * HqlCppTranslator::doBuildRegexFindInstance(BuildCtx & ctx, IHqlExpression * compiled, IHqlExpression * search, bool cloneSearch)
  14180. {
  14181. OwnedHqlExpr searchKey = createAttribute(_regexFindInstance_Atom, LINK(compiled), LINK(search), createConstant(cloneSearch));
  14182. HqlExprAssociation * match = ctx.queryMatchExpr(searchKey);
  14183. if (match)
  14184. return match->queryExpr();
  14185. bool isUnicode = isUnicodeType(search->queryType());
  14186. StringBuffer tempName;
  14187. getUniqueId(tempName.append("fi"));
  14188. ITypeInfo * type = makeClassType(isUnicode ? "rtlUStrRegexFindInstance" : "rtlStrRegexFindInstance");
  14189. OwnedHqlExpr regexInstance = createVariable(tempName.str(), type);
  14190. ctx.addDeclare(regexInstance);
  14191. //Would be better if I allowed classes in my external functions instead of faking booleans
  14192. OwnedHqlExpr castCompiled = createValue(no_typetransfer, makeBoolType(), LINK(compiled));
  14193. HqlExprArray args;
  14194. args.append(*LINK(regexInstance));
  14195. args.append(*createTranslated(castCompiled));
  14196. args.append(*LINK(search));
  14197. if (!isUnicode)
  14198. args.append(*createConstant(cloneSearch));
  14199. _ATOM func = isUnicode ? regexNewUStrFindAtom : regexNewStrFindAtom;
  14200. buildFunctionCall(ctx, func, args);
  14201. ctx.associateExpr(searchKey, regexInstance);
  14202. return regexInstance;
  14203. }
  14204. void HqlCppTranslator::doBuildNewRegexFindReplace(BuildCtx & ctx, const CHqlBoundTarget * target, IHqlExpression * expr, CHqlBoundExpr * bound)
  14205. {
  14206. CHqlBoundExpr boundMatch;
  14207. if (ctx.getMatchExpr(expr, boundMatch))
  14208. {
  14209. if (bound)
  14210. bound->set(boundMatch);
  14211. else
  14212. assign(ctx, *target, boundMatch);
  14213. return;
  14214. }
  14215. IHqlExpression * pattern = expr->queryChild(0);
  14216. IHqlExpression * search = expr->queryChild(1);
  14217. bool isUnicode = isUnicodeType(search->queryType());
  14218. IHqlExpression * compiled = doBuildRegexCompileInstance(ctx, pattern, isUnicode, !expr->hasProperty(noCaseAtom));
  14219. if (expr->getOperator() == no_regex_replace)
  14220. {
  14221. HqlExprArray args;
  14222. args.append(*LINK(compiled));
  14223. args.append(*LINK(search));
  14224. args.append(*LINK(expr->queryChild(2)));
  14225. _ATOM func = isUnicode ? regexNewUStrReplaceXAtom : regexNewStrReplaceXAtom;
  14226. OwnedHqlExpr call = bindFunctionCall(func, args);
  14227. //Need to associate???
  14228. buildExprOrAssign(ctx, target, call, bound);
  14229. return;
  14230. }
  14231. // Because the search instance is created locally, the search parameter is always going to be valid
  14232. // as long as the find instance. Only exception could be if call created a temporary class instance.
  14233. bool cloneSearch = false;
  14234. IHqlExpression * findInstance = doBuildRegexFindInstance(ctx, compiled, search, cloneSearch);
  14235. if(expr->queryType() == queryBoolType())
  14236. {
  14237. HqlExprArray args;
  14238. args.append(*LINK(findInstance));
  14239. _ATOM func= isUnicode ? regexNewUStrFoundAtom : regexNewStrFoundAtom;
  14240. OwnedHqlExpr call = bindFunctionCall(func, args);
  14241. buildExprOrAssign(ctx, target, call, bound);
  14242. }
  14243. else
  14244. {
  14245. HqlExprArray args;
  14246. args.append(*LINK(findInstance));
  14247. args.append(*LINK(expr->queryChild(2)));
  14248. _ATOM func= isUnicode ? regexNewUStrFoundXAtom : regexNewStrFoundXAtom;
  14249. OwnedHqlExpr call = bindFunctionCall(func, args);
  14250. buildExprOrAssign(ctx, target, call, bound);
  14251. }
  14252. }
  14253. void HqlCppTranslator::doBuildExprRegexFindReplace(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & bound)
  14254. {
  14255. doBuildNewRegexFindReplace(ctx, NULL, expr, &bound);
  14256. }
  14257. void HqlCppTranslator::doBuildAssignRegexFindReplace(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr)
  14258. {
  14259. doBuildNewRegexFindReplace(ctx, &target, expr, NULL);
  14260. }
  14261. //---------------------------------------------------------------------------
  14262. //-- no_null [DATASET] --
  14263. ABoundActivity * HqlCppTranslator::doBuildActivityNull(BuildCtx & ctx, IHqlExpression * expr, bool isRoot)
  14264. {
  14265. StringBuffer s;
  14266. ThorActivityKind kind = expr->isAction() ? TAKemptyaction : TAKnull;
  14267. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, kind, expr,"Null");
  14268. if (options.minimizeActivityClasses)
  14269. instance->setImplementationClass(newNullArgAtom);
  14270. //-----------------
  14271. buildActivityFramework(instance, isRoot);
  14272. buildInstancePrefix(instance);
  14273. buildInstanceSuffix(instance);
  14274. return instance->getBoundActivity();
  14275. }
  14276. //---------------------------------------------------------------------------
  14277. ABoundActivity * HqlCppTranslator::doBuildActivityAction(BuildCtx & ctx, IHqlExpression * expr, bool isRoot, bool expandChildren)
  14278. {
  14279. StringBuffer s;
  14280. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKsideeffect, expr,"Action");
  14281. //-----------------
  14282. instance->graphLabel.set(getOpString(expr->getOperator())); // label node as "fail"
  14283. buildActivityFramework(instance, isRoot);
  14284. buildInstancePrefix(instance);
  14285. BuildCtx funcctx(instance->startctx);
  14286. funcctx.addQuotedCompoundOpt("virtual void action()");
  14287. if (expandChildren)
  14288. {
  14289. unsigned numChildren = expr->numChildren();
  14290. for (unsigned idx=1; idx < numChildren; idx++)
  14291. {
  14292. IHqlExpression * cur = expr->queryChild(idx);
  14293. if (!cur->isAttribute())
  14294. buildStmt(funcctx, cur);
  14295. }
  14296. }
  14297. else
  14298. {
  14299. buildStmt(funcctx, expr);
  14300. }
  14301. buildInstanceSuffix(instance);
  14302. return instance->getBoundActivity();
  14303. }
  14304. //---------------------------------------------------------------------------
  14305. // if (ctx->currentWorkflowId() == number)
  14306. // doCode();
  14307. void HqlCppTranslator::buildWorkflowItem(BuildCtx & ctx, IHqlStmt * switchStmt, unsigned wfid, IHqlExpression * expr)
  14308. {
  14309. OwnedHqlExpr value = getSizetConstant(wfid);
  14310. BuildCtx condctx(ctx);
  14311. IHqlStmt * caseStmt = condctx.addCase(switchStmt, value);
  14312. //Unwind the statement list to prevent very deep recursion.
  14313. HqlExprArray exprs;
  14314. unwindCommaCompound(exprs, expr);
  14315. ForEachItemIn(i, exprs)
  14316. buildStmt(condctx, &exprs.item(i));
  14317. if (caseStmt->numChildren() == 0)
  14318. condctx.addGroup(); // ensure a break statement is generated...
  14319. }
  14320. void HqlCppTranslator::buildWorkflowPersistCheck(BuildCtx & ctx, IHqlExpression * expr)
  14321. {
  14322. IHqlExpression * original = queryPropertyChild(expr, _original_Atom, 0);
  14323. OwnedHqlExpr resultName = ::createResultName(queryPropertyChild(expr, namedAtom, 0));
  14324. resultName.setown(ensureExprType(resultName, unknownVarStringType));
  14325. IHqlExpression * filesRead = expr->queryProperty(_files_Atom);
  14326. DependenciesUsed dependencies(true);
  14327. if (filesRead)
  14328. {
  14329. ForEachChild(i, filesRead)
  14330. dependencies.tablesRead.append(*getNormalizedFilename(filesRead->queryChild(i)));
  14331. }
  14332. IHqlExpression * resultsRead = expr->queryProperty(_results_Atom);
  14333. if (resultsRead)
  14334. unwindChildren(dependencies.resultsRead, resultsRead);
  14335. unsigned crc = getExpressionCRC(original) + PERSIST_VERSION;
  14336. OwnedHqlExpr crcVal = createConstant((__int64)crc);
  14337. OwnedHqlExpr crcExpr = calculatePersistInputCrc(ctx, dependencies);
  14338. HqlExprArray args;
  14339. args.append(*LINK(resultName));
  14340. args.append(*LINK(crcVal));
  14341. args.append(*LINK(crcExpr));
  14342. args.append(*createConstant(expr->hasProperty(fileAtom)));
  14343. buildFunctionCall(ctx, returnPersistVersionAtom, args);
  14344. }
  14345. void HqlCppTranslator::buildWorkflow(WorkflowArray & workflow)
  14346. {
  14347. BuildCtx classctx(*code, goAtom);
  14348. classctx.addQuotedCompound("struct MyEclProcess : public EclProcess", ";");
  14349. BuildCtx performctx(classctx);
  14350. performctx.addQuotedCompound("virtual int perform(IGlobalCodeContext * gctx, unsigned wfid)");
  14351. performctx.addQuoted("ICodeContext * ctx;");
  14352. performctx.addQuoted("ctx = gctx->queryCodeContext();");
  14353. performctx.associateExpr(globalContextMarkerExpr, globalContextMarkerExpr);
  14354. performctx.associateExpr(codeContextMarkerExpr, codeContextMarkerExpr);
  14355. OwnedHqlExpr function = createQuoted("wfid", LINK(unsignedType));
  14356. BuildCtx switchctx(performctx);
  14357. IHqlStmt * switchStmt = switchctx.addSwitch(function);
  14358. optimizePersists(workflow);
  14359. ForEachItemIn(idx, workflow)
  14360. {
  14361. WorkflowItem & action = workflow.item(idx);
  14362. HqlExprArray & exprs = action.queryExprs();
  14363. bool isEmpty = exprs.ordinality() == 0;
  14364. if (exprs.ordinality() == 1 && (exprs.item(0).getOperator() == no_workflow_action))
  14365. isEmpty = true;
  14366. if (!isEmpty)
  14367. {
  14368. OwnedHqlExpr expr = createActionList(action.queryExprs());
  14369. unsigned wfid = action.queryWfid();
  14370. IHqlExpression * persistAttr = expr->queryProperty(_workflowPersist_Atom);
  14371. if (persistAttr)
  14372. {
  14373. if (!options.freezePersists)
  14374. {
  14375. HqlExprArray args2;
  14376. unwindChildren(args2, expr);
  14377. OwnedHqlExpr setResult = createSetResult(args2);
  14378. buildWorkflowItem(switchctx, switchStmt, wfid, setResult);
  14379. }
  14380. }
  14381. else
  14382. buildWorkflowItem(switchctx, switchStmt, wfid, expr);
  14383. }
  14384. }
  14385. OwnedHqlExpr returnExpr = getSizetConstant(maxSequence);
  14386. performctx.addReturn(returnExpr);
  14387. }
  14388. //---------------------------------------------------------------------------
  14389. void HqlCppTranslator::doBuildStmtWait(BuildCtx & ctx, IHqlExpression * expr)
  14390. {
  14391. throwError(HQLERR_WaitNotSupported);
  14392. }
  14393. void HqlCppTranslator::doBuildStmtNotify(BuildCtx & ctx, IHqlExpression * expr)
  14394. {
  14395. IHqlExpression * event = expr->queryChild(0);
  14396. IHqlExpression * target = queryRealChild(expr, 1);
  14397. HqlExprArray args;
  14398. args.append(*LINK(event->queryChild(0)));
  14399. args.append(*LINK(event->queryChild(1)));
  14400. if (target)
  14401. {
  14402. args.append(*LINK(target));
  14403. buildFunctionCall(ctx, doNotifyTargetAtom, args);
  14404. }
  14405. else
  14406. buildFunctionCall(ctx, doNotifyAtom, args);
  14407. }
  14408. //---------------------------------------------------------------------------
  14409. // no_thorresult
  14410. // no_thorremoteresult
  14411. ABoundActivity * HqlCppTranslator::doBuildActivitySetResult(BuildCtx & ctx, IHqlExpression * expr, bool isRoot)
  14412. {
  14413. IHqlExpression * sequence = queryPropertyChild(expr, sequenceAtom, 0);
  14414. IHqlExpression * name = queryPropertyChild(expr, namedAtom, 0);
  14415. IHqlExpression * persist = expr->queryProperty(_workflowPersist_Atom);
  14416. HqlExprAttr dataset, row, attribute;
  14417. if (expr->getOperator() == no_extractresult)
  14418. {
  14419. row.set(expr->queryChild(0));
  14420. dataset.set(row->queryNormalizedSelector(true));
  14421. attribute.set(expr->queryChild(1));
  14422. }
  14423. else
  14424. {
  14425. if (!options.canGenerateSimpleAction)
  14426. {
  14427. row.setown(createDataset(no_null, LINK(queryNullRecord()), NULL));
  14428. dataset.set(row);
  14429. }
  14430. attribute.set(expr->queryChild(0));
  14431. }
  14432. // splitSetResultValue(dataset, row, attribute, value);
  14433. if (attribute->isAction())
  14434. {
  14435. //This code is decidedly strange - as far as I can see there is no explicit link between this activity and
  14436. //the child root activity, it works because they happen to be generated in the same graph
  14437. switch (attribute->getOperator())
  14438. {
  14439. case no_output:
  14440. buildRootActivity(ctx, attribute);
  14441. break;
  14442. default:
  14443. buildStmt(ctx, attribute);
  14444. break;
  14445. }
  14446. attribute.set(queryBoolExpr(true));
  14447. }
  14448. Owned<ABoundActivity> boundDataset;
  14449. ThorActivityKind kind = row ? TAKremoteresult : TAKsimpleaction;
  14450. if (row)
  14451. boundDataset.setown(buildCachedActivity(ctx, row));
  14452. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, kind, expr, kind == TAKremoteresult ? "RemoteResult" : "Action");
  14453. if (sequence)
  14454. {
  14455. StringBuffer graphLabel;
  14456. graphLabel.append("Store\n");
  14457. getStoredDescription(graphLabel, sequence, name, true);
  14458. instance->graphLabel.set(graphLabel.str());
  14459. }
  14460. buildActivityFramework(instance, isRoot && !isInternalSeq(sequence));
  14461. buildInstancePrefix(instance);
  14462. noteResultDefined(ctx, instance, sequence, name, isRoot);
  14463. if (attribute->isDatarow())
  14464. attribute.setown(::ensureSerialized(attribute));
  14465. if (kind == TAKremoteresult)
  14466. {
  14467. doBuildSequenceFunc(instance->classctx, sequence, true);
  14468. BuildCtx sendctx(instance->startctx);
  14469. sendctx.addQuotedCompound("virtual void sendResult(const void * _self)");
  14470. sendctx.addQuoted("const unsigned char * self = (const unsigned char *)_self;");
  14471. if (dataset->isDatarow())
  14472. {
  14473. OwnedHqlExpr bound = createVariable("self", makeRowReferenceType(dataset));
  14474. bindRow(sendctx, dataset, bound);
  14475. }
  14476. else
  14477. bindTableCursor(sendctx, dataset, "self");
  14478. buildSetResultInfo(sendctx, expr, attribute, NULL, (persist != NULL), false);
  14479. }
  14480. else
  14481. {
  14482. BuildCtx sendctx(instance->startctx);
  14483. sendctx.addQuotedCompound("virtual void action()");
  14484. buildSetResultInfo(sendctx, expr, attribute, NULL, (persist != NULL), false);
  14485. }
  14486. buildInstanceSuffix(instance);
  14487. if (boundDataset)
  14488. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  14489. associateRemoteResult(*instance, sequence, name);
  14490. return instance->getBoundActivity();
  14491. }
  14492. //---------------------------------------------------------------------------
  14493. //-- no_distribution
  14494. static void getInterfaceName(StringBuffer & name, ITypeInfo * type)
  14495. {
  14496. switch (type->getTypeCode())
  14497. {
  14498. case type_boolean:
  14499. name.append("IBoolDistributionTable");
  14500. break;
  14501. case type_real:
  14502. name.append("IRealDistributionTable");
  14503. break;
  14504. case type_string:
  14505. case type_data:
  14506. case type_qstring:
  14507. assertex(type->getSize() != UNKNOWN_LENGTH);
  14508. name.append("IStringDistributionTable");
  14509. break;
  14510. case type_int:
  14511. case type_swapint:
  14512. case type_packedint:
  14513. if (type->isSigned())
  14514. {
  14515. if (type->getSize() == 8)
  14516. name.append("IInt64DistributionTable");
  14517. else
  14518. name.append("IIntDistributionTable");
  14519. }
  14520. else
  14521. {
  14522. if (type->getSize() == 8)
  14523. name.append("IUInt64DistributionTable");
  14524. else
  14525. name.append("IUIntDistributionTable");
  14526. }
  14527. break;
  14528. default:
  14529. UNIMPLEMENTED;
  14530. }
  14531. }
  14532. static bool expandFieldName(StringBuffer & s, IHqlExpression * e)
  14533. {
  14534. if (e->getOperator() == no_select)
  14535. {
  14536. if (expandFieldName(s, e->queryChild(0)))
  14537. s.append('.');
  14538. const char * name = e->queryChild(1)->queryName()->str();
  14539. s.appendLower(strlen(name), name);
  14540. return true;
  14541. }
  14542. return false;
  14543. }
  14544. void HqlCppTranslator::doBuildDistributionClearFunc(BuildCtx & ctx, IHqlExpression * dataset, HqlExprArray & selects)
  14545. {
  14546. StringBuffer s, func;
  14547. BuildCtx funcctx(ctx);
  14548. funcctx.addQuotedCompound("virtual void clearAggregate(IDistributionTable * * tables)");
  14549. ForEachItemIn(idx, selects)
  14550. {
  14551. IHqlExpression * original = &selects.item(idx);
  14552. ITypeInfo * type = original->queryType();
  14553. getInterfaceName(func.clear().append("create"), type);
  14554. s.clear().append("tables[").append(idx).append("] = ").append(func).append("(\"");
  14555. expandFieldName(s, original);
  14556. s.append("\", ").append(type->getSize()).append(");");
  14557. funcctx.addQuoted(s);
  14558. }
  14559. }
  14560. void HqlCppTranslator::doBuildDistributionNextFunc(BuildCtx & ctx, IHqlExpression * dataset, HqlExprArray & selects)
  14561. {
  14562. StringBuffer s;
  14563. BuildCtx funcctx(ctx);
  14564. funcctx.addQuotedCompound("virtual void process(IDistributionTable * * tables, const void * _src)");
  14565. funcctx.addQuoted("unsigned char * src = (unsigned char *) _src;");
  14566. bindTableCursor(funcctx, dataset, "src");
  14567. ForEachItemIn(idx, selects)
  14568. {
  14569. IHqlExpression * original = &selects.item(idx);
  14570. ITypeInfo * type = original->queryType();
  14571. CHqlBoundExpr bound;
  14572. buildExpr(funcctx, original, bound);
  14573. s.clear().append("((");
  14574. getInterfaceName(s, type);
  14575. s.append(" *)tables[").append(idx).append("])->noteValue(");
  14576. switch (type->getTypeCode())
  14577. {
  14578. case type_string:
  14579. case type_data:
  14580. case type_qstring:
  14581. {
  14582. if (bound.length)
  14583. generateExprCpp(s, bound.length);
  14584. else
  14585. s.append(type->getSize());
  14586. s.append(",");
  14587. OwnedHqlExpr addr = getElementPointer(bound.expr);
  14588. generateExprCpp(s, addr);
  14589. }
  14590. break;
  14591. default:
  14592. generateExprCpp(s, bound.expr);
  14593. break;
  14594. }
  14595. s.append(");");
  14596. funcctx.addQuoted(s);
  14597. }
  14598. }
  14599. void HqlCppTranslator::doBuildDistributionFunc(BuildCtx & funcctx, unsigned numFields, const char * action)
  14600. {
  14601. StringBuffer s;
  14602. s.clear().append("for (unsigned i=0;i<").append(numFields).append(";i++)");
  14603. funcctx.addQuotedCompound(s);
  14604. s.clear().append("tables[i]->").append(action).append(";");
  14605. funcctx.addQuoted(s);
  14606. }
  14607. void HqlCppTranslator::doBuildDistributionDestructFunc(BuildCtx & ctx, unsigned numFields)
  14608. {
  14609. BuildCtx funcctx(ctx);
  14610. funcctx.addQuotedCompound("virtual void destruct(IDistributionTable * * tables)");
  14611. doBuildDistributionFunc(funcctx, numFields, "Release()");
  14612. }
  14613. void HqlCppTranslator::doBuildDistributionSerializeFunc(BuildCtx & ctx, unsigned numFields)
  14614. {
  14615. BuildCtx funcctx(ctx);
  14616. funcctx.addQuotedCompound("virtual void serialize(IDistributionTable * * tables, MemoryBuffer & out)");
  14617. doBuildDistributionFunc(funcctx, numFields, "serialize(out)");
  14618. }
  14619. void HqlCppTranslator::doBuildDistributionMergeFunc(BuildCtx & ctx, unsigned numFields)
  14620. {
  14621. BuildCtx funcctx(ctx);
  14622. funcctx.addQuotedCompound("virtual void merge(IDistributionTable * * tables, MemoryBuffer & in)");
  14623. doBuildDistributionFunc(funcctx, numFields, "merge(in)");
  14624. }
  14625. void HqlCppTranslator::doBuildDistributionGatherFunc(BuildCtx & ctx, unsigned numFields)
  14626. {
  14627. BuildCtx funcctx(ctx);
  14628. funcctx.addQuotedCompound("virtual void gatherResult(IDistributionTable * * tables, StringBuffer & out)");
  14629. doBuildDistributionFunc(funcctx, numFields, "report(out)");
  14630. }
  14631. static void expandDistributionFields(IHqlExpression * record, HqlExprArray & selects, IHqlExpression * selector)
  14632. {
  14633. ForEachChild(idx, record)
  14634. {
  14635. IHqlExpression * cur = record->queryChild(idx);
  14636. switch (cur->getOperator())
  14637. {
  14638. case no_ifblock:
  14639. expandDistributionFields(cur->queryChild(1), selects, selector);
  14640. break;
  14641. case no_record:
  14642. expandDistributionFields(cur, selects, selector);
  14643. break;
  14644. case no_field:
  14645. {
  14646. OwnedHqlExpr selected = selector ? createSelectExpr(LINK(selector), LINK(cur)) : LINK(cur);
  14647. _ATOM name = cur->queryName();
  14648. if (cur->queryType()->getTypeCode() == type_row)
  14649. {
  14650. expandDistributionFields(cur->queryRecord(), selects, selected);
  14651. return;
  14652. }
  14653. selects.append(*selected.getClear());
  14654. break;
  14655. }
  14656. case no_attr:
  14657. case no_attr_expr:
  14658. case no_attr_link:
  14659. break;
  14660. default:
  14661. UNIMPLEMENTED;
  14662. break;
  14663. }
  14664. }
  14665. }
  14666. ABoundActivity * HqlCppTranslator::doBuildActivityDistribution(BuildCtx & ctx, IHqlExpression * expr, bool isRoot)
  14667. {
  14668. IHqlExpression * dataset = expr->queryChild(0);
  14669. IHqlExpression * fields = queryRealChild(expr, 1);
  14670. IHqlExpression * sequence = expr->queryProperty(sequenceAtom);
  14671. IHqlExpression * name = queryPropertyChild(expr, namedAtom, 0);
  14672. if (!sequence)
  14673. throwError(HQLERR_DistributionNoSequence);
  14674. useInclude("rtldistr.hpp");
  14675. Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
  14676. Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKdistribution, expr, "Distribution");
  14677. buildActivityFramework(instance, isRoot);
  14678. buildInstancePrefix(instance);
  14679. HqlExprArray selects;
  14680. if (fields)
  14681. unwindChildren(selects, fields);
  14682. else
  14683. expandDistributionFields(dataset->queryRecord(), selects, dataset);
  14684. unsigned numFields = selects.ordinality();
  14685. doBuildDistributionClearFunc(instance->startctx, dataset, selects);
  14686. doBuildDistributionNextFunc(instance->startctx, dataset, selects);
  14687. doBuildDistributionDestructFunc(instance->startctx, numFields);
  14688. doBuildDistributionGatherFunc(instance->startctx, numFields);
  14689. doBuildDistributionMergeFunc(instance->startctx, numFields);
  14690. doBuildDistributionSerializeFunc(instance->startctx, numFields);
  14691. //Need an extra meta information for the internal aggregate record
  14692. {
  14693. HqlExprArray fields;
  14694. fields.append(*createField(unnamedAtom, makeDataType(numFields*sizeof(void*)), NULL, NULL));
  14695. OwnedHqlExpr tempRecord = createRecord(fields);
  14696. OwnedHqlExpr nullDataset = createDataset(no_anon, tempRecord.getLink());
  14697. buildMetaMember(instance->classctx, nullDataset, "queryInternalRecordSize");
  14698. }
  14699. //Generate the send Result method().
  14700. {
  14701. BuildCtx funcctx(instance->startctx);
  14702. funcctx.addQuotedCompound("virtual void sendResult(size32_t length, const char * text)");
  14703. CHqlBoundExpr bound;
  14704. HqlExprAttr translated;
  14705. bound.length.setown(createVariable("length", makeIntType(sizeof(size32_t), false)));
  14706. bound.expr.setown(createVariable("text", makeStringType(UNKNOWN_LENGTH, NULL, NULL)));
  14707. translated.setown(bound.getTranslatedExpr());
  14708. buildSetResultInfo(funcctx, expr, translated, NULL, false, false);
  14709. }
  14710. buildInstanceSuffix(instance);
  14711. buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
  14712. return instance->getBoundActivity();
  14713. }
  14714. //---------------------------------------------------------------------------
  14715. //-- pure hole table --
  14716. void HqlCppTranslator::addFileDependency(IHqlExpression * name, ABoundActivity * whoAmI)
  14717. {
  14718. if (name && activeGraphCtx)
  14719. {
  14720. OwnedHqlExpr search = createAttribute(fileAtom, getNormalizedFilename(name));
  14721. HqlExprAssociation * match = activeGraphCtx->queryMatchExpr(search);
  14722. if (match)
  14723. addDependency(*activeGraphCtx, ((ABoundActivity *)match->queryExpr()->queryUnknownExtra()), whoAmI, sourceAtom);
  14724. }
  14725. }
  14726. //---------------------------------------------------------------------------
  14727. StringBuffer &expandDotLiteral(StringBuffer &s, const char *f)
  14728. {
  14729. unsigned lines = 0;
  14730. unsigned chars = 0;
  14731. char c;
  14732. while ((c = *f++) != 0)
  14733. {
  14734. switch (c)
  14735. {
  14736. case '\t':
  14737. s.append(" ");
  14738. break;
  14739. case '\r':
  14740. break;
  14741. case '\n':
  14742. s.append("\\l"); // Means left justify.
  14743. if (lines++ > 10)
  14744. return s;
  14745. break;
  14746. case '}':
  14747. case '{':
  14748. case '<':
  14749. case '>': // Special chars in dot graphs
  14750. case '\\':
  14751. case '\"':
  14752. case '\'':
  14753. s.append('\\');
  14754. // fall into...
  14755. default:
  14756. if (chars++ > 1000)
  14757. return s;
  14758. s.append(c);
  14759. break;
  14760. }
  14761. }
  14762. return s;
  14763. }
  14764. void HqlCppTranslator::logGraphEdge(IPropertyTree * subGraph, unsigned __int64 source, unsigned __int64 target, unsigned outputIndex, unsigned inputIndex, const char * label, bool nWay)
  14765. {
  14766. addSimpleGraphEdge(subGraph, source, target, outputIndex, inputIndex, NULL, label, nWay);
  14767. }
  14768. void HqlCppTranslator::buildActivityFramework(ActivityInstance * instance)
  14769. {
  14770. assertex(!instance->isAction()); // All actions should be calling the 2 parameter version below instead
  14771. buildActivityFramework(instance, false);
  14772. }
  14773. void HqlCppTranslator::buildActivityFramework(ActivityInstance * instance, bool alwaysExecuted)
  14774. {
  14775. instance->createGraphNode(graph, alwaysExecuted);
  14776. if (options.trackDuplicateActivities)
  14777. {
  14778. IHqlExpression * search = instance->dataset;
  14779. node_operator op = search->getOperator();
  14780. if ((op != no_select) && (op != no_workunit_dataset))
  14781. {
  14782. OwnedHqlExpr searchNorm = getUnadornedExpr(search);
  14783. unsigned crc = getExpressionCRC(search);
  14784. ForEachItemIn(i, tracking.activityExprs)
  14785. {
  14786. if (search == &tracking.activityExprs.item(i))
  14787. instance->addAttributeInt("_duplicateActivity_", tracking.activityIds.item(i));
  14788. else if (crc == tracking.activityCrcs.item(i))
  14789. instance->addAttributeInt("_duplicateCrcActivity_", tracking.activityIds.item(i));
  14790. else if (searchNorm == &tracking.activityNorms.item(i))
  14791. instance->addAttributeInt("_duplicateNormActivity_", tracking.activityIds.item(i));
  14792. }
  14793. tracking.activityExprs.append(*LINK(search));
  14794. tracking.activityNorms.append(*LINK(searchNorm));
  14795. tracking.activityIds.append(instance->activityId);
  14796. tracking.activityCrcs.append(crc);
  14797. }
  14798. }
  14799. }
  14800. void HqlCppTranslator::buildConnectOrders(BuildCtx & ctx, ABoundActivity * slaveActivity, ABoundActivity * masterActivity)
  14801. {
  14802. //I'm not sure we even use this information, but at least it's there if needed.
  14803. if (targetThor())
  14804. {
  14805. IPropertyTree *edge = createPTree();
  14806. edge->setPropInt64("@target", slaveActivity->queryActivityId());
  14807. edge->setPropInt64("@source", masterActivity->queryActivityId());
  14808. addGraphAttributeBool(edge, "cosort", true);
  14809. SubGraphInfo * activeSubgraph = queryActiveSubGraph(ctx);
  14810. assertex(activeSubgraph);
  14811. activeSubgraph->tree->addPropTree("edge", edge);
  14812. }
  14813. }
  14814. ColumnToOffsetMap * HqlCppTranslator::queryRecordOffsetMap(IHqlExpression * record)
  14815. {
  14816. if (record)
  14817. return recordMap.queryMapping(record, options.maxRecordSize);
  14818. return NULL;
  14819. }
  14820. unsigned HqlCppTranslator::getFixedRecordSize(IHqlExpression * record)
  14821. {
  14822. return queryRecordOffsetMap(record)->getFixedRecordSize();
  14823. }
  14824. bool HqlCppTranslator::isFixedRecordSize(IHqlExpression * record)
  14825. {
  14826. return queryRecordOffsetMap(record)->isFixedWidth();
  14827. }
  14828. void HqlCppTranslator::buildReturnRecordSize(BuildCtx & ctx, BoundRow * cursor)
  14829. {
  14830. OwnedHqlExpr size = getRecordSize(cursor->querySelector());
  14831. buildReturn(ctx, size);
  14832. }
  14833. bool HqlCppTranslator::recordContainsIfBlock(IHqlExpression * record)
  14834. {
  14835. return queryRecordOffsetMap(record)->queryContainsIfBlock();
  14836. }
  14837. IHqlExpression * HqlCppTranslator::createRecordInheritMaxLength(HqlExprArray & fields, IHqlExpression * donor)
  14838. {
  14839. IHqlExpression * record = donor->queryRecord();
  14840. unsigned prevLength = fields.ordinality();
  14841. LinkedHqlExpr max;
  14842. if (!queryProperty(maxLengthAtom, fields))
  14843. {
  14844. max.set(donor->queryProperty(maxLengthAtom));
  14845. if (!max && hasMaxLength(record))
  14846. {
  14847. //maxlength inherited somewhere...
  14848. OwnedHqlExpr serializedDonorRecord = getSerializedForm(record);
  14849. ColumnToOffsetMap * map = queryRecordOffsetMap(serializedDonorRecord);
  14850. max.setown(createAttribute(maxLengthAtom, getSizetConstant(map->getMaxSize())));
  14851. }
  14852. if (max)
  14853. fields.append(*LINK(max));
  14854. }
  14855. OwnedHqlExpr ret = createRecord(fields);
  14856. fields.trunc(prevLength);
  14857. if (max)
  14858. {
  14859. OwnedHqlExpr serialized = getSerializedForm(ret);
  14860. if (isFixedRecordSize(serialized))
  14861. ret.setown(createRecord(fields));
  14862. }
  14863. return ret.getClear();
  14864. }
  14865. //-- Code to transform the expressions ready for generating source code.
  14866. static void logECL(const LogMsgCategory & category, size32_t len, const char * ecl)
  14867. {
  14868. const size32_t chunkSize = 31000;
  14869. while (len > chunkSize)
  14870. {
  14871. const char * next = strchr(ecl+chunkSize, '\n');
  14872. if (!next || !next[1])
  14873. break;
  14874. unsigned size = next-ecl;
  14875. if (ecl[size-1] == '\r')
  14876. size--;
  14877. LOG(category, unknownJob, "%.*s", size, ecl);
  14878. len -= (next+1-ecl);
  14879. ecl = next+1;
  14880. }
  14881. LOG(category, unknownJob, "%s", ecl);
  14882. }
  14883. void HqlCppTranslator::traceExpression(const char * title, IHqlExpression * expr, unsigned level)
  14884. {
  14885. checkAbort();
  14886. LOG(MCdebugInfo(200), unknownJob, "Tracing expressions: %s", title);
  14887. LogMsgCategory debug500 = MCdebugInfo(level);
  14888. if(REJECTLOG(debug500))
  14889. return;
  14890. StringBuffer s;
  14891. processedTreeToECL(expr, s);
  14892. logECL(debug500, s.length(), s.str());
  14893. }
  14894. void HqlCppTranslator::traceExpressions(const char * title, HqlExprArray & exprs, unsigned level)
  14895. {
  14896. checkAbort();
  14897. // PrintLog(title);
  14898. LOG(MCdebugInfo(200), unknownJob, "Tracing expressions: %s", title);
  14899. LogMsgCategory debug500 = MCdebugInfo(level);
  14900. if(REJECTLOG(debug500))
  14901. return;
  14902. OwnedHqlExpr compound = createComma(exprs);
  14903. if (compound)
  14904. {
  14905. StringBuffer s;
  14906. processedTreeToECL(compound, s);
  14907. logECL(debug500, s.length(), s.str());
  14908. }
  14909. }
  14910. void HqlCppTranslator::traceExpressions(const char * title, WorkflowArray & workflow)
  14911. {
  14912. checkAbort();
  14913. // PrintLog(title);
  14914. LOG(MCdebugInfo(200), unknownJob, "Tracing expressions: %s", title);
  14915. static LogMsgCategory debug500 = MCdebugInfo(500);
  14916. static LogMsgCategory debug5000 = MCdebugInfo(5000);
  14917. if(REJECTLOG(debug500))
  14918. return;
  14919. ForEachItemIn(idx1, workflow)
  14920. {
  14921. WorkflowItem & cur = workflow.item(idx1);
  14922. OwnedHqlExpr compound = createComma(cur.queryExprs());
  14923. if (compound)
  14924. {
  14925. StringBuffer s;
  14926. processedTreeToECL(compound, s);
  14927. LOG(debug500, unknownJob, "%s: #%d: id[%d]", title, idx1, cur.queryWfid());
  14928. logECL(debug500, s.length(), s.str());
  14929. }
  14930. }
  14931. }
  14932. void HqlCppTranslator::checkNormalized(WorkflowArray & workflow)
  14933. {
  14934. if (options.paranoidCheckDependencies)
  14935. checkDependencyConsistency(workflow);
  14936. if (options.paranoidCheckNormalized)
  14937. {
  14938. ForEachItemIn(i, workflow)
  14939. {
  14940. OwnedHqlExpr compound = createActionList(workflow.item(i).queryExprs());
  14941. ::checkNormalized(compound);
  14942. }
  14943. }
  14944. }
  14945. void HqlCppTranslator::checkNormalized(IHqlExpression * expr)
  14946. {
  14947. if (options.paranoidCheckDependencies)
  14948. checkDependencyConsistency(expr);
  14949. if (options.paranoidCheckNormalized)
  14950. {
  14951. ::checkNormalized(expr);
  14952. }
  14953. }
  14954. void HqlCppTranslator::checkNormalized(HqlExprArray & exprs)
  14955. {
  14956. if (options.paranoidCheckDependencies)
  14957. checkDependencyConsistency(exprs);
  14958. if (options.paranoidCheckNormalized)
  14959. {
  14960. ForEachItemIn(i, exprs)
  14961. ::checkNormalized(&exprs.item(i));
  14962. }
  14963. }
  14964. void HqlCppTranslator::checkNormalized(BuildCtx & ctx, IHqlExpression * expr)
  14965. {
  14966. if (options.paranoidCheckDependencies)
  14967. checkDependencyConsistency(expr);
  14968. if (options.paranoidCheckNormalized)
  14969. {
  14970. HqlExprArray activeTables;
  14971. //Added in reverse order, but normalize checker doesn't care
  14972. RowAssociationIterator iter(ctx);
  14973. ForEach(iter)
  14974. {
  14975. BoundRow & cur = iter.get();
  14976. if ((cur.querySide() != no_self) && !cur.isBuilder())
  14977. activeTables.append(*LINK(cur.querySelector()));
  14978. }
  14979. ::checkNormalized(expr, activeTables);
  14980. }
  14981. }
  14982. void createCompoundEnsure(HqlExprArray & exprs, unsigned first, unsigned last)
  14983. {
  14984. if (first >= last || last == NotFound)
  14985. return;
  14986. IHqlExpression * action = &exprs.item(last);
  14987. OwnedHqlExpr actionExpr = createActionList(exprs, first, last);
  14988. OwnedHqlExpr compound = createCompound(actionExpr.getClear(), LINK(action->queryChild(0)));
  14989. OwnedHqlExpr newAction = replaceChild(action, 0, compound);
  14990. exprs.replace(*newAction.getClear(), first);
  14991. exprs.removen(first+1, (last-first));
  14992. }
  14993. //move any set results inside a no_ensureresult so they don't get evaluated unless necessary.
  14994. void HqlCppTranslator::optimizePersists(HqlExprArray & exprs)
  14995. {
  14996. //If there is a single ensure result, and no set results created from other workflow items
  14997. //then move all previous actions inside the ensure result
  14998. unsigned max = exprs.ordinality();
  14999. if (max == 0)
  15000. return;
  15001. if (exprs.item(max-1).getOperator() != no_ensureresult)
  15002. return;
  15003. for (unsigned i=0; i < max-1; i++)
  15004. {
  15005. IHqlExpression & cur = exprs.item(i);
  15006. if ((cur.getOperator() == no_ensureresult) || cur.hasProperty(_workflow_Atom))
  15007. return;
  15008. }
  15009. createCompoundEnsure(exprs, 0, max-1);
  15010. }
  15011. void HqlCppTranslator::optimizePersists(WorkflowArray & workflow)
  15012. {
  15013. ForEachItemIn(idx, workflow)
  15014. optimizePersists(workflow.item(idx).queryExprs());
  15015. }
  15016. IHqlExpression * HqlCppTranslator::extractGlobalCSE(IHqlExpression * expr)
  15017. {
  15018. AutoScopeMigrateTransformer transformer(wu(), *this);
  15019. HqlExprArray exprs;
  15020. unwindCommaCompound(exprs, expr);
  15021. transformer.analyseArray(exprs, 0);
  15022. if (!transformer.worthTransforming())
  15023. return LINK(expr);
  15024. HqlExprArray results;
  15025. transformer.transformRoot(exprs, results);
  15026. return createActionList(results);
  15027. }
  15028. IHqlExpression * HqlCppTranslator::spotGlobalCSE(IHqlExpression * _expr)
  15029. {
  15030. if (!_expr->isAction())
  15031. return LINK(_expr);
  15032. LinkedHqlExpr expr = _expr;
  15033. switch (expr->getOperator())
  15034. {
  15035. case no_if:
  15036. {
  15037. IHqlExpression * left = expr->queryChild(1);
  15038. IHqlExpression * right = expr->queryChild(2);
  15039. HqlExprArray args;
  15040. args.append(*LINK(expr->queryChild(0)));
  15041. args.append(*extractGlobalCSE(left));
  15042. if (right)
  15043. args.append(*extractGlobalCSE(right));
  15044. if ((left != &args.item(1)) || (right && right != &args.item(2)))
  15045. {
  15046. unwindChildren(args, expr, 3);
  15047. expr.setown(_expr->clone(args));
  15048. }
  15049. break;
  15050. }
  15051. case no_sequential:
  15052. {
  15053. HqlExprArray args;
  15054. ForEachChild(i, expr)
  15055. args.append(*extractGlobalCSE(expr->queryChild(i)));
  15056. expr.setown(_expr->clone(args));
  15057. break;
  15058. }
  15059. case no_nothor:
  15060. return LINK(expr);
  15061. }
  15062. bool same = true;
  15063. HqlExprArray args;
  15064. ForEachChild(i, expr)
  15065. {
  15066. IHqlExpression * cur = expr->queryChild(i);
  15067. IHqlExpression * next = spotGlobalCSE(cur);
  15068. args.append(*next);
  15069. if (cur != next)
  15070. same = false;
  15071. }
  15072. if (same)
  15073. return expr.getClear();
  15074. return expr->clone(args);
  15075. }
  15076. void HqlCppTranslator::spotGlobalCSE(HqlExprArray & exprs)
  15077. {
  15078. HqlExprArray results;
  15079. AutoScopeMigrateTransformer transformer(wu(), *this);
  15080. transformer.analyseArray(exprs, 0);
  15081. if (transformer.worthTransforming())
  15082. {
  15083. transformer.transformRoot(exprs, results);
  15084. replaceArray(exprs, results);
  15085. }
  15086. if (!options.resourceConditionalActions)
  15087. {
  15088. //Now need to recursively walk actions, and if any conditional actions could do with things being hoisted within them
  15089. ForEachItemIn(i, exprs)
  15090. exprs.replace(*spotGlobalCSE(&exprs.item(i)), i);
  15091. }
  15092. }
  15093. void HqlCppTranslator::spotGlobalCSE(WorkflowArray & array)
  15094. {
  15095. if (!insideLibrary() && options.globalAutoHoist)
  15096. {
  15097. unsigned startTime = msTick();
  15098. ForEachItemIn(idx, array)
  15099. spotGlobalCSE(array.item(idx).queryExprs());
  15100. DEBUG_TIMER("EclServer: tree transform: spot global cse", msTick()-startTime);
  15101. }
  15102. }
  15103. void HqlCppTranslator::flattenDatasets(WorkflowArray & array)
  15104. {
  15105. //MORE: Should project fields needed outside <ds>.<ds> so that they are available.
  15106. }
  15107. // Code to check whether thor is required for a query. It should err towards true.
  15108. // The idea is to prevent some very simple queries going to thor, mainly when users are examining data. The main examples are:
  15109. // 1. Unfiltered table count
  15110. // 2. Filtered index count.
  15111. // 3. Restricted set of records from an index/table.
  15112. enum { NRTfiltered = 0x0001, NRTcount = 0x0002, NRTlimited = 0x0004 };
  15113. static bool needsRealThor(IHqlExpression *expr, unsigned flags)
  15114. {
  15115. unsigned numChildrenToCheck = (unsigned)-1;
  15116. switch (expr->getOperator())
  15117. {
  15118. case no_table:
  15119. //only allow non filtered limited outputs, and non filtered counts
  15120. return !((flags == NRTlimited) || (flags == NRTcount));
  15121. case no_newkeyindex:
  15122. case no_keyindex:
  15123. case no_compound_indexread:
  15124. //Don't allow count(choosen(...)) otherwise likely to be better in hthor
  15125. if (flags & NRTcount)
  15126. return (flags & NRTlimited) != 0;
  15127. //unfiltered index read should go via thor
  15128. if (flags == 0)
  15129. return true;
  15130. //filtered index reads likely to be much better in hthor.
  15131. return false;
  15132. case no_attr:
  15133. case no_attr_expr:
  15134. case no_attr_link:
  15135. case no_datasetfromrow:
  15136. case no_rows:
  15137. case no_libraryinput:
  15138. case no_fail:
  15139. case no_persist_check:
  15140. return false;
  15141. case no_distribution:
  15142. case no_buildindex:
  15143. case no_keydiff:
  15144. case no_keypatch:
  15145. case no_forcelocal:
  15146. case no_forcenolocal:
  15147. case no_allnodes:
  15148. case no_thisnode:
  15149. return true;
  15150. case no_hqlproject:
  15151. //If count project, a count will not be done as a compound operation
  15152. if (expr->hasProperty(_countProject_Atom) && (flags & NRTcount))
  15153. return true;
  15154. break;
  15155. case no_compound_indexcount:
  15156. case no_transformascii:
  15157. case no_transformebcdic:
  15158. case no_selectfields:
  15159. case no_thor:
  15160. case no_apply:
  15161. case no_distributed:
  15162. case no_preservemeta:
  15163. case no_sorted:
  15164. case no_limit:
  15165. case no_catchds:
  15166. case no_keyedlimit:
  15167. case no_assertsorted:
  15168. case no_assertgrouped:
  15169. case no_assertdistributed:
  15170. case no_section:
  15171. case no_sectioninput:
  15172. case no_forcegraph:
  15173. case no_nofold:
  15174. case no_nohoist:
  15175. case no_actionlist:
  15176. case no_externalcall:
  15177. case no_call:
  15178. case no_compound_fetch:
  15179. case no_addfiles:
  15180. case no_nonempty:
  15181. //i.e. go through children...
  15182. break;
  15183. case no_compound:
  15184. case no_comma:
  15185. case no_executewhen:
  15186. numChildrenToCheck = expr->numChildren();
  15187. break;
  15188. case no_choosen:
  15189. case no_selectnth:
  15190. flags |= NRTlimited;
  15191. break;
  15192. case no_filter:
  15193. flags |= NRTfiltered;
  15194. break;
  15195. case no_newaggregate:
  15196. case no_newusertable:
  15197. if (isAggregateDataset(expr))
  15198. {
  15199. //Only allow aggregates we can do on an index directly
  15200. if (datasetHasGroupBy(expr))
  15201. return true;
  15202. node_operator aggOp = querySingleAggregate(expr, false, false, true);
  15203. if ((aggOp != no_exists) && (aggOp != no_count))
  15204. return true;
  15205. flags |= NRTcount;
  15206. }
  15207. break;
  15208. case no_if:
  15209. {
  15210. if (needsRealThor(expr->queryChild(0), 0))
  15211. return true;
  15212. if (needsRealThor(expr->queryChild(1), flags))
  15213. return true;
  15214. IHqlExpression * c2 = expr->queryChild(2);
  15215. return (c2 && needsRealThor(c2, flags));
  15216. }
  15217. case no_colon:
  15218. case no_globalscope:
  15219. case no_extractresult:
  15220. return needsRealThor(expr->queryChild(0), flags);
  15221. case no_fetch:
  15222. return needsRealThor(expr->queryChild(1), flags);
  15223. case no_output:
  15224. {
  15225. //Assume any output to files needs to stay where it is.
  15226. IHqlExpression *child0 = expr->queryChild(0);
  15227. IHqlExpression *filename = queryRealChild(expr, 1);
  15228. if (filename)
  15229. return true;
  15230. return needsRealThor(child0, flags);
  15231. }
  15232. case no_ensureresult:
  15233. case no_setresult:
  15234. {
  15235. IHqlExpression * child0 = expr->queryChild(0);
  15236. if (!child0->queryType()->isScalar())
  15237. return needsRealThor(child0, flags);
  15238. if (child0->getOperator() == no_evalonce)
  15239. child0 = child0->queryChild(0);
  15240. switch (child0->getOperator())
  15241. {
  15242. case no_externalcall:
  15243. case no_countfile:
  15244. case no_countindex:
  15245. case no_constant:
  15246. case no_all:
  15247. return false;
  15248. case no_select:
  15249. return needsRealThor(child0->queryChild(0), flags);
  15250. }
  15251. if (!containsAnyDataset(child0))
  15252. return false;
  15253. // return needsRealThor(child0, isFiltered);
  15254. //fallthrough...
  15255. }
  15256. default:
  15257. if (expr->isDataset())
  15258. return true;
  15259. ITypeInfo * type = expr->queryType();
  15260. if (!type || (type->getTypeCode() != type_void))
  15261. return false; //MORE Doesn't cope with scalar expressions that require thor e.g., counts of sorts of ....
  15262. //MORE: This means that lots of scalar expressions go to thor instead of hthor.
  15263. return true;
  15264. }
  15265. if (numChildrenToCheck == (unsigned)-1)
  15266. numChildrenToCheck = expr->isDataset() ? getNumChildTables(expr) : expr->numChildren();
  15267. for (unsigned idx=0; idx < numChildrenToCheck; idx++)
  15268. {
  15269. if (needsRealThor(expr->queryChild(idx), flags))
  15270. return true;
  15271. }
  15272. return false;
  15273. }
  15274. bool needsRealThor(IHqlExpression *expr)
  15275. {
  15276. return needsRealThor(expr, 0);
  15277. }
  15278. IHqlExpression * HqlCppTranslator::getDefaultOutputAttr(IHqlExpression * expr)
  15279. {
  15280. return createAttribute(workunitAtom); // backwards compatibility!
  15281. IHqlExpression * dataset = expr->queryChild(0);
  15282. if (dataset->getOperator() == no_selectfields)
  15283. dataset = dataset->queryChild(0);
  15284. if (dataset->getOperator()==no_choosen)
  15285. {
  15286. //If choosen() is specified, then output to SDS if small enough, else a temporary file.
  15287. IHqlExpression * count = dataset->queryChild(1);
  15288. if (count->queryValue())
  15289. {
  15290. unsigned __int64 value = count->queryValue()->getIntValue();
  15291. if (value <= MAX_ROWS_OUTPUT_TO_SDS)
  15292. return createAttribute(workunitAtom);
  15293. }
  15294. return createAttribute(diskAtom);
  15295. }
  15296. //No support yet in IFileView for delayed browsing - so output to disk.
  15297. return createAttribute(diskAtom);
  15298. }
  15299. void HqlCppTranslator::modifyOutputLocations(HqlExprArray & exprs)
  15300. {
  15301. ForEachItemIn(idx, exprs)
  15302. {
  15303. IHqlExpression &expr = exprs.item(idx);
  15304. IHqlExpression * filename = queryRealChild(&expr, 1);
  15305. //Deduce whether OUTPUT(x) should goes to SDS or a disk
  15306. if (expr.getOperator()==no_output && !filename)
  15307. {
  15308. if (!expr.hasProperty(workunitAtom) && !expr.hasProperty(firstAtom) && !expr.hasProperty(diskAtom))
  15309. {
  15310. IHqlExpression * attr = getDefaultOutputAttr(&expr);
  15311. HqlExprArray args;
  15312. expr.unwindList(args, no_output);
  15313. args.append(*attr);
  15314. IHqlExpression * transformed = expr.clone(args);
  15315. exprs.replace(*transformed, idx);
  15316. }
  15317. }
  15318. }
  15319. }
  15320. void HqlCppTranslator::pickBestEngine(HqlExprArray & exprs)
  15321. {
  15322. // At this point it is not too late to decide whether real thor is needed.
  15323. // Basically, we will use real thor if available, unless it matches a very minimal set of patterns:
  15324. // 1. output(holequery)
  15325. // 2. output(choosen(holequery))
  15326. // 3. output([filtered/projected/firstn] thordiskfile); (not sure about the filtered)
  15327. // These correspond closely to the things that can eventually get turned into lazy remote views
  15328. //It would be more sensible to do this much earlier.....or not at all.
  15329. if (targetThor())
  15330. {
  15331. unsigned time = msTick();
  15332. ForEachItemIn(idx, exprs)
  15333. {
  15334. if (needsRealThor(&exprs.item(idx)))
  15335. return;
  15336. }
  15337. // if we got this far, thor not required
  15338. setTargetClusterType(HThorCluster);
  15339. DBGLOG("Thor query redirected to hthor instead");
  15340. DEBUG_TIMER("EclServer: tree transform: pick engine", msTick()-time);
  15341. }
  15342. }
  15343. void HqlCppTranslator::pickBestEngine(WorkflowArray & array)
  15344. {
  15345. if (targetThor())
  15346. {
  15347. unsigned time = msTick();
  15348. ForEachItemIn(idx2, array)
  15349. {
  15350. HqlExprArray & exprs = array.item(idx2).queryExprs();
  15351. ForEachItemIn(idx, exprs)
  15352. {
  15353. if (needsRealThor(&exprs.item(idx)))
  15354. return;
  15355. }
  15356. // if we got this far, thor not required
  15357. }
  15358. setTargetClusterType(HThorCluster);
  15359. DBGLOG("Thor query redirected to hthor instead");
  15360. DEBUG_TIMER("EclServer: tree transform: pick engine", msTick()-time);
  15361. }
  15362. }
  15363. unsigned getVirtualFieldSize(IHqlExpression * record)
  15364. {
  15365. unsigned size = 0;
  15366. ForEachChild(idx, record)
  15367. {
  15368. IHqlExpression * cur = record->queryChild(idx);
  15369. switch (cur->getOperator())
  15370. {
  15371. case no_field:
  15372. {
  15373. ITypeInfo * type = cur->queryType();
  15374. if (type->isScalar())
  15375. {
  15376. if (cur->hasProperty(virtualAtom))
  15377. size += type->getSize();
  15378. }
  15379. else
  15380. size += getVirtualFieldSize(cur->queryRecord());
  15381. break;
  15382. }
  15383. case no_ifblock:
  15384. size += getVirtualFieldSize(cur->queryChild(1));
  15385. break;
  15386. case no_record:
  15387. size += getVirtualFieldSize(cur);
  15388. break;
  15389. }
  15390. }
  15391. return size;
  15392. }
  15393. //---------------------------------------------------------------------------
  15394. /*
  15395. The following transforms are applied to the graphs before the code is generated:
  15396. o replaceStoredValues(exprs, foldStored);
  15397. - Replaces any #stored values in the graph. Done first so everything remains consistent.
  15398. o normalizeHqlTree(exprs);
  15399. - Converts expressions to their normal form. Main change is to remove default values from fields, so the graph can be
  15400. transformed without having to remap fields its value (e.g., dataset on a select) changes. Impossible to do anything without
  15401. this stage.
  15402. - Also converts x : global to global(x) and normalizes a few simple constructs e.g., trim,right->trim
  15403. o allocateSequenceNumbers(exprs);
  15404. - Adds sequence numbers to all outputs
  15405. o foldHqlExpression
  15406. - does a global constant fold to simplify the graph before it gets broken up at all.
  15407. o optimizeHqlExpression
  15408. - could globally optimize the graph, but not enabled at the moment. I'm not sure why, maybe because we still don't trust it enough...
  15409. o extractWorkflow(exprs)
  15410. - Converts a list of expressions into a list of workflow items. Once this is done each workflow item can be treated independently.
  15411. --- The following aim to work out where each part of the query will be executed, and gather all parts that will be executed in the same place
  15412. --- together so increase likely hood for cses and reduce connections to the query engines.
  15413. o optimizeThorCounts
  15414. - A first go at transforming count(table) to a no_countdisk and a count(index) to a no_countindex.
  15415. !!This should be deleted/changed once we have aggregate source activities
  15416. o migrateExprToNaturalLevel [NewScopeMigrateTransformer]
  15417. - Ensure expressions are evaluated at the best level - e.g., counts moved to most appropriate level.
  15418. * global(x) is extracted to a global set/get result pair.
  15419. - This includes datasets, and specifically ignores any conditional context
  15420. * SET(dataset, field) is converted to a workunit output, workunit read pair
  15421. !!Needs revisiting for child queries
  15422. * global count(), max() on global tables used inside an activity are always hoisted
  15423. !!regardless of whether they are conditional or not.
  15424. * local(x) and global() interactions are processed
  15425. !!Once aggregate sources are implemented, this may not be so useful, or even needed.
  15426. o markThorBoundaries
  15427. - work out which engine is going to perform which operation.
  15428. o normalizeResultFormat
  15429. * convert thor(scalar|row) into compound(setresult,getresult) or global setresult, local getresult
  15430. !!It should common up conditional expressions with non conditional
  15431. * IF(count(x)...) inside thor, hoist the count(x) so it is global
  15432. * IF(ds[n]...) inside thor, hoist it so it is global.
  15433. !!Conditionals should be handled differently I am sure...
  15434. o flattenDatasets(array);
  15435. - Currently does nothing. It should add projects to hole to ensure fields are available.
  15436. o mergeThorGraphs
  15437. - Make sure thor graphs are together as much as possible to save transfers to thor, and maximise the cse chances.
  15438. - Combine results of above. Should probably just be a single transformation.
  15439. o spotGlobalCSE
  15440. - Spot CSEs between different graphs. E.g., f(a), if(x, g(a), h(a)) Should ensure a is spilled.
  15441. !!Current problem is that global can create a implicit dependency which isn't spotted.
  15442. !!also doesn't handle commoning up scalars very well.
  15443. o removeTrivialGraphs
  15444. - don't implement setresult(getresult) in thor - a waste of time....
  15445. o convertLogicalToActivities
  15446. - change representation from logical to actual implementation. E.g., dedup(a, b) becomes group(dedup(group(a,b,all)))
  15447. */