hthor.cpp 311 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383638463856386638763886389639063916392639363946395639663976398639964006401640264036404640564066407640864096410641164126413641464156416641764186419642064216422642364246425642664276428642964306431643264336434643564366437643864396440644164426443644464456446644764486449645064516452645364546455645664576458645964606461646264636464646564666467646864696470647164726473647464756476647764786479648064816482648364846485648664876488648964906491649264936494649564966497649864996500650165026503650465056506650765086509651065116512651365146515651665176518651965206521652265236524652565266527652865296530653165326533653465356536653765386539654065416542654365446545654665476548654965506551655265536554655565566557655865596560656165626563656465656566656765686569657065716572657365746575657665776578657965806581658265836584658565866587658865896590659165926593659465956596659765986599660066016602660366046605660666076608660966106611661266136614661566166617661866196620662166226623662466256626662766286629663066316632663366346635663666376638663966406641664266436644664566466647664866496650665166526653665466556656665766586659666066616662666366646665666666676668666966706671667266736674667566766677667866796680668166826683668466856686668766886689669066916692669366946695669666976698669967006701670267036704670567066707670867096710671167126713671467156716671767186719672067216722672367246725672667276728672967306731673267336734673567366737673867396740674167426743674467456746674767486749675067516752675367546755675667576758675967606761676267636764676567666767676867696770677167726773677467756776677767786779678067816782678367846785678667876788678967906791679267936794679567966797679867996800680168026803680468056806680768086809681068116812681368146815681668176818681968206821682268236824682568266827682868296830683168326833683468356836683768386839684068416842684368446845684668476848684968506851685268536854685568566857685868596860686168626863686468656866686768686869687068716872687368746875687668776878687968806881688268836884688568866887688868896890689168926893689468956896689768986899690069016902690369046905690669076908690969106911691269136914691569166917691869196920692169226923692469256926692769286929693069316932693369346935693669376938693969406941694269436944694569466947694869496950695169526953695469556956695769586959696069616962696369646965696669676968696969706971697269736974697569766977697869796980698169826983698469856986698769886989699069916992699369946995699669976998699970007001700270037004700570067007700870097010701170127013701470157016701770187019702070217022702370247025702670277028702970307031703270337034703570367037703870397040704170427043704470457046704770487049705070517052705370547055705670577058705970607061706270637064706570667067706870697070707170727073707470757076707770787079708070817082708370847085708670877088708970907091709270937094709570967097709870997100710171027103710471057106710771087109711071117112711371147115711671177118711971207121712271237124712571267127712871297130713171327133713471357136713771387139714071417142714371447145714671477148714971507151715271537154715571567157715871597160716171627163716471657166716771687169717071717172717371747175717671777178717971807181718271837184718571867187718871897190719171927193719471957196719771987199720072017202720372047205720672077208720972107211721272137214721572167217721872197220722172227223722472257226722772287229723072317232723372347235723672377238723972407241724272437244724572467247724872497250725172527253725472557256725772587259726072617262726372647265726672677268726972707271727272737274727572767277727872797280728172827283728472857286728772887289729072917292729372947295729672977298729973007301730273037304730573067307730873097310731173127313731473157316731773187319732073217322732373247325732673277328732973307331733273337334733573367337733873397340734173427343734473457346734773487349735073517352735373547355735673577358735973607361736273637364736573667367736873697370737173727373737473757376737773787379738073817382738373847385738673877388738973907391739273937394739573967397739873997400740174027403740474057406740774087409741074117412741374147415741674177418741974207421742274237424742574267427742874297430743174327433743474357436743774387439744074417442744374447445744674477448744974507451745274537454745574567457745874597460746174627463746474657466746774687469747074717472747374747475747674777478747974807481748274837484748574867487748874897490749174927493749474957496749774987499750075017502750375047505750675077508750975107511751275137514751575167517751875197520752175227523752475257526752775287529753075317532753375347535753675377538753975407541754275437544754575467547754875497550755175527553755475557556755775587559756075617562756375647565756675677568756975707571757275737574757575767577757875797580758175827583758475857586758775887589759075917592759375947595759675977598759976007601760276037604760576067607760876097610761176127613761476157616761776187619762076217622762376247625762676277628762976307631763276337634763576367637763876397640764176427643764476457646764776487649765076517652765376547655765676577658765976607661766276637664766576667667766876697670767176727673767476757676767776787679768076817682768376847685768676877688768976907691769276937694769576967697769876997700770177027703770477057706770777087709771077117712771377147715771677177718771977207721772277237724772577267727772877297730773177327733773477357736773777387739774077417742774377447745774677477748774977507751775277537754775577567757775877597760776177627763776477657766776777687769777077717772777377747775777677777778777977807781778277837784778577867787778877897790779177927793779477957796779777987799780078017802780378047805780678077808780978107811781278137814781578167817781878197820782178227823782478257826782778287829783078317832783378347835783678377838783978407841784278437844784578467847784878497850785178527853785478557856785778587859786078617862786378647865786678677868786978707871787278737874787578767877787878797880788178827883788478857886788778887889789078917892789378947895789678977898789979007901790279037904790579067907790879097910791179127913791479157916791779187919792079217922792379247925792679277928792979307931793279337934793579367937793879397940794179427943794479457946794779487949795079517952795379547955795679577958795979607961796279637964796579667967796879697970797179727973797479757976797779787979798079817982798379847985798679877988798979907991799279937994799579967997799879998000800180028003800480058006800780088009801080118012801380148015801680178018801980208021802280238024802580268027802880298030803180328033803480358036803780388039804080418042804380448045804680478048804980508051805280538054805580568057805880598060806180628063806480658066806780688069807080718072807380748075807680778078807980808081808280838084808580868087808880898090809180928093809480958096809780988099810081018102810381048105810681078108810981108111811281138114811581168117811881198120812181228123812481258126812781288129813081318132813381348135813681378138813981408141814281438144814581468147814881498150815181528153815481558156815781588159816081618162816381648165816681678168816981708171817281738174817581768177817881798180818181828183818481858186818781888189819081918192819381948195819681978198819982008201820282038204820582068207820882098210821182128213821482158216821782188219822082218222822382248225822682278228822982308231823282338234823582368237823882398240824182428243824482458246824782488249825082518252825382548255825682578258825982608261826282638264826582668267826882698270827182728273827482758276827782788279828082818282828382848285828682878288828982908291829282938294829582968297829882998300830183028303830483058306830783088309831083118312831383148315831683178318831983208321832283238324832583268327832883298330833183328333833483358336833783388339834083418342834383448345834683478348834983508351835283538354835583568357835883598360836183628363836483658366836783688369837083718372837383748375837683778378837983808381838283838384838583868387838883898390839183928393839483958396839783988399840084018402840384048405840684078408840984108411841284138414841584168417841884198420842184228423842484258426842784288429843084318432843384348435843684378438843984408441844284438444844584468447844884498450845184528453845484558456845784588459846084618462846384648465846684678468846984708471847284738474847584768477847884798480848184828483848484858486848784888489849084918492849384948495849684978498849985008501850285038504850585068507850885098510851185128513851485158516851785188519852085218522852385248525852685278528852985308531853285338534853585368537853885398540854185428543854485458546854785488549855085518552855385548555855685578558855985608561856285638564856585668567856885698570857185728573857485758576857785788579858085818582858385848585858685878588858985908591859285938594859585968597859885998600860186028603860486058606860786088609861086118612861386148615861686178618861986208621862286238624862586268627862886298630863186328633863486358636863786388639864086418642864386448645864686478648864986508651865286538654865586568657865886598660866186628663866486658666866786688669867086718672867386748675867686778678867986808681868286838684868586868687868886898690869186928693869486958696869786988699870087018702870387048705870687078708870987108711871287138714871587168717871887198720872187228723872487258726872787288729873087318732873387348735873687378738873987408741874287438744874587468747874887498750875187528753875487558756875787588759876087618762876387648765876687678768876987708771877287738774877587768777877887798780878187828783878487858786878787888789879087918792879387948795879687978798879988008801880288038804880588068807880888098810881188128813881488158816881788188819882088218822882388248825882688278828882988308831883288338834883588368837883888398840884188428843884488458846884788488849885088518852885388548855885688578858885988608861886288638864886588668867886888698870887188728873887488758876887788788879888088818882888388848885888688878888888988908891889288938894889588968897889888998900890189028903890489058906890789088909891089118912891389148915891689178918891989208921892289238924892589268927892889298930893189328933893489358936893789388939894089418942894389448945894689478948894989508951895289538954895589568957895889598960896189628963896489658966896789688969897089718972897389748975897689778978897989808981898289838984898589868987898889898990899189928993899489958996899789988999900090019002900390049005900690079008900990109011901290139014901590169017901890199020902190229023902490259026902790289029903090319032903390349035903690379038903990409041904290439044904590469047904890499050905190529053905490559056905790589059906090619062906390649065906690679068906990709071907290739074907590769077907890799080908190829083908490859086908790889089909090919092909390949095909690979098909991009101910291039104910591069107910891099110911191129113911491159116911791189119912091219122912391249125912691279128912991309131913291339134913591369137913891399140914191429143914491459146914791489149915091519152915391549155915691579158915991609161916291639164916591669167916891699170917191729173917491759176917791789179918091819182918391849185918691879188918991909191919291939194919591969197919891999200920192029203920492059206920792089209921092119212921392149215921692179218921992209221922292239224922592269227922892299230923192329233923492359236923792389239924092419242924392449245924692479248924992509251925292539254925592569257925892599260926192629263926492659266926792689269927092719272927392749275927692779278927992809281928292839284928592869287928892899290929192929293929492959296929792989299930093019302930393049305930693079308930993109311931293139314931593169317931893199320932193229323932493259326932793289329933093319332933393349335933693379338933993409341934293439344934593469347934893499350935193529353935493559356935793589359936093619362936393649365936693679368936993709371937293739374937593769377937893799380938193829383938493859386938793889389939093919392939393949395939693979398939994009401940294039404940594069407940894099410941194129413941494159416941794189419942094219422942394249425942694279428942994309431943294339434943594369437943894399440944194429443944494459446944794489449945094519452945394549455945694579458945994609461946294639464946594669467946894699470947194729473947494759476947794789479948094819482948394849485948694879488948994909491949294939494949594969497949894999500950195029503950495059506950795089509951095119512951395149515951695179518951995209521952295239524952595269527952895299530953195329533953495359536953795389539954095419542954395449545954695479548954995509551955295539554955595569557955895599560956195629563956495659566956795689569957095719572957395749575957695779578957995809581958295839584958595869587958895899590959195929593959495959596959795989599960096019602960396049605960696079608960996109611961296139614961596169617961896199620962196229623962496259626962796289629963096319632963396349635963696379638963996409641964296439644964596469647964896499650965196529653965496559656965796589659966096619662966396649665966696679668966996709671967296739674967596769677967896799680968196829683968496859686968796889689969096919692969396949695969696979698969997009701970297039704970597069707970897099710971197129713971497159716971797189719972097219722972397249725972697279728972997309731973297339734973597369737973897399740974197429743974497459746974797489749975097519752975397549755975697579758975997609761976297639764976597669767976897699770977197729773977497759776977797789779978097819782978397849785978697879788978997909791979297939794979597969797979897999800980198029803980498059806980798089809981098119812981398149815981698179818981998209821982298239824982598269827982898299830983198329833983498359836983798389839984098419842984398449845984698479848984998509851985298539854985598569857985898599860986198629863986498659866986798689869987098719872987398749875987698779878987998809881988298839884988598869887988898899890989198929893989498959896989798989899990099019902990399049905990699079908990999109911991299139914991599169917991899199920992199229923992499259926992799289929993099319932993399349935993699379938993999409941994299439944994599469947994899499950995199529953995499559956995799589959996099619962996399649965996699679968996999709971997299739974997599769977997899799980998199829983998499859986998799889989999099919992999399949995999699979998999910000100011000210003100041000510006100071000810009100101001110012100131001410015100161001710018100191002010021100221002310024100251002610027100281002910030100311003210033100341003510036100371003810039100401004110042100431004410045100461004710048100491005010051100521005310054100551005610057100581005910060100611006210063100641006510066100671006810069100701007110072100731007410075100761007710078100791008010081100821008310084100851008610087100881008910090100911009210093100941009510096100971009810099101001010110102101031010410105101061010710108101091011010111101121011310114101151011610117101181011910120101211012210123101241012510126101271012810129101301013110132101331013410135101361013710138101391014010141101421014310144101451014610147101481014910150101511015210153101541015510156101571015810159101601016110162101631016410165101661016710168101691017010171101721017310174101751017610177101781017910180101811018210183101841018510186101871018810189101901019110192101931019410195101961019710198101991020010201102021020310204102051020610207102081020910210102111021210213102141021510216102171021810219102201022110222102231022410225102261022710228102291023010231102321023310234102351023610237102381023910240102411024210243102441024510246102471024810249102501025110252102531025410255102561025710258102591026010261102621026310264102651026610267102681026910270102711027210273102741027510276102771027810279102801028110282102831028410285102861028710288102891029010291102921029310294102951029610297102981029910300103011030210303103041030510306103071030810309103101031110312103131031410315103161031710318
  1. /*##############################################################################
  2. HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. ############################################################################## */
  13. #include <algorithm>
  14. #include "hthor.ipp"
  15. #include "jexcept.hpp"
  16. #include "jmisc.hpp"
  17. #include "jthread.hpp"
  18. #include "jsocket.hpp"
  19. #include "jprop.hpp"
  20. #include "jdebug.hpp"
  21. #include "jlzw.hpp"
  22. #include "jisem.hpp"
  23. #include "roxiedebug.hpp"
  24. #include "roxierow.hpp"
  25. #include "roxiemem.hpp"
  26. #include "eclhelper.hpp"
  27. #include "workunit.hpp"
  28. #include "jfile.hpp"
  29. #include "keybuild.hpp"
  30. #include "hrpc.hpp"
  31. #include "hrpcsock.hpp"
  32. #include "dafdesc.hpp"
  33. #include "dautils.hpp"
  34. #include "dasess.hpp"
  35. #include "dadfs.hpp"
  36. #include "thorfile.hpp"
  37. #include "thorsort.hpp"
  38. #include "thorparse.ipp"
  39. #include "thorxmlwrite.hpp"
  40. #include "jsmartsock.hpp"
  41. #include "thorstep.hpp"
  42. #include "eclagent.ipp"
  43. #include "roxierowbuff.hpp"
  44. #include "ftbase.ipp"
  45. #include "rtldynfield.hpp"
  46. #define EMPTY_LOOP_LIMIT 1000
  47. static unsigned const hthorReadBufferSize = 0x10000;
  48. static offset_t const defaultHThorDiskWriteSizeLimit = I64C(10*1024*1024*1024); //10 GB, per Nigel
  49. static size32_t const spillStreamBufferSize = 0x10000;
  50. static unsigned const hthorPipeWaitTimeout = 100; //100ms - fairly arbitrary choice
  51. using roxiemem::IRowManager;
  52. using roxiemem::OwnedRoxieRow;
  53. using roxiemem::OwnedRoxieString;
  54. using roxiemem::OwnedConstRoxieRow;
  55. IRowManager * theRowManager;
  56. void setHThorRowManager(IRowManager * manager)
  57. {
  58. theRowManager = manager;
  59. }
  60. IRowManager * queryRowManager()
  61. {
  62. return theRowManager;
  63. }
  64. void throwOOMException(size_t size, char const * label)
  65. {
  66. throw MakeStringException(0, "Out of Memory in hthor: trying to allocate %" I64F "u bytes for %s", (unsigned __int64) size, label);
  67. }
  68. void * checked_malloc(size_t size, char const * label)
  69. {
  70. void * ret = malloc(size);
  71. if(!ret)
  72. throwOOMException(size, label);
  73. return ret;
  74. }
  75. void * checked_calloc(size_t size, size_t num, char const * label)
  76. {
  77. void * ret = calloc(size, num);
  78. if(!ret)
  79. throwOOMException(size*num, label);
  80. return ret;
  81. }
  82. inline bool checkIsCompressed(unsigned int flags, size32_t fixedSize, bool grouped)
  83. {
  84. return ((flags & TDWnewcompress) || ((flags & TDXcompress) && (fixedSize+(grouped?1:0) >= MIN_ROWCOMPRESS_RECSIZE)));
  85. }
  86. //=====================================================================================================
  87. //=====================================================================================================
  88. CRowBuffer::CRowBuffer(IRecordSize * _recsize, bool _grouped) : recsize(_recsize), grouped(_grouped)
  89. {
  90. fixsize = recsize->getFixedSize();
  91. count = 0;
  92. index = 0;
  93. }
  94. void CRowBuffer::insert(const void * next)
  95. {
  96. buff.append(next);
  97. count++;
  98. }
  99. bool CRowBuffer::pull(IHThorInput * input, unsigned __int64 rowLimit)
  100. {
  101. while(true)
  102. {
  103. OwnedConstRoxieRow next(input->nextRow());
  104. if(!next)
  105. {
  106. next.setown(input->nextRow());
  107. if(!next)
  108. break;
  109. if(grouped)
  110. buff.append(NULL);
  111. }
  112. insert(next.getClear());
  113. if(count > rowLimit)
  114. return false;
  115. }
  116. return true;
  117. }
  118. void CRowBuffer::clear()
  119. {
  120. buff.clear();
  121. index = 0;
  122. count = 0;
  123. }
  124. const void * CRowBuffer::next()
  125. {
  126. if(buff.isItem(index))
  127. return buff.itemClear(index++);
  128. else
  129. return NULL;
  130. }
  131. //=====================================================================================================
  132. CHThorActivityBase::CHThorActivityBase(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorArg & _help, ThorActivityKind _kind) : agent(_agent), help(_help), outputMeta(help.queryOutputMeta()), kind(_kind), activityId(_activityId), subgraphId(_subgraphId)
  133. {
  134. input = NULL;
  135. processed = 0;
  136. rowAllocator = NULL;
  137. }
  138. void CHThorActivityBase::setInput(unsigned index, IHThorInput *_input)
  139. {
  140. assertex(index == 0);
  141. input = _input;
  142. }
  143. IHThorInput *CHThorActivityBase::queryOutput(unsigned index)
  144. {
  145. agent.fail(255, "internal logic error: CHThorActivityBase::queryOutput");
  146. // never returns....
  147. return NULL;
  148. }
  149. void CHThorActivityBase::ready()
  150. {
  151. if (input)
  152. input->ready();
  153. if (needsAllocator())
  154. createRowAllocator();
  155. initialProcessed = processed;
  156. }
  157. CHThorActivityBase::~CHThorActivityBase()
  158. {
  159. ::Release(rowAllocator);
  160. }
  161. void CHThorActivityBase::createRowAllocator()
  162. {
  163. if (!rowAllocator)
  164. rowAllocator = agent.queryCodeContext()->getRowAllocator(outputMeta.queryOriginal(), activityId);
  165. }
  166. __int64 CHThorActivityBase::getCount()
  167. {
  168. throw MakeStringException(2, "Internal error: CHThorActivityBase::getCount");
  169. return 0;
  170. }
  171. void CHThorActivityBase::execute()
  172. {
  173. agent.fail(255, "internal logic error: CHThorActivityBase::execute");
  174. }
  175. void CHThorActivityBase::extractResult(unsigned & len, void * & ret)
  176. {
  177. agent.fail(255, "internal logic error: CHThorActivityBase::extractResult");
  178. }
  179. void CHThorActivityBase::stop()
  180. {
  181. if (input)
  182. input->stop();
  183. }
  184. void CHThorActivityBase::resetEOF()
  185. {
  186. if (input)
  187. input->resetEOF();
  188. }
  189. void CHThorActivityBase::updateProgress(IStatisticGatherer &progress) const
  190. {
  191. updateProgressForOther(progress, activityId, subgraphId);
  192. if (input)
  193. input->updateProgress(progress);
  194. }
  195. void CHThorActivityBase::updateProgressForOther(IStatisticGatherer &progress, unsigned otherActivity, unsigned otherSubgraph) const
  196. {
  197. updateProgressForOther(progress, otherActivity, otherSubgraph, 0, processed);
  198. }
  199. void CHThorActivityBase::updateProgressForOther(IStatisticGatherer &progress, unsigned otherActivity, unsigned otherSubgraph, unsigned whichOutput, unsigned __int64 numProcessed) const
  200. {
  201. StatsEdgeScope scope(progress, otherActivity, whichOutput);
  202. progress.addStatistic(StNumRowsProcessed, numProcessed);
  203. progress.addStatistic(StNumStarts, 1); // wrong for an activity in a subquery
  204. progress.addStatistic(StNumStops, 1);
  205. progress.addStatistic(StNumSlaves, 1); // MORE: A bit pointless for an hthor graph
  206. }
  207. ILocalEclGraphResults * CHThorActivityBase::resolveLocalQuery(__int64 graphId)
  208. {
  209. return static_cast<ILocalEclGraphResults *>(agent.queryCodeContext()->resolveLocalQuery(graphId));
  210. }
  211. IException * CHThorActivityBase::makeWrappedException(IException * e) const
  212. {
  213. if(dynamic_cast<IHThorException *>(e) || dynamic_cast<IUserException *>(e))
  214. return e;
  215. else
  216. return makeHThorException(kind, activityId, subgraphId, e);
  217. }
  218. IException * CHThorActivityBase::makeWrappedException(IException * e, char const * extra) const
  219. {
  220. if(dynamic_cast<IHThorException *>(e) || dynamic_cast<IUserException *>(e))
  221. return e;
  222. else
  223. return makeHThorException(kind, activityId, subgraphId, e, extra);
  224. }
  225. bool CHThorActivityBase::isPassThrough()
  226. {
  227. return false;
  228. }
  229. //=====================================================================================================
  230. CHThorSimpleActivityBase::CHThorSimpleActivityBase(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorArg & _help, ThorActivityKind _kind) : CHThorActivityBase(_agent, _activityId, _subgraphId, _help, _kind)
  231. {
  232. }
  233. IHThorInput * CHThorSimpleActivityBase::queryOutput(unsigned index)
  234. {
  235. assertex(index == 0);
  236. return this;
  237. }
  238. bool CHThorSimpleActivityBase::isGrouped()
  239. {
  240. return input ? input->isGrouped() : outputMeta.isGrouped();
  241. }
  242. IOutputMetaData * CHThorSimpleActivityBase::queryOutputMeta() const
  243. {
  244. return outputMeta;
  245. }
  246. //=====================================================================================================
  247. class CHThorClusterWriteHandler : public ClusterWriteHandler
  248. {
  249. IAgentContext &agent;
  250. public:
  251. CHThorClusterWriteHandler(char const * _logicalName, char const * _activityType, IAgentContext &_agent)
  252. : agent(_agent), ClusterWriteHandler(_logicalName, _activityType)
  253. {
  254. }
  255. private:
  256. virtual void getTempFilename(StringAttr & out) const
  257. {
  258. StringBuffer buff;
  259. agent.getTempfileBase(buff).append(PATHSEPCHAR).appendf("cluster_write_%p.%" I64F "d_%u", this, (__int64)GetCurrentThreadId(), GetCurrentProcessId());
  260. out.set(buff.str());
  261. }
  262. };
  263. ClusterWriteHandler *createClusterWriteHandler(IAgentContext &agent, IHThorIndexWriteArg *iwHelper, IHThorDiskWriteArg *dwHelper, const char * lfn, StringAttr &fn, bool extend)
  264. {
  265. Owned<CHThorClusterWriteHandler> clusterHandler;
  266. unsigned clusterIdx = 0;
  267. while(true)
  268. {
  269. OwnedRoxieString cluster(iwHelper ? iwHelper->getCluster(clusterIdx++) : dwHelper->getCluster(clusterIdx++));
  270. if(!cluster)
  271. break;
  272. if(!clusterHandler)
  273. {
  274. if(extend)
  275. throw MakeStringException(0, "Cannot combine EXTEND and CLUSTER flags on disk write of file %s", lfn);
  276. clusterHandler.setown(new CHThorClusterWriteHandler(lfn, "OUTPUT", agent));
  277. }
  278. clusterHandler->addCluster(cluster);
  279. }
  280. if(clusterHandler)
  281. {
  282. clusterHandler->getLocalPhysicalFilename(fn);
  283. }
  284. else if (!agent.queryResolveFilesLocally())
  285. {
  286. StringBuffer filenameText;
  287. bool wasDFS;
  288. makeSinglePhysicalPartName(lfn, filenameText, true, wasDFS);
  289. fn.set(filenameText.str());
  290. }
  291. else
  292. {
  293. fn.set(lfn);
  294. }
  295. StringBuffer dir;
  296. splitFilename(fn, &dir, &dir, NULL, NULL);
  297. recursiveCreateDirectory(dir.str());
  298. return clusterHandler.getClear();
  299. }
  300. //=====================================================================================================
  301. CHThorDiskWriteActivity::CHThorDiskWriteActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorDiskWriteArg &_arg, ThorActivityKind _kind) : CHThorActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  302. {
  303. incomplete = false;
  304. }
  305. CHThorDiskWriteActivity::~CHThorDiskWriteActivity()
  306. {
  307. diskout.clear();
  308. if(incomplete)
  309. {
  310. PROGLOG("Disk write incomplete, deleting physical file: %s", filename.get());
  311. diskout.clear();
  312. outSeq.clear();
  313. file->remove();
  314. }
  315. }
  316. void CHThorDiskWriteActivity::ready()
  317. {
  318. CHThorActivityBase::ready();
  319. grouped = (helper.getFlags() & TDXgrouped) != 0;
  320. extend = ((helper.getFlags() & TDWextend) != 0);
  321. overwrite = ((helper.getFlags() & TDWoverwrite) != 0);
  322. resolve();
  323. uncompressedBytesWritten = 0;
  324. numRecords = 0;
  325. sizeLimit = agent.queryWorkUnit()->getDebugValueInt64("hthorDiskWriteSizeLimit", defaultHThorDiskWriteSizeLimit);
  326. rowIf.setown(createRowInterfaces(input->queryOutputMeta(), activityId, 0, agent.queryCodeContext()));
  327. open();
  328. }
  329. void CHThorDiskWriteActivity::execute()
  330. {
  331. // Loop thru the results
  332. numRecords = 0;
  333. while (next())
  334. numRecords++;
  335. finishOutput();
  336. }
  337. void CHThorDiskWriteActivity::stop()
  338. {
  339. outSeq->flush();
  340. if(blockcompressed)
  341. uncompressedBytesWritten = outSeq->getPosition();
  342. updateWorkUnitResult(numRecords);
  343. close();
  344. if((helper.getFlags() & (TDXtemporary | TDXjobtemp) ) == 0 && !agent.queryResolveFilesLocally())
  345. publish();
  346. incomplete = false;
  347. if(clusterHandler)
  348. clusterHandler->finish(file);
  349. CHThorActivityBase::stop();
  350. }
  351. void CHThorDiskWriteActivity::resolve()
  352. {
  353. OwnedRoxieString rawname = helper.getFileName();
  354. mangleHelperFileName(mangledHelperFileName, rawname, agent.queryWuid(), helper.getFlags());
  355. assertex(mangledHelperFileName.str());
  356. if((helper.getFlags() & (TDXtemporary | TDXjobtemp)) == 0)
  357. {
  358. Owned<ILocalOrDistributedFile> f = agent.resolveLFN(mangledHelperFileName.str(),"Cannot write, invalid logical name",true,false,true,&lfn);
  359. if (f)
  360. {
  361. if (f->queryDistributedFile())
  362. {
  363. // An already existing dali file
  364. if(extend)
  365. agent.logFileAccess(f->queryDistributedFile(), "HThor", "EXTENDED");
  366. else if(overwrite) {
  367. PrintLog("Removing %s from DFS", lfn.str());
  368. agent.logFileAccess(f->queryDistributedFile(), "HThor", "DELETED");
  369. if (!agent.queryResolveFilesLocally())
  370. f->queryDistributedFile()->detach();
  371. else
  372. {
  373. Owned<IFile> file = createIFile(lfn);
  374. if (file->exists())
  375. file->remove();
  376. }
  377. }
  378. else
  379. throw MakeStringException(99, "Cannot write %s, file already exists (missing OVERWRITE attribute?)", lfn.str());
  380. }
  381. else if (f->exists() || f->isExternal() || agent.queryResolveFilesLocally())
  382. {
  383. // special/local/external file
  384. if (f->numParts()!=1)
  385. throw MakeStringException(99, "Cannot write %s, external file has multiple parts)", lfn.str());
  386. RemoteFilename rfn;
  387. f->getPartFilename(rfn,0);
  388. StringBuffer full;
  389. if (rfn.isLocal())
  390. rfn.getLocalPath(full);
  391. else
  392. rfn.getRemotePath(full);
  393. filename.set(full);
  394. if (isSpecialPath(filename))
  395. {
  396. PROGLOG("Writing to query %s", filename.get());
  397. return;
  398. }
  399. if (stdIoHandle(filename)>=0) {
  400. PROGLOG("Writing to %s", filename.get());
  401. return;
  402. }
  403. Owned<IFile> file = createIFile(filename);
  404. if (file->exists())
  405. {
  406. if (!overwrite)
  407. throw MakeStringException(99, "Cannot write %s, file already exists (missing OVERWRITE attribute?)", full.str());
  408. file->remove();
  409. }
  410. //Ensure target folder exists
  411. if (!recursiveCreateDirectoryForFile(filename.get()))
  412. {
  413. throw MakeStringException(99, "Cannot create file folder for %s", filename.str());
  414. }
  415. PROGLOG("Writing to file %s", filename.get());
  416. }
  417. f.clear();
  418. }
  419. if (filename.isEmpty()) // wasn't local or special (i.e. DFS file)
  420. {
  421. CDfsLogicalFileName dfsLogicalName;
  422. dfsLogicalName.allowOsPath(agent.queryResolveFilesLocally());
  423. if (!dfsLogicalName.setValidate(lfn.str()))
  424. {
  425. throw MakeStringException(99, "Could not resolve DFS Logical file %s", lfn.str());
  426. }
  427. clusterHandler.setown(createClusterWriteHandler(agent, NULL, &helper, dfsLogicalName.get(), filename, extend));
  428. }
  429. }
  430. else
  431. {
  432. StringBuffer mangledName;
  433. mangleLocalTempFilename(mangledName, mangledHelperFileName.str());
  434. filename.set(agent.noteTemporaryFile(mangledName.str()));
  435. PROGLOG("DISKWRITE: using temporary filename %s", filename.get());
  436. }
  437. }
  438. void CHThorDiskWriteActivity::open()
  439. {
  440. // Open an output file...
  441. file.setown(createIFile(filename));
  442. serializedOutputMeta.set(input->queryOutputMeta()->querySerializedDiskMeta());//returns outputMeta if serialization not needed
  443. Linked<IRecordSize> groupedMeta = input->queryOutputMeta()->querySerializedDiskMeta();
  444. if(grouped)
  445. groupedMeta.setown(createDeltaRecordSize(groupedMeta, +1));
  446. blockcompressed = checkIsCompressed(helper.getFlags(), serializedOutputMeta.getFixedSize(), grouped);//TDWnewcompress for new compression, else check for row compression
  447. void *ekey;
  448. size32_t ekeylen;
  449. helper.getEncryptKey(ekeylen,ekey);
  450. encrypted = false;
  451. Owned<ICompressor> ecomp;
  452. if (ekeylen!=0) {
  453. ecomp.setown(createAESCompressor256(ekeylen,ekey));
  454. memset(ekey,0,ekeylen);
  455. rtlFree(ekey);
  456. encrypted = true;
  457. blockcompressed = true;
  458. }
  459. Owned<IFileIO> io;
  460. if(blockcompressed)
  461. io.setown(createCompressedFileWriter(file, groupedMeta->getFixedSize(), extend, true, ecomp));
  462. else
  463. io.setown(file->open(extend ? IFOwrite : IFOcreate));
  464. if(!io)
  465. throw MakeStringException(errno, "Failed to create%s file %s for writing", (encrypted ? " encrypted" : (blockcompressed ? " compressed" : "")), filename.get());
  466. incomplete = true;
  467. diskout.setown(createBufferedIOStream(io));
  468. if(extend)
  469. diskout->seek(0, IFSend);
  470. unsigned rwFlags = rw_autoflush;
  471. if(grouped)
  472. rwFlags |= rw_grouped;
  473. if(!agent.queryWorkUnit()->getDebugValueBool("skipFileFormatCrcCheck", false) && !(helper.getFlags() & TDRnocrccheck))
  474. rwFlags |= rw_crc;
  475. IExtRowWriter * writer = createRowWriter(diskout, rowIf, rwFlags);
  476. outSeq.setown(writer);
  477. }
  478. const void * CHThorDiskWriteActivity::getNext()
  479. { // through operation (writes and returns row)
  480. // needs a one row lookahead to preserve group
  481. if (!nextrow.get())
  482. {
  483. nextrow.setown(input->nextRow());
  484. if (!nextrow.get())
  485. {
  486. nextrow.setown(input->nextRow());
  487. if (nextrow.get()&&grouped) // only write eog if not at eof
  488. outSeq->putRow(NULL);
  489. return NULL;
  490. }
  491. }
  492. outSeq->putRow(nextrow.getLink());
  493. checkSizeLimit();
  494. return nextrow.getClear();
  495. }
  496. bool CHThorDiskWriteActivity::next()
  497. {
  498. if (!nextrow.get())
  499. {
  500. OwnedConstRoxieRow row(input->nextRow());
  501. if (!row.get())
  502. {
  503. row.setown(input->nextRow());
  504. if (!row.get())
  505. return false; // we are done
  506. if (grouped)
  507. outSeq->putRow(NULL);
  508. }
  509. outSeq->putRow(row.getClear());
  510. }
  511. else
  512. outSeq->putRow(nextrow.getClear());
  513. checkSizeLimit();
  514. return true;
  515. }
  516. void CHThorDiskWriteActivity::finishOutput()
  517. {
  518. }
  519. void CHThorDiskWriteActivity::close()
  520. {
  521. diskout.clear();
  522. outSeq.clear();
  523. if(clusterHandler)
  524. clusterHandler->copyPhysical(file, agent.queryWorkUnit()->getDebugValueBool("__output_cluster_no_copy_physical", false));
  525. }
  526. void CHThorDiskWriteActivity::publish()
  527. {
  528. StringBuffer dir,base;
  529. offset_t fileSize = file->size();
  530. if(clusterHandler)
  531. clusterHandler->splitPhysicalFilename(dir, base);
  532. else
  533. splitFilename(filename, &dir, &dir, &base, &base);
  534. Owned<IFileDescriptor> desc = createFileDescriptor();
  535. desc->setDefaultDir(dir.str());
  536. Owned<IPropertyTree> attrs;
  537. if(clusterHandler)
  538. attrs.setown(createPTree("Part")); // clusterHandler is going to set attributes
  539. else
  540. {
  541. // add cluster
  542. StringBuffer mygroupname;
  543. Owned<IGroup> mygrp = agent.getHThorGroup(mygroupname);
  544. ClusterPartDiskMapSpec partmap; // will get this from group at some point
  545. desc->setNumParts(1);
  546. desc->setPartMask(base.str());
  547. desc->addCluster(mygroupname.str(),mygrp, partmap);
  548. attrs.set(&desc->queryPart(0)->queryProperties());
  549. }
  550. //properties of the first file part.
  551. if(blockcompressed)
  552. {
  553. attrs->setPropInt64("@size", uncompressedBytesWritten);
  554. attrs->setPropInt64("@compressedSize", fileSize);
  555. }
  556. else
  557. attrs->setPropInt64("@size", fileSize);
  558. CDateTime createTime, modifiedTime, accessedTime;
  559. file->getTime(&createTime, &modifiedTime, &accessedTime);
  560. // round file time down to nearest sec. Nanosec accurancy is not preserved elsewhere and can lead to mismatch later.
  561. unsigned hour, min, sec, nanosec;
  562. modifiedTime.getTime(hour, min, sec, nanosec);
  563. modifiedTime.setTime(hour, min, sec, 0);
  564. StringBuffer timestr;
  565. modifiedTime.getString(timestr);
  566. if(timestr.length())
  567. attrs->setProp("@modified", timestr.str());
  568. if(clusterHandler)
  569. clusterHandler->setDescriptorParts(desc, base.str(), attrs);
  570. // properties of the logical file
  571. IPropertyTree & properties = desc->queryProperties();
  572. properties.setPropInt64("@size", (blockcompressed) ? uncompressedBytesWritten : fileSize);
  573. if (encrypted)
  574. properties.setPropBool("@encrypted", true);
  575. if (blockcompressed)
  576. {
  577. properties.setPropInt64("@compressedSize", fileSize);
  578. properties.setPropBool("@blockCompressed", true);
  579. }
  580. if (helper.getFlags() & TDWpersist)
  581. properties.setPropBool("@persistent", true);
  582. if (grouped)
  583. properties.setPropBool("@grouped", true);
  584. properties.setPropInt64("@recordCount", numRecords);
  585. properties.setProp("@owner", agent.queryWorkUnit()->queryUser());
  586. if (helper.getFlags() & (TDWowned|TDXjobtemp|TDXtemporary))
  587. properties.setPropBool("@owned", true);
  588. if (helper.getFlags() & TDWresult)
  589. properties.setPropBool("@result", true);
  590. properties.setProp("@workunit", agent.queryWorkUnit()->queryWuid());
  591. properties.setProp("@job", agent.queryWorkUnit()->queryJobName());
  592. setFormat(desc);
  593. if (helper.getFlags() & TDWexpires)
  594. setExpiryTime(properties, helper.getExpiryDays());
  595. if (helper.getFlags() & TDWupdate)
  596. {
  597. unsigned eclCRC;
  598. unsigned __int64 totalCRC;
  599. helper.getUpdateCRCs(eclCRC, totalCRC);
  600. properties.setPropInt("@eclCRC", eclCRC);
  601. properties.setPropInt64("@totalCRC", totalCRC);
  602. }
  603. properties.setPropInt("@formatCrc", helper.getFormatCrc());
  604. StringBuffer lfn;
  605. expandLogicalFilename(lfn, mangledHelperFileName.str(), agent.queryWorkUnit(), agent.queryResolveFilesLocally(), false);
  606. CDfsLogicalFileName logicalName;
  607. if (agent.queryResolveFilesLocally())
  608. logicalName.allowOsPath(true);
  609. if (!logicalName.setValidate(lfn.str()))
  610. throw MakeStringException(99, "Cannot publish %s, invalid logical name", lfn.str());
  611. if (!logicalName.isExternal()) // no need to publish externals
  612. {
  613. Owned<IDistributedFile> file = queryDistributedFileDirectory().createNew(desc);
  614. if(file->getModificationTime(modifiedTime))
  615. file->setAccessedTime(modifiedTime);
  616. file->attach(logicalName.get(), agent.queryCodeContext()->queryUserDescriptor());
  617. agent.logFileAccess(file, "HThor", "CREATED");
  618. }
  619. }
  620. void CHThorDiskWriteActivity::updateWorkUnitResult(unsigned __int64 reccount)
  621. {
  622. if(lfn.length()) //this is required as long as temp files don't get a name which can be stored in the WU and automatically deleted by the WU
  623. {
  624. WorkunitUpdate wu = agent.updateWorkUnit();
  625. StringArray clusters;
  626. if (clusterHandler)
  627. clusterHandler->getClusters(clusters);
  628. else
  629. clusters.append(wu->queryClusterName());
  630. unsigned flags = helper.getFlags();
  631. if (!agent.queryResolveFilesLocally())
  632. {
  633. WUFileKind fileKind;
  634. if (TDXtemporary & flags)
  635. fileKind = WUFileTemporary;
  636. else if(TDXjobtemp & flags)
  637. fileKind = WUFileJobOwned;
  638. else if(TDWowned & flags)
  639. fileKind = WUFileOwned;
  640. else
  641. fileKind = WUFileStandard;
  642. wu->addFile(lfn.str(), &clusters, helper.getTempUsageCount(), fileKind, NULL);
  643. }
  644. else if ((TDXtemporary | TDXjobtemp) & flags)
  645. agent.noteTemporaryFilespec(filename);//note for later deletion
  646. if (!(flags & TDXtemporary) && helper.getSequence() >= 0)
  647. {
  648. Owned<IWUResult> result = wu->updateResultBySequence(helper.getSequence());
  649. if (result)
  650. {
  651. result->setResultTotalRowCount(reccount);
  652. result->setResultStatus(ResultStatusCalculated);
  653. if (helper.getFlags() & TDWresult)
  654. result->setResultFilename(lfn.str());
  655. else
  656. result->setResultLogicalName(lfn.str());
  657. }
  658. }
  659. }
  660. }
  661. void CHThorDiskWriteActivity::setFormat(IFileDescriptor * desc)
  662. {
  663. if ((serializedOutputMeta.isFixedSize()) && !isOutputTransformed())
  664. desc->queryProperties().setPropInt("@recordSize", serializedOutputMeta.getFixedSize() + (grouped ? 1 : 0));
  665. const char *recordECL = helper.queryRecordECL();
  666. if (recordECL && *recordECL)
  667. desc->queryProperties().setProp("ECL", recordECL);
  668. desc->queryProperties().setProp("@kind", "flat");
  669. }
  670. void CHThorDiskWriteActivity::checkSizeLimit()
  671. {
  672. if(sizeLimit && outSeq && (outSeq->getPosition() > sizeLimit))
  673. {
  674. StringBuffer msg;
  675. msg.append("Exceeded disk write size limit of ").append(sizeLimit).append(" while writing file ").append(mangledHelperFileName.str());
  676. throw MakeStringExceptionDirect(0, msg.str());
  677. }
  678. }
  679. //=====================================================================================================
  680. CHThorSpillActivity::CHThorSpillActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorSpillArg &_arg, ThorActivityKind _kind) : CHThorDiskWriteActivity(_agent, _activityId, _subgraphId, _arg, _kind)
  681. {
  682. }
  683. void CHThorSpillActivity::setInput(unsigned index, IHThorInput *_input)
  684. {
  685. CHThorActivityBase::setInput(index, _input);
  686. }
  687. void CHThorSpillActivity::ready()
  688. {
  689. CHThorDiskWriteActivity::ready();
  690. }
  691. void CHThorSpillActivity::execute()
  692. {
  693. UNIMPLEMENTED;
  694. }
  695. const void *CHThorSpillActivity::nextRow()
  696. {
  697. const void *nextrec = getNext();
  698. if (nextrec)
  699. {
  700. numRecords++;
  701. processed++;
  702. }
  703. return nextrec;
  704. }
  705. void CHThorSpillActivity::stop()
  706. {
  707. for (;;)
  708. {
  709. OwnedConstRoxieRow nextrec(nextRow());
  710. if (!nextrec)
  711. {
  712. nextrec.setown(nextRow());
  713. if (!nextrec)
  714. break;
  715. }
  716. }
  717. finishOutput();
  718. CHThorDiskWriteActivity::stop();
  719. }
  720. //=====================================================================================================
  721. CHThorCsvWriteActivity::CHThorCsvWriteActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorCsvWriteArg &_arg, ThorActivityKind _kind) : CHThorDiskWriteActivity(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  722. {
  723. csvOutput.init(helper.queryCsvParameters(),agent.queryWorkUnit()->getDebugValueBool("oldCSVoutputFormat", false));
  724. }
  725. void CHThorCsvWriteActivity::execute()
  726. {
  727. OwnedRoxieString header(helper.queryCsvParameters()->getHeader());
  728. if (header) {
  729. csvOutput.beginLine();
  730. csvOutput.writeHeaderLn(strlen(header), header);
  731. diskout->write(csvOutput.length(), csvOutput.str());
  732. }
  733. // Loop thru the results
  734. numRecords = 0;
  735. for (;;)
  736. {
  737. OwnedConstRoxieRow nextrec(input->nextRow());
  738. if (!nextrec)
  739. {
  740. nextrec.setown(input->nextRow());
  741. if (!nextrec)
  742. break;
  743. }
  744. try
  745. {
  746. csvOutput.beginLine();
  747. helper.writeRow((const byte *)nextrec.get(), &csvOutput);
  748. csvOutput.endLine();
  749. }
  750. catch(IException * e)
  751. {
  752. throw makeWrappedException(e);
  753. }
  754. diskout->write(csvOutput.length(), csvOutput.str());
  755. numRecords++;
  756. }
  757. OwnedRoxieString footer(helper.queryCsvParameters()->getFooter());
  758. if (footer) {
  759. csvOutput.beginLine();
  760. csvOutput.writeHeaderLn(strlen(footer), footer);
  761. diskout->write(csvOutput.length(), csvOutput.str());
  762. }
  763. }
  764. void CHThorCsvWriteActivity::setFormat(IFileDescriptor * desc)
  765. {
  766. // MORE - should call parent's setFormat too?
  767. ICsvParameters * csvInfo = helper.queryCsvParameters();
  768. OwnedRoxieString rs(csvInfo->getSeparator(0));
  769. StringBuffer separator;
  770. const char *s = rs;
  771. while (s && *s)
  772. {
  773. if (',' == *s)
  774. separator.append("\\,");
  775. else
  776. separator.append(*s);
  777. ++s;
  778. }
  779. desc->queryProperties().setProp("@csvSeparate", separator.str());
  780. desc->queryProperties().setProp("@csvQuote", rs.setown(csvInfo->getQuote(0)));
  781. desc->queryProperties().setProp("@csvTerminate", rs.setown(csvInfo->getTerminator(0)));
  782. desc->queryProperties().setProp("@csvEscape", rs.setown(csvInfo->getEscape(0)));
  783. desc->queryProperties().setProp("@format","utf8n");
  784. desc->queryProperties().setProp("@kind", "csv");
  785. const char *recordECL = helper.queryRecordECL();
  786. if (recordECL && *recordECL)
  787. desc->queryProperties().setProp("ECL", recordECL);
  788. }
  789. //=====================================================================================================
  790. CHThorXmlWriteActivity::CHThorXmlWriteActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorXmlWriteArg &_arg, ThorActivityKind _kind) : CHThorDiskWriteActivity(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg), headerLength(0), footerLength(0)
  791. {
  792. OwnedRoxieString xmlpath(helper.getXmlIteratorPath());
  793. if (!xmlpath)
  794. rowTag.append(DEFAULTXMLROWTAG);
  795. else
  796. {
  797. const char *path = xmlpath;
  798. if (*path == '/') path++;
  799. if (strchr(path, '/')) UNIMPLEMENTED; // more what do we do with /mydata/row
  800. rowTag.append(path);
  801. }
  802. }
  803. void CHThorXmlWriteActivity::execute()
  804. {
  805. // Loop thru the results
  806. numRecords = 0;
  807. StringBuffer header;
  808. OwnedRoxieString suppliedHeader(helper.getHeader());
  809. if (kind==TAKjsonwrite)
  810. buildJsonHeader(header, suppliedHeader, rowTag);
  811. else if (suppliedHeader)
  812. header.set(suppliedHeader);
  813. else
  814. header.append(DEFAULTXMLHEADER).newline();
  815. headerLength = header.length();
  816. diskout->write(headerLength, header.str());
  817. Owned<IXmlWriterExt> writer = createIXmlWriterExt(helper.getXmlFlags(), 0, NULL, (kind==TAKjsonwrite) ? WTJSON : WTStandard);
  818. writer->outputBeginArray(rowTag); //need to set up the array
  819. writer->clear(); //but not output it
  820. for (;;)
  821. {
  822. OwnedConstRoxieRow nextrec(input->nextRow());
  823. if (!nextrec)
  824. {
  825. nextrec.setown(input->nextRow());
  826. if (!nextrec)
  827. break;
  828. }
  829. try
  830. {
  831. writer->clear().outputBeginNested(rowTag, false);
  832. helper.toXML((const byte *)nextrec.get(), *writer);
  833. writer->outputEndNested(rowTag);
  834. }
  835. catch(IException * e)
  836. {
  837. throw makeWrappedException(e);
  838. }
  839. diskout->write(writer->length(), writer->str());
  840. numRecords++;
  841. }
  842. OwnedRoxieString suppliedFooter(helper.getFooter());
  843. StringBuffer footer;
  844. if (kind==TAKjsonwrite)
  845. buildJsonFooter(footer.newline(), suppliedFooter, rowTag);
  846. else if (suppliedFooter)
  847. footer.append(suppliedFooter);
  848. else
  849. footer.append(DEFAULTXMLFOOTER).newline();
  850. footerLength=footer.length();
  851. diskout->write(footerLength, footer);
  852. }
  853. void CHThorXmlWriteActivity::setFormat(IFileDescriptor * desc)
  854. {
  855. desc->queryProperties().setProp("@format","utf8n");
  856. desc->queryProperties().setProp("@rowTag",rowTag.str());
  857. desc->queryProperties().setProp("@kind", (kind==TAKjsonwrite) ? "json" : "xml");
  858. desc->queryProperties().setPropInt(FPheaderLength, headerLength);
  859. desc->queryProperties().setPropInt(FPfooterLength, footerLength);
  860. const char *recordECL = helper.queryRecordECL();
  861. if (recordECL && *recordECL)
  862. desc->queryProperties().setProp("ECL", recordECL);
  863. }
  864. //=====================================================================================================
  865. void throwPipeProcessError(unsigned err, char const * preposition, char const * program, IPipeProcess * pipe)
  866. {
  867. StringBuffer msg;
  868. msg.append("Error piping ").append(preposition).append(" (").append(program).append("): process failed with code ").append(err);
  869. if(pipe->hasError())
  870. {
  871. try
  872. {
  873. char error[512];
  874. size32_t sz = pipe->readError(sizeof(error), error);
  875. if(sz && sz!=(size32_t)-1)
  876. msg.append(", stderr: '").append(sz, error).append("'");
  877. }
  878. catch (IException *e)
  879. {
  880. EXCLOG(e, "Error reading pipe stderr");
  881. e->Release();
  882. }
  883. }
  884. throw MakeStringExceptionDirect(2, msg.str());
  885. }
  886. //=====================================================================================================
  887. CHThorIndexWriteActivity::CHThorIndexWriteActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorIndexWriteArg &_arg, ThorActivityKind _kind) : CHThorActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  888. {
  889. incomplete = false;
  890. StringBuffer lfn;
  891. OwnedRoxieString fname(helper.getFileName());
  892. expandLogicalFilename(lfn, fname, agent.queryWorkUnit(), agent.queryResolveFilesLocally(), false);
  893. if (!agent.queryResolveFilesLocally())
  894. {
  895. Owned<IDistributedFile> f = queryDistributedFileDirectory().lookup(lfn, agent.queryCodeContext()->queryUserDescriptor(), true);
  896. if (f)
  897. {
  898. if (TIWoverwrite & helper.getFlags())
  899. {
  900. PrintLog("Removing %s from DFS", lfn.str());
  901. agent.logFileAccess(f, "HThor", "DELETED");
  902. f->detach();
  903. }
  904. else // not quite sure about raising exceptions in constructors
  905. throw MakeStringException(99, "Cannot write %s, file already exists (missing OVERWRITE attribute?)", lfn.str());
  906. }
  907. }
  908. clusterHandler.setown(createClusterWriteHandler(agent, &helper, NULL, lfn, filename, false));
  909. sizeLimit = agent.queryWorkUnit()->getDebugValueInt64("hthorDiskWriteSizeLimit", defaultHThorDiskWriteSizeLimit);
  910. }
  911. CHThorIndexWriteActivity::~CHThorIndexWriteActivity()
  912. {
  913. if(incomplete)
  914. {
  915. PROGLOG("Index write incomplete, deleting physical file: %s", filename.get());
  916. file->remove();
  917. }
  918. }
  919. void CHThorIndexWriteActivity::execute()
  920. {
  921. size32_t maxDiskRecordSize;
  922. if (helper.queryDiskRecordSize()->isVariableSize())
  923. {
  924. if (helper.getFlags() & TIWmaxlength)
  925. maxDiskRecordSize = helper.getMaxKeySize();
  926. else
  927. maxDiskRecordSize = KEYBUILD_MAXLENGTH; // Current default behaviour, could be improved in the future
  928. }
  929. else
  930. maxDiskRecordSize = helper.queryDiskRecordSize()->getFixedSize();
  931. if (maxDiskRecordSize > KEYBUILD_MAXLENGTH)
  932. throw MakeStringException(99, "Index maximum record length (%d) exceeds 32K internal limit", maxDiskRecordSize);
  933. OwnedMalloc<char> rowBuffer(maxDiskRecordSize, true);
  934. // Loop thru the results
  935. unsigned __int64 reccount = 0;
  936. unsigned int fileCrc = -1;
  937. file.setown(createIFile(filename.get()));
  938. {
  939. OwnedIFileIO io;
  940. try
  941. {
  942. io.setown(file->open(IFOcreate));
  943. }
  944. catch(IException * e)
  945. {
  946. e->Release();
  947. clearKeyStoreCache(false);
  948. io.setown(file->open(IFOcreate));
  949. }
  950. incomplete = true;
  951. Owned<IFileIOStream> out = createIOStream(io);
  952. bool isVariable = helper.queryDiskRecordSize()->isVariableSize();
  953. unsigned flags = COL_PREFIX | HTREE_FULLSORT_KEY;
  954. if (helper.getFlags() & TIWrowcompress)
  955. flags |= HTREE_COMPRESSED_KEY|HTREE_QUICK_COMPRESSED_KEY;
  956. else if (!(helper.getFlags() & TIWnolzwcompress))
  957. flags |= HTREE_COMPRESSED_KEY;
  958. if (isVariable)
  959. flags |= HTREE_VARSIZE;
  960. Owned<IPropertyTree> metadata;
  961. buildUserMetadata(metadata);
  962. buildLayoutMetadata(metadata);
  963. unsigned nodeSize = metadata ? metadata->getPropInt("_nodeSize", NODESIZE) : NODESIZE;
  964. size32_t keyMaxSize = helper.queryDiskRecordSize()->getRecordSize(NULL);
  965. if (hasTrailingFileposition(helper.queryDiskRecordSize()->queryTypeInfo()))
  966. keyMaxSize -= sizeof(offset_t);
  967. Owned<IKeyBuilder> builder = createKeyBuilder(out, flags, keyMaxSize, nodeSize, helper.getKeyedSize(), 0);
  968. class BcWrapper : implements IBlobCreator
  969. {
  970. IKeyBuilder *builder;
  971. public:
  972. BcWrapper(IKeyBuilder *_builder) : builder(_builder) {}
  973. virtual unsigned __int64 createBlob(size32_t size, const void * ptr)
  974. {
  975. return builder->createBlob(size, (const char *) ptr);
  976. }
  977. } bc(builder);
  978. for (;;)
  979. {
  980. OwnedConstRoxieRow nextrec(input->nextRow());
  981. if (!nextrec)
  982. {
  983. nextrec.setown(input->nextRow());
  984. if (!nextrec)
  985. break;
  986. }
  987. try
  988. {
  989. unsigned __int64 fpos;
  990. RtlStaticRowBuilder rowBuilder(rowBuffer, maxDiskRecordSize);
  991. size32_t thisSize = helper.transform(rowBuilder, nextrec, &bc, fpos);
  992. builder->processKeyData(rowBuffer, fpos, thisSize);
  993. }
  994. catch(IException * e)
  995. {
  996. throw makeWrappedException(e);
  997. }
  998. if(sizeLimit && (out->tell() > sizeLimit))
  999. {
  1000. StringBuffer msg;
  1001. OwnedRoxieString fname(helper.getFileName());
  1002. msg.append("Exceeded disk write size limit of ").append(sizeLimit).append(" while writing index ").append(fname);
  1003. throw MakeStringExceptionDirect(0, msg.str());
  1004. }
  1005. reccount++;
  1006. }
  1007. builder->finish(metadata, &fileCrc);
  1008. out->flush();
  1009. out.clear();
  1010. }
  1011. if(clusterHandler)
  1012. clusterHandler->copyPhysical(file, agent.queryWorkUnit()->getDebugValueBool("__output_cluster_no_copy_physical", false));
  1013. // Now publish to name services
  1014. StringBuffer dir,base;
  1015. offset_t indexFileSize = file->size();
  1016. if(clusterHandler)
  1017. clusterHandler->splitPhysicalFilename(dir, base);
  1018. else
  1019. splitFilename(filename, &dir, &dir, &base, &base);
  1020. Owned<IFileDescriptor> desc = createFileDescriptor();
  1021. desc->setDefaultDir(dir.str());
  1022. //properties of the first file part.
  1023. Owned<IPropertyTree> attrs;
  1024. if(clusterHandler)
  1025. attrs.setown(createPTree("Part")); // clusterHandler is going to set attributes
  1026. else
  1027. {
  1028. // add cluster
  1029. StringBuffer mygroupname;
  1030. Owned<IGroup> mygrp = NULL;
  1031. if (!agent.queryResolveFilesLocally())
  1032. mygrp.setown(agent.getHThorGroup(mygroupname));
  1033. ClusterPartDiskMapSpec partmap; // will get this from group at some point
  1034. desc->setNumParts(1);
  1035. desc->setPartMask(base.str());
  1036. desc->addCluster(mygroupname.str(),mygrp, partmap);
  1037. attrs.set(&desc->queryPart(0)->queryProperties());
  1038. }
  1039. attrs->setPropInt64("@size", indexFileSize);
  1040. CDateTime createTime, modifiedTime, accessedTime;
  1041. file->getTime(&createTime, &modifiedTime, &accessedTime);
  1042. // round file time down to nearest sec. Nanosec accurancy is not preserved elsewhere and can lead to mismatch later.
  1043. unsigned hour, min, sec, nanosec;
  1044. modifiedTime.getTime(hour, min, sec, nanosec);
  1045. modifiedTime.setTime(hour, min, sec, 0);
  1046. StringBuffer timestr;
  1047. modifiedTime.getString(timestr);
  1048. if(timestr.length())
  1049. attrs->setProp("@modified", timestr.str());
  1050. if(clusterHandler)
  1051. clusterHandler->setDescriptorParts(desc, base.str(), attrs);
  1052. // properties of the logical file
  1053. IPropertyTree & properties = desc->queryProperties();
  1054. properties.setProp("@kind", "key");
  1055. properties.setPropInt64("@size", indexFileSize);
  1056. properties.setPropInt64("@recordCount", reccount);
  1057. properties.setProp("@owner", agent.queryWorkUnit()->queryUser());
  1058. properties.setProp("@workunit", agent.queryWorkUnit()->queryWuid());
  1059. properties.setProp("@job", agent.queryWorkUnit()->queryJobName());
  1060. char const * rececl = helper.queryRecordECL();
  1061. if(rececl && *rececl)
  1062. properties.setProp("ECL", rececl);
  1063. if (helper.getFlags() & TIWexpires)
  1064. setExpiryTime(properties, helper.getExpiryDays());
  1065. if (helper.getFlags() & TIWupdate)
  1066. {
  1067. unsigned eclCRC;
  1068. unsigned __int64 totalCRC;
  1069. helper.getUpdateCRCs(eclCRC, totalCRC);
  1070. properties.setPropInt("@eclCRC", eclCRC);
  1071. properties.setPropInt64("@totalCRC", totalCRC);
  1072. }
  1073. properties.setPropInt("@fileCrc", fileCrc);
  1074. properties.setPropInt("@formatCrc", helper.getFormatCrc());
  1075. // Legacy record layout info
  1076. void * layoutMetaBuff;
  1077. size32_t layoutMetaSize;
  1078. if(helper.getIndexLayout(layoutMetaSize, layoutMetaBuff))
  1079. {
  1080. properties.setPropBin("_record_layout", layoutMetaSize, layoutMetaBuff);
  1081. rtlFree(layoutMetaBuff);
  1082. }
  1083. // New record layout info
  1084. if (helper.queryDiskRecordSize()->queryTypeInfo())
  1085. {
  1086. MemoryBuffer out;
  1087. dumpTypeInfo(out, helper.queryDiskRecordSize()->queryTypeInfo());
  1088. properties.setPropBin("_rtlType", out.length(), out.toByteArray());
  1089. }
  1090. StringBuffer lfn;
  1091. Owned<IDistributedFile> dfile = NULL;
  1092. if (!agent.queryResolveFilesLocally())
  1093. {
  1094. dfile.setown(queryDistributedFileDirectory().createNew(desc));
  1095. OwnedRoxieString fname(helper.getFileName());
  1096. expandLogicalFilename(lfn, fname, agent.queryWorkUnit(), agent.queryResolveFilesLocally(), false);
  1097. dfile->attach(lfn.str(),agent.queryCodeContext()->queryUserDescriptor());
  1098. agent.logFileAccess(dfile, "HThor", "CREATED");
  1099. }
  1100. else
  1101. lfn = filename;
  1102. incomplete = false;
  1103. if(clusterHandler)
  1104. clusterHandler->finish(file);
  1105. // and update wu info
  1106. if (helper.getSequence() >= 0)
  1107. {
  1108. WorkunitUpdate wu = agent.updateWorkUnit();
  1109. Owned<IWUResult> result = wu->updateResultBySequence(helper.getSequence());
  1110. if (result)
  1111. {
  1112. result->setResultTotalRowCount(reccount);
  1113. result->setResultStatus(ResultStatusCalculated);
  1114. result->setResultLogicalName(lfn.str());
  1115. }
  1116. }
  1117. }
  1118. void CHThorIndexWriteActivity::buildUserMetadata(Owned<IPropertyTree> & metadata)
  1119. {
  1120. size32_t nameLen;
  1121. char * nameBuff;
  1122. size32_t valueLen;
  1123. char * valueBuff;
  1124. unsigned idx = 0;
  1125. while(helper.getIndexMeta(nameLen, nameBuff, valueLen, valueBuff, idx++))
  1126. {
  1127. StringBuffer name(nameLen, nameBuff);
  1128. StringBuffer value(valueLen, valueBuff);
  1129. if(*nameBuff == '_' && strcmp(name, "_nodeSize") != 0)
  1130. {
  1131. OwnedRoxieString fname(helper.getFileName());
  1132. throw MakeStringException(0, "Invalid name %s in user metadata for index %s (names beginning with underscore are reserved)", name.str(), fname.get());
  1133. }
  1134. if(!validateXMLTag(name.str()))
  1135. {
  1136. OwnedRoxieString fname(helper.getFileName());
  1137. throw MakeStringException(0, "Invalid name %s in user metadata for index %s (not legal XML element name)", name.str(), fname.get());
  1138. }
  1139. if(!metadata) metadata.setown(createPTree("metadata"));
  1140. metadata->setProp(name.str(), value.str());
  1141. }
  1142. }
  1143. void CHThorIndexWriteActivity::buildLayoutMetadata(Owned<IPropertyTree> & metadata)
  1144. {
  1145. if(!metadata) metadata.setown(createPTree("metadata"));
  1146. metadata->setProp("_record_ECL", helper.queryRecordECL());
  1147. if (helper.queryDiskRecordSize()->queryTypeInfo())
  1148. {
  1149. MemoryBuffer out;
  1150. dumpTypeInfo(out, helper.queryDiskRecordSize()->queryTypeInfo());
  1151. metadata->setPropBin("_rtlType", out.length(), out.toByteArray());
  1152. }
  1153. }
  1154. //=====================================================================================================
  1155. class CHThorPipeReadActivity : public CHThorSimpleActivityBase
  1156. {
  1157. IHThorPipeReadArg &helper;
  1158. Owned<IPipeProcess> pipe;
  1159. StringAttr pipeCommand;
  1160. Owned<IOutputRowDeserializer> rowDeserializer;
  1161. Owned<IReadRowStream> readTransformer;
  1162. bool groupSignalled;
  1163. public:
  1164. CHThorPipeReadActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorPipeReadArg &_arg, ThorActivityKind _kind)
  1165. : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  1166. {
  1167. groupSignalled = true;
  1168. }
  1169. virtual bool needsAllocator() const { return true; }
  1170. virtual void ready()
  1171. {
  1172. groupSignalled = true; // i.e. don't start with a NULL row
  1173. CHThorSimpleActivityBase::ready();
  1174. rowDeserializer.setown(rowAllocator->createDiskDeserializer(agent.queryCodeContext()));
  1175. OwnedRoxieString xmlIteratorPath(helper.getXmlIteratorPath());
  1176. readTransformer.setown(createReadRowStream(rowAllocator, rowDeserializer, helper.queryXmlTransformer(), helper.queryCsvTransformer(), xmlIteratorPath, helper.getPipeFlags()));
  1177. OwnedRoxieString pipeProgram(helper.getPipeProgram());
  1178. openPipe(pipeProgram);
  1179. }
  1180. virtual void stop()
  1181. {
  1182. //Need to close the output (or read it in its entirety), otherwise we might wait forever for the
  1183. //program to finish
  1184. if (pipe)
  1185. pipe->closeOutput();
  1186. pipe.clear();
  1187. readTransformer->setStream(NULL);
  1188. CHThorSimpleActivityBase::stop();
  1189. }
  1190. virtual const void *nextRow()
  1191. {
  1192. while (!waitForPipe())
  1193. {
  1194. if (!pipe)
  1195. return NULL;
  1196. if (helper.getPipeFlags() & TPFgroupeachrow)
  1197. {
  1198. if (!groupSignalled)
  1199. {
  1200. groupSignalled = true;
  1201. return NULL;
  1202. }
  1203. }
  1204. }
  1205. const void *ret = readTransformer->next();
  1206. assertex(ret != NULL); // if ret can ever be NULL then we need to recode this logic
  1207. processed++;
  1208. groupSignalled = false;
  1209. return ret;
  1210. }
  1211. protected:
  1212. bool waitForPipe()
  1213. {
  1214. if (!pipe)
  1215. return false; // done
  1216. if (!readTransformer->eos())
  1217. return true;
  1218. verifyPipe();
  1219. return false;
  1220. }
  1221. void openPipe(char const * cmd)
  1222. {
  1223. pipeCommand.setown(cmd);
  1224. pipe.setown(createPipeProcess(agent.queryAllowedPipePrograms()));
  1225. if(!pipe->run(NULL, cmd, ".", false, true, true, 0x10000))
  1226. throw MakeStringException(2, "Could not run pipe process %s", cmd);
  1227. Owned<ISimpleReadStream> pipeReader = pipe->getOutputStream();
  1228. readTransformer->setStream(pipeReader.get());
  1229. }
  1230. void verifyPipe()
  1231. {
  1232. if (pipe)
  1233. {
  1234. unsigned err = pipe->wait();
  1235. if(err && !(helper.getPipeFlags() & TPFnofail))
  1236. throwPipeProcessError(err, "from", pipeCommand.get(), pipe);
  1237. pipe.clear();
  1238. }
  1239. }
  1240. };
  1241. //=====================================================================================================
  1242. // Through pipe code - taken from Roxie implementation
  1243. interface IPipeRecordPullerCallback : extends IExceptionHandler
  1244. {
  1245. virtual void processRow(const void *row) = 0;
  1246. virtual void processDone() = 0;
  1247. virtual const void *nextInput() = 0;
  1248. };
  1249. class CPipeRecordPullerThread : public Thread
  1250. {
  1251. protected:
  1252. IPipeRecordPullerCallback *helper;
  1253. bool eog;
  1254. public:
  1255. CPipeRecordPullerThread() : Thread("PipeRecordPullerThread")
  1256. {
  1257. helper = NULL;
  1258. eog = false;
  1259. }
  1260. void setInput(IPipeRecordPullerCallback *_helper)
  1261. {
  1262. helper = _helper;
  1263. }
  1264. virtual int run()
  1265. {
  1266. try
  1267. {
  1268. for (;;)
  1269. {
  1270. const void * row = helper->nextInput();
  1271. if (row)
  1272. {
  1273. eog = false;
  1274. helper->processRow(row);
  1275. }
  1276. else if (!eog)
  1277. {
  1278. eog = true;
  1279. }
  1280. else
  1281. {
  1282. break;
  1283. }
  1284. }
  1285. helper->processDone();
  1286. }
  1287. catch (IException *e)
  1288. {
  1289. helper->fireException(e);
  1290. }
  1291. catch (...)
  1292. {
  1293. helper->fireException(MakeStringException(2, "Unexpected exception caught in PipeRecordPullerThread::run"));
  1294. }
  1295. return 0;
  1296. }
  1297. };
  1298. class CHThorPipeThroughActivity : public CHThorSimpleActivityBase, implements IPipeRecordPullerCallback
  1299. {
  1300. IHThorPipeThroughArg &helper;
  1301. CPipeRecordPullerThread puller;
  1302. Owned<IPipeProcess> pipe;
  1303. StringAttr pipeCommand;
  1304. InterruptableSemaphore pipeVerified;
  1305. InterruptableSemaphore pipeOpened;
  1306. CachedOutputMetaData inputMeta;
  1307. Owned<IOutputRowSerializer> rowSerializer;
  1308. Owned<IOutputRowDeserializer> rowDeserializer;
  1309. Owned<IPipeWriteXformHelper> writeTransformer;
  1310. Owned<IReadRowStream> readTransformer;
  1311. bool firstRead;
  1312. bool recreate;
  1313. bool inputExhausted;
  1314. bool groupSignalled;
  1315. public:
  1316. CHThorPipeThroughActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorPipeThroughArg &_arg, ThorActivityKind _kind)
  1317. : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  1318. {
  1319. recreate = helper.recreateEachRow();
  1320. groupSignalled = true;
  1321. firstRead = false;
  1322. inputExhausted = false;
  1323. puller.setInput(this);
  1324. }
  1325. virtual void ready()
  1326. {
  1327. CHThorSimpleActivityBase::ready();
  1328. // From the create() in roxie
  1329. inputMeta.set(input->queryOutputMeta());
  1330. rowSerializer.setown(inputMeta.createDiskSerializer(agent.queryCodeContext(), activityId));
  1331. rowDeserializer.setown(rowAllocator->createDiskDeserializer(agent.queryCodeContext()));
  1332. writeTransformer.setown(createPipeWriteXformHelper(helper.getPipeFlags(), helper.queryXmlOutput(), helper.queryCsvOutput(), rowSerializer));
  1333. // From the start() in roxie
  1334. firstRead = true;
  1335. inputExhausted = false;
  1336. groupSignalled = true; // i.e. don't start with a NULL row
  1337. pipeVerified.reinit();
  1338. pipeOpened.reinit();
  1339. writeTransformer->ready();
  1340. if (!readTransformer)
  1341. {
  1342. OwnedRoxieString xmlIterator(helper.getXmlIteratorPath());
  1343. readTransformer.setown(createReadRowStream(rowAllocator, rowDeserializer, helper.queryXmlTransformer(), helper.queryCsvTransformer(), xmlIterator, helper.getPipeFlags()));
  1344. }
  1345. if(!recreate)
  1346. {
  1347. OwnedRoxieString pipeProgram(helper.getPipeProgram());
  1348. openPipe(pipeProgram);
  1349. }
  1350. puller.start();
  1351. }
  1352. void stop()
  1353. {
  1354. //Need to close the output (or read it in its entirety), otherwise we might wait forever for the
  1355. //program to finish
  1356. if (pipe)
  1357. pipe->closeOutput();
  1358. pipeVerified.interrupt(NULL);
  1359. pipeOpened.interrupt(NULL);
  1360. puller.join();
  1361. CHThorSimpleActivityBase::stop();
  1362. pipe.clear();
  1363. readTransformer->setStream(NULL);
  1364. }
  1365. virtual bool needsAllocator() const { return true; }
  1366. virtual const void *nextRow()
  1367. {
  1368. while (!waitForPipe())
  1369. {
  1370. if (!pipe)
  1371. return NULL;
  1372. if (helper.getPipeFlags() & TPFgroupeachrow)
  1373. {
  1374. if (!groupSignalled)
  1375. {
  1376. groupSignalled = true;
  1377. return NULL;
  1378. }
  1379. }
  1380. }
  1381. const void *ret = readTransformer->next();
  1382. assertex(ret != NULL); // if ret can ever be NULL then we need to recode this logic
  1383. processed++;
  1384. groupSignalled = false;
  1385. return ret;
  1386. }
  1387. virtual bool isGrouped()
  1388. {
  1389. return outputMeta.isGrouped();
  1390. }
  1391. virtual void processRow(const void *row)
  1392. {
  1393. // called from puller thread
  1394. if(recreate)
  1395. openPipe(helper.getNameFromRow(row));
  1396. writeTransformer->writeTranslatedText(row, pipe);
  1397. ReleaseRoxieRow(row);
  1398. if(recreate)
  1399. {
  1400. closePipe();
  1401. pipeVerified.wait();
  1402. }
  1403. }
  1404. virtual void processDone()
  1405. {
  1406. // called from puller thread
  1407. if(recreate)
  1408. {
  1409. inputExhausted = true;
  1410. pipeOpened.signal();
  1411. }
  1412. else
  1413. {
  1414. closePipe();
  1415. pipeVerified.wait();
  1416. }
  1417. }
  1418. virtual const void *nextInput()
  1419. {
  1420. return input->nextRow();
  1421. }
  1422. virtual bool fireException(IException *e)
  1423. {
  1424. inputExhausted = true;
  1425. pipeOpened.interrupt(LINK(e));
  1426. pipeVerified.interrupt(e);
  1427. return true;
  1428. }
  1429. private:
  1430. bool waitForPipe()
  1431. {
  1432. if (firstRead)
  1433. {
  1434. pipeOpened.wait();
  1435. firstRead = false;
  1436. }
  1437. if (!pipe)
  1438. return false; // done
  1439. if (!readTransformer->eos())
  1440. return true;
  1441. verifyPipe();
  1442. if (recreate && !inputExhausted)
  1443. pipeOpened.wait();
  1444. return false;
  1445. }
  1446. void openPipe(char const * cmd)
  1447. {
  1448. pipeCommand.setown(cmd);
  1449. pipe.setown(createPipeProcess(agent.queryAllowedPipePrograms()));
  1450. if(!pipe->run(NULL, cmd, ".", true, true, true, 0x10000))
  1451. throw MakeStringException(2, "Could not run pipe process %s", cmd);
  1452. writeTransformer->writeHeader(pipe);
  1453. Owned<ISimpleReadStream> pipeReader = pipe->getOutputStream();
  1454. readTransformer->setStream(pipeReader.get());
  1455. pipeOpened.signal();
  1456. }
  1457. void closePipe()
  1458. {
  1459. writeTransformer->writeFooter(pipe);
  1460. pipe->closeInput();
  1461. }
  1462. void verifyPipe()
  1463. {
  1464. if (pipe)
  1465. {
  1466. unsigned err = pipe->wait();
  1467. if(err && !(helper.getPipeFlags() & TPFnofail))
  1468. throwPipeProcessError(err, "through", pipeCommand.get(), pipe);
  1469. pipe.clear();
  1470. pipeVerified.signal();
  1471. }
  1472. }
  1473. };
  1474. class CHThorPipeWriteActivity : public CHThorActivityBase
  1475. {
  1476. IHThorPipeWriteArg &helper;
  1477. Owned<IPipeProcess> pipe;
  1478. StringAttr pipeCommand;
  1479. CachedOutputMetaData inputMeta;
  1480. Owned<IOutputRowSerializer> rowSerializer;
  1481. Owned<IPipeWriteXformHelper> writeTransformer;
  1482. bool firstRead;
  1483. bool recreate;
  1484. bool inputExhausted;
  1485. public:
  1486. IMPLEMENT_SINKACTIVITY;
  1487. CHThorPipeWriteActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorPipeWriteArg &_arg, ThorActivityKind _kind)
  1488. : CHThorActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  1489. {
  1490. recreate = helper.recreateEachRow();
  1491. firstRead = false;
  1492. inputExhausted = false;
  1493. }
  1494. virtual bool needsAllocator() const { return true; }
  1495. virtual void ready()
  1496. {
  1497. CHThorActivityBase::ready();
  1498. inputMeta.set(input->queryOutputMeta());
  1499. rowSerializer.setown(inputMeta.createDiskSerializer(agent.queryCodeContext(), activityId));
  1500. writeTransformer.setown(createPipeWriteXformHelper(helper.getPipeFlags(), helper.queryXmlOutput(), helper.queryCsvOutput(), rowSerializer));
  1501. firstRead = true;
  1502. inputExhausted = false;
  1503. writeTransformer->ready();
  1504. if(!recreate)
  1505. {
  1506. OwnedRoxieString pipeProgram(helper.getPipeProgram());
  1507. openPipe(pipeProgram);
  1508. }
  1509. }
  1510. virtual void execute()
  1511. {
  1512. for (;;)
  1513. {
  1514. const void *row = input->nextRow();
  1515. if (!row)
  1516. {
  1517. row = input->nextRow();
  1518. if (!row)
  1519. break;
  1520. }
  1521. processed++;
  1522. if(recreate)
  1523. openPipe(helper.getNameFromRow(row));
  1524. writeTransformer->writeTranslatedText(row, pipe);
  1525. ReleaseRoxieRow(row);
  1526. if(recreate)
  1527. closePipe();
  1528. }
  1529. closePipe();
  1530. if (helper.getSequence() >= 0)
  1531. {
  1532. WorkunitUpdate wu = agent.updateWorkUnit();
  1533. Owned<IWUResult> result = wu->updateResultBySequence(helper.getSequence());
  1534. if (result)
  1535. {
  1536. result->setResultTotalRowCount(processed);
  1537. result->setResultStatus(ResultStatusCalculated);
  1538. }
  1539. }
  1540. }
  1541. private:
  1542. void openPipe(char const * cmd)
  1543. {
  1544. pipeCommand.setown(cmd);
  1545. pipe.setown(createPipeProcess(agent.queryAllowedPipePrograms()));
  1546. if(!pipe->run(NULL, cmd, ".", true, false, true, 0x10000))
  1547. throw MakeStringException(2, "Could not run pipe process %s", cmd);
  1548. writeTransformer->writeHeader(pipe);
  1549. }
  1550. void closePipe()
  1551. {
  1552. writeTransformer->writeFooter(pipe);
  1553. pipe->closeInput();
  1554. unsigned err = pipe->wait();
  1555. if(err && !(helper.getPipeFlags() & TPFnofail))
  1556. throwPipeProcessError(err, "to", pipeCommand.get(), pipe);
  1557. pipe.clear();
  1558. }
  1559. };
  1560. //=====================================================================================================
  1561. CHThorIterateActivity::CHThorIterateActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorIterateArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  1562. {
  1563. }
  1564. void CHThorIterateActivity::stop()
  1565. {
  1566. CHThorSimpleActivityBase::stop();
  1567. right.clear();
  1568. left.clear();
  1569. }
  1570. void CHThorIterateActivity::ready()
  1571. {
  1572. CHThorSimpleActivityBase::ready();
  1573. if (!defaultRecord)
  1574. {
  1575. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  1576. size32_t thisSize = helper.createDefault(rowBuilder);
  1577. defaultRecord.setown(rowBuilder.finalizeRowClear(thisSize));
  1578. }
  1579. counter = 0;
  1580. }
  1581. const void *CHThorIterateActivity::nextRow()
  1582. {
  1583. for (;;)
  1584. {
  1585. right.setown(input->nextRow());
  1586. if(!right)
  1587. {
  1588. bool skippedGroup = (!left) && (counter > 0); //we have just skipped entire group, but shouldn't output a double null
  1589. left.clear();
  1590. counter = 0;
  1591. if(skippedGroup) continue;
  1592. return NULL;
  1593. }
  1594. try
  1595. {
  1596. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  1597. unsigned outSize = helper.transform(rowBuilder, left ? left : defaultRecord, right, ++counter);
  1598. if (outSize)
  1599. {
  1600. left.setown(rowBuilder.finalizeRowClear(outSize));
  1601. processed++;
  1602. return left.getLink();
  1603. }
  1604. }
  1605. catch(IException * e)
  1606. {
  1607. throw makeWrappedException(e);
  1608. }
  1609. }
  1610. }
  1611. //=====================================================================================================
  1612. CHThorProcessActivity::CHThorProcessActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorProcessArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  1613. {
  1614. }
  1615. CHThorProcessActivity::~CHThorProcessActivity()
  1616. {
  1617. }
  1618. void CHThorProcessActivity::ready()
  1619. {
  1620. CHThorSimpleActivityBase::ready();
  1621. rightRowAllocator.setown(agent.queryCodeContext()->getRowAllocator( helper.queryRightRecordSize(), activityId));
  1622. RtlDynamicRowBuilder rowBuilder(rightRowAllocator);
  1623. size32_t thisSize = helper.createInitialRight(rowBuilder);
  1624. initialRight.setown(rowBuilder.finalizeRowClear(thisSize));
  1625. curRight.set(initialRight);
  1626. counter = 0;
  1627. }
  1628. const void *CHThorProcessActivity::nextRow()
  1629. {
  1630. try
  1631. {
  1632. for (;;)
  1633. {
  1634. OwnedConstRoxieRow next(input->nextRow());
  1635. if (!next)
  1636. {
  1637. bool eog = (curRight != initialRight); // processed any records?
  1638. counter = 0;
  1639. curRight.set(initialRight);
  1640. if (eog)
  1641. return NULL;
  1642. next.setown(input->nextRow());
  1643. if (!next)
  1644. return NULL;
  1645. }
  1646. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  1647. RtlDynamicRowBuilder rightRowBuilder(rightRowAllocator);
  1648. size32_t outSize = helper.transform(rowBuilder, rightRowBuilder, next, curRight, ++counter);
  1649. if (outSize)
  1650. {
  1651. size32_t rightSize = rightRowAllocator->queryOutputMeta()->getRecordSize(rightRowBuilder.getSelf()); // yuk
  1652. curRight.setown(rightRowBuilder.finalizeRowClear(rightSize));
  1653. processed++;
  1654. return rowBuilder.finalizeRowClear(outSize);
  1655. }
  1656. }
  1657. }
  1658. catch(IException * e)
  1659. {
  1660. throw makeWrappedException(e);
  1661. }
  1662. }
  1663. //=====================================================================================================
  1664. CHThorNormalizeActivity::CHThorNormalizeActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorNormalizeArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  1665. {
  1666. IRecordSize* recSize = outputMeta;
  1667. if (recSize == NULL)
  1668. throw MakeStringException(2, "Unexpected null pointer from helper.queryOutputMeta()");
  1669. }
  1670. CHThorNormalizeActivity::~CHThorNormalizeActivity()
  1671. {
  1672. }
  1673. void CHThorNormalizeActivity::ready()
  1674. {
  1675. CHThorSimpleActivityBase::ready();
  1676. numThisRow = 0;
  1677. curRow = 0;
  1678. numProcessedLastGroup = processed;
  1679. }
  1680. const void *CHThorNormalizeActivity::nextRow()
  1681. {
  1682. for (;;)
  1683. {
  1684. while (curRow == numThisRow)
  1685. {
  1686. inbuff.setown(input->nextRow());
  1687. if (!inbuff && (processed == numProcessedLastGroup))
  1688. inbuff.setown(input->nextRow());
  1689. if (!inbuff)
  1690. {
  1691. numProcessedLastGroup = processed;
  1692. return NULL;
  1693. }
  1694. curRow = 0;
  1695. numThisRow = helper.numExpandedRows(inbuff);
  1696. }
  1697. try
  1698. {
  1699. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  1700. memsize_t thisSize = helper.transform(rowBuilder, inbuff, ++curRow);
  1701. if(thisSize != 0)
  1702. {
  1703. processed++;
  1704. return rowBuilder.finalizeRowClear(thisSize);
  1705. }
  1706. }
  1707. catch(IException * e)
  1708. {
  1709. throw makeWrappedException(e);
  1710. }
  1711. }
  1712. }
  1713. //=====================================================================================================
  1714. CHThorNormalizeChildActivity::CHThorNormalizeChildActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorNormalizeChildArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  1715. {
  1716. }
  1717. CHThorNormalizeChildActivity::~CHThorNormalizeChildActivity()
  1718. {
  1719. }
  1720. bool CHThorNormalizeChildActivity::advanceInput()
  1721. {
  1722. for (;;)
  1723. {
  1724. inbuff.setown(input->nextRow());
  1725. if (!inbuff && (processed == numProcessedLastGroup))
  1726. inbuff.setown(input->nextRow());
  1727. if (!inbuff)
  1728. {
  1729. numProcessedLastGroup = processed;
  1730. return false;
  1731. }
  1732. curChildRow = cursor->first(inbuff);
  1733. if (curChildRow)
  1734. {
  1735. curRow = 0;
  1736. return true;
  1737. }
  1738. }
  1739. }
  1740. void CHThorNormalizeChildActivity::stop()
  1741. {
  1742. inbuff.clear();
  1743. CHThorSimpleActivityBase::stop();
  1744. }
  1745. void CHThorNormalizeChildActivity::ready()
  1746. {
  1747. CHThorSimpleActivityBase::ready();
  1748. curRow = 0;
  1749. numProcessedLastGroup = processed;
  1750. cursor = helper.queryIterator();
  1751. curChildRow = NULL;
  1752. }
  1753. const void *CHThorNormalizeChildActivity::nextRow()
  1754. {
  1755. for (;;)
  1756. {
  1757. if (!inbuff)
  1758. {
  1759. if (!advanceInput())
  1760. return NULL;
  1761. }
  1762. try
  1763. {
  1764. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  1765. size32_t outSize = helper.transform(rowBuilder, inbuff, curChildRow, ++curRow);
  1766. curChildRow = cursor->next();
  1767. if (!curChildRow)
  1768. inbuff.clear();
  1769. if (outSize != 0)
  1770. {
  1771. processed++;
  1772. return rowBuilder.finalizeRowClear(outSize);
  1773. }
  1774. }
  1775. catch(IException * e)
  1776. {
  1777. throw makeWrappedException(e);
  1778. }
  1779. }
  1780. }
  1781. //=================================================================================
  1782. bool CHThorNormalizeLinkedChildActivity::advanceInput()
  1783. {
  1784. for (;;)
  1785. {
  1786. curParent.setown(input->nextRow());
  1787. if (!curParent && (processed == numProcessedLastGroup))
  1788. curParent.setown(input->nextRow());
  1789. if (!curParent)
  1790. {
  1791. numProcessedLastGroup = processed;
  1792. return false;
  1793. }
  1794. curChild.set(helper.first(curParent));
  1795. if (curChild)
  1796. return true;
  1797. }
  1798. }
  1799. CHThorNormalizeLinkedChildActivity::CHThorNormalizeLinkedChildActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorNormalizeLinkedChildArg &_arg, ThorActivityKind _kind)
  1800. : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  1801. {
  1802. }
  1803. CHThorNormalizeLinkedChildActivity::~CHThorNormalizeLinkedChildActivity()
  1804. {
  1805. }
  1806. void CHThorNormalizeLinkedChildActivity::ready()
  1807. {
  1808. numProcessedLastGroup = 0;
  1809. CHThorSimpleActivityBase::ready();
  1810. }
  1811. void CHThorNormalizeLinkedChildActivity::stop()
  1812. {
  1813. curParent.clear();
  1814. curChild.clear();
  1815. CHThorSimpleActivityBase::stop();
  1816. }
  1817. const void * CHThorNormalizeLinkedChildActivity::nextRow()
  1818. {
  1819. for (;;)
  1820. {
  1821. if (!curParent)
  1822. {
  1823. if (!advanceInput())
  1824. return NULL;
  1825. }
  1826. try
  1827. {
  1828. const void *ret = curChild.getClear();
  1829. curChild.set(helper.next());
  1830. if (!curChild)
  1831. curParent.clear();
  1832. if (ret)
  1833. {
  1834. processed++;
  1835. return ret;
  1836. }
  1837. }
  1838. catch (IException *E)
  1839. {
  1840. throw makeWrappedException(E);
  1841. }
  1842. }
  1843. }
  1844. //=====================================================================================================
  1845. CHThorProjectActivity::CHThorProjectActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorProjectArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  1846. {
  1847. }
  1848. CHThorProjectActivity::~CHThorProjectActivity()
  1849. {
  1850. }
  1851. void CHThorProjectActivity::ready()
  1852. {
  1853. CHThorSimpleActivityBase::ready();
  1854. numProcessedLastGroup = processed;
  1855. }
  1856. const void * CHThorProjectActivity::nextRow()
  1857. {
  1858. for (;;)
  1859. {
  1860. OwnedConstRoxieRow in(input->nextRow());
  1861. if (!in)
  1862. {
  1863. if (numProcessedLastGroup == processed)
  1864. in.setown(input->nextRow());
  1865. if (!in)
  1866. {
  1867. numProcessedLastGroup = processed;
  1868. return NULL;
  1869. }
  1870. }
  1871. try
  1872. {
  1873. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  1874. size32_t outSize = helper.transform(rowBuilder, in);
  1875. if (outSize)
  1876. {
  1877. processed++;
  1878. return rowBuilder.finalizeRowClear(outSize);
  1879. }
  1880. }
  1881. catch(IException * e)
  1882. {
  1883. throw makeWrappedException(e);
  1884. }
  1885. }
  1886. }
  1887. //=====================================================================================================
  1888. CHThorPrefetchProjectActivity::CHThorPrefetchProjectActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorPrefetchProjectArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  1889. {
  1890. }
  1891. void CHThorPrefetchProjectActivity::ready()
  1892. {
  1893. CHThorSimpleActivityBase::ready();
  1894. recordCount = 0;
  1895. numProcessedLastGroup = processed;
  1896. eof = !helper.canMatchAny();
  1897. child = helper.queryChild();
  1898. }
  1899. const void * CHThorPrefetchProjectActivity::nextRow()
  1900. {
  1901. if (eof)
  1902. return NULL;
  1903. for (;;)
  1904. {
  1905. try
  1906. {
  1907. OwnedConstRoxieRow row(input->nextRow());
  1908. if (!row)
  1909. {
  1910. if (numProcessedLastGroup == processed)
  1911. row.setown(input->nextRow());
  1912. if (!row)
  1913. {
  1914. numProcessedLastGroup = processed;
  1915. return NULL;
  1916. }
  1917. }
  1918. ++recordCount;
  1919. rtlRowBuilder extract;
  1920. if (helper.preTransform(extract,row,recordCount))
  1921. {
  1922. Owned<IEclGraphResults> results;
  1923. if (child)
  1924. {
  1925. results.setown(child->evaluate(extract.size(), extract.getbytes()));
  1926. }
  1927. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  1928. size32_t outSize = helper.transform(rowBuilder, row, results, recordCount);
  1929. if (outSize)
  1930. {
  1931. processed++;
  1932. return rowBuilder.finalizeRowClear(outSize);
  1933. }
  1934. }
  1935. }
  1936. catch(IException * e)
  1937. {
  1938. throw makeWrappedException(e);
  1939. }
  1940. }
  1941. }
  1942. //=====================================================================================================
  1943. CHThorFilterProjectActivity::CHThorFilterProjectActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorFilterProjectArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  1944. {
  1945. }
  1946. CHThorFilterProjectActivity::~CHThorFilterProjectActivity()
  1947. {
  1948. }
  1949. void CHThorFilterProjectActivity::ready()
  1950. {
  1951. CHThorSimpleActivityBase::ready();
  1952. recordCount = 0;
  1953. numProcessedLastGroup = processed;
  1954. eof = !helper.canMatchAny();
  1955. }
  1956. const void * CHThorFilterProjectActivity::nextRow()
  1957. {
  1958. if (eof)
  1959. return NULL;
  1960. for (;;)
  1961. {
  1962. OwnedConstRoxieRow in = input->nextRow();
  1963. if (!in)
  1964. {
  1965. recordCount = 0;
  1966. if (numProcessedLastGroup == processed)
  1967. in.setown(input->nextRow());
  1968. if (!in)
  1969. {
  1970. numProcessedLastGroup = processed;
  1971. return NULL;
  1972. }
  1973. }
  1974. try
  1975. {
  1976. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  1977. size32_t outSize = helper.transform(rowBuilder, in, ++recordCount);
  1978. if (outSize)
  1979. {
  1980. processed++;
  1981. return rowBuilder.finalizeRowClear(outSize);
  1982. }
  1983. }
  1984. catch(IException * e)
  1985. {
  1986. throw makeWrappedException(e);
  1987. }
  1988. }
  1989. }
  1990. //=====================================================================================================
  1991. CHThorCountProjectActivity::CHThorCountProjectActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorCountProjectArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  1992. {
  1993. }
  1994. CHThorCountProjectActivity::~CHThorCountProjectActivity()
  1995. {
  1996. }
  1997. void CHThorCountProjectActivity::ready()
  1998. {
  1999. CHThorSimpleActivityBase::ready();
  2000. recordCount = 0;
  2001. numProcessedLastGroup = processed;
  2002. }
  2003. const void * CHThorCountProjectActivity::nextRow()
  2004. {
  2005. for (;;)
  2006. {
  2007. OwnedConstRoxieRow in = input->nextRow();
  2008. if (!in)
  2009. {
  2010. recordCount = 0;
  2011. if (numProcessedLastGroup == processed)
  2012. in.setown(input->nextRow());
  2013. if (!in)
  2014. {
  2015. numProcessedLastGroup = processed;
  2016. return NULL;
  2017. }
  2018. }
  2019. try
  2020. {
  2021. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  2022. size32_t outSize = helper.transform(rowBuilder, in, ++recordCount);
  2023. if (outSize)
  2024. {
  2025. processed++;
  2026. return rowBuilder.finalizeRowClear(outSize);
  2027. }
  2028. }
  2029. catch(IException * e)
  2030. {
  2031. throw makeWrappedException(e);
  2032. }
  2033. }
  2034. }
  2035. //=====================================================================================================
  2036. CHThorRollupActivity::CHThorRollupActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorRollupArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  2037. {
  2038. }
  2039. CHThorRollupActivity::~CHThorRollupActivity()
  2040. {
  2041. }
  2042. void CHThorRollupActivity::ready()
  2043. {
  2044. CHThorSimpleActivityBase::ready();
  2045. left.setown(input->nextRow());
  2046. prev.set(left);
  2047. }
  2048. void CHThorRollupActivity::stop()
  2049. {
  2050. left.clear();
  2051. prev.clear();
  2052. right.clear();
  2053. CHThorSimpleActivityBase::stop();
  2054. }
  2055. const void *CHThorRollupActivity::nextRow()
  2056. {
  2057. for (;;)
  2058. {
  2059. right.setown(input->nextRow());
  2060. if(!prev || !right || !helper.matches(prev,right))
  2061. {
  2062. const void * ret = left.getClear();
  2063. if(ret)
  2064. {
  2065. processed++;
  2066. }
  2067. left.setown(right.getClear());
  2068. prev.set(left);
  2069. return ret;
  2070. }
  2071. try
  2072. {
  2073. //MORE: could optimise by reusing buffer, but would have to make sure to call destructor on previous contents before overwriting
  2074. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  2075. if(unsigned outSize = helper.transform(rowBuilder, left, right))
  2076. {
  2077. left.setown(rowBuilder.finalizeRowClear(outSize));
  2078. }
  2079. if (helper.getFlags() & RFrolledismatchleft)
  2080. prev.set(left);
  2081. else
  2082. prev.set(right);
  2083. }
  2084. catch(IException * e)
  2085. {
  2086. throw makeWrappedException(e);
  2087. }
  2088. }
  2089. }
  2090. //=====================================================================================================
  2091. CHThorGroupDedupActivity::CHThorGroupDedupActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorDedupArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  2092. {
  2093. }
  2094. void CHThorGroupDedupActivity::ready()
  2095. {
  2096. CHThorSimpleActivityBase::ready();
  2097. numToKeep = helper.numToKeep();
  2098. numKept = 0;
  2099. }
  2100. //=====================================================================================================
  2101. CHThorGroupDedupKeepLeftActivity::CHThorGroupDedupKeepLeftActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorDedupArg &_arg, ThorActivityKind _kind) : CHThorGroupDedupActivity(_agent, _activityId, _subgraphId, _arg, _kind)
  2102. {
  2103. }
  2104. void CHThorGroupDedupKeepLeftActivity::ready()
  2105. {
  2106. CHThorGroupDedupActivity::ready();
  2107. prev.clear();
  2108. }
  2109. void CHThorGroupDedupKeepLeftActivity::stop()
  2110. {
  2111. prev.clear();
  2112. CHThorSimpleActivityBase::stop();
  2113. }
  2114. const void *CHThorGroupDedupKeepLeftActivity::nextRow()
  2115. {
  2116. OwnedConstRoxieRow next;
  2117. for (;;)
  2118. {
  2119. next.setown(input->nextRow());
  2120. if (!prev || !next || !helper.matches(prev,next))
  2121. {
  2122. numKept = 0;
  2123. break;
  2124. }
  2125. if (numKept < numToKeep-1)
  2126. {
  2127. numKept++;
  2128. break;
  2129. }
  2130. }
  2131. const void * ret = next.getClear();
  2132. prev.set(ret);
  2133. if(ret)
  2134. processed++;
  2135. return ret;
  2136. }
  2137. const void * CHThorGroupDedupKeepLeftActivity::nextRowGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
  2138. {
  2139. OwnedConstRoxieRow next;
  2140. for (;;)
  2141. {
  2142. next.setown(input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra));
  2143. if (!prev || !next || !helper.matches(prev,next))
  2144. {
  2145. numKept = 0;
  2146. break;
  2147. }
  2148. if (numKept < numToKeep-1)
  2149. {
  2150. numKept++;
  2151. break;
  2152. }
  2153. }
  2154. const void * ret = next.getClear();
  2155. prev.set(ret);
  2156. if(ret)
  2157. processed++;
  2158. return ret;
  2159. }
  2160. void CHThorGroupDedupKeepLeftActivity::setInput(unsigned index, IHThorInput *_input)
  2161. {
  2162. CHThorGroupDedupActivity::setInput(index, _input);
  2163. if (input)
  2164. inputStepping = input->querySteppingMeta();
  2165. }
  2166. IInputSteppingMeta * CHThorGroupDedupKeepLeftActivity::querySteppingMeta()
  2167. {
  2168. return inputStepping;
  2169. }
  2170. bool CHThorGroupDedupKeepLeftActivity::gatherConjunctions(ISteppedConjunctionCollector & collector)
  2171. {
  2172. return input->gatherConjunctions(collector);
  2173. }
  2174. void CHThorGroupDedupKeepLeftActivity::resetEOF()
  2175. {
  2176. input->resetEOF();
  2177. }
  2178. //=====================================================================================================
  2179. CHThorGroupDedupKeepRightActivity::CHThorGroupDedupKeepRightActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorDedupArg &_arg, ThorActivityKind _kind) : CHThorGroupDedupActivity(_agent, _activityId, _subgraphId, _arg, _kind), compareBest(nullptr)
  2180. {
  2181. }
  2182. void CHThorGroupDedupKeepRightActivity::ready()
  2183. {
  2184. CHThorGroupDedupActivity::ready();
  2185. assertex(numToKeep==1);
  2186. firstDone = false;
  2187. if (helper.keepBest())
  2188. compareBest = helper.queryCompareBest();
  2189. }
  2190. void CHThorGroupDedupKeepRightActivity::stop()
  2191. {
  2192. kept.clear();
  2193. CHThorGroupDedupActivity::stop();
  2194. }
  2195. const void *CHThorGroupDedupKeepRightActivity::nextRow()
  2196. {
  2197. if (!firstDone)
  2198. {
  2199. firstDone = true;
  2200. kept.setown(input->nextRow());
  2201. }
  2202. OwnedConstRoxieRow next;
  2203. for (;;)
  2204. {
  2205. next.setown(input->nextRow());
  2206. if (!kept || !next || !helper.matches(kept,next))
  2207. {
  2208. numKept = 0;
  2209. break;
  2210. }
  2211. if (compareBest)
  2212. {
  2213. if (compareBest->docompare(kept,next) > 0)
  2214. kept.setown(next.getClear());
  2215. }
  2216. else
  2217. {
  2218. if (numKept < numToKeep-1)
  2219. {
  2220. numKept++;
  2221. break;
  2222. }
  2223. kept.setown(next.getClear());
  2224. }
  2225. }
  2226. const void * ret = kept.getClear();
  2227. kept.setown(next.getClear());
  2228. if(ret)
  2229. processed++;
  2230. return ret;
  2231. }
  2232. //=====================================================================================================
  2233. CHThorGroupDedupAllActivity::CHThorGroupDedupAllActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorDedupArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  2234. {
  2235. }
  2236. void CHThorGroupDedupAllActivity::ready()
  2237. {
  2238. CHThorSimpleActivityBase::ready();
  2239. keepLeft = helper.keepLeft();
  2240. primaryCompare = helper.queryComparePrimary();
  2241. assertex(helper.numToKeep() == 1);
  2242. firstDone = false;
  2243. survivorIndex = 0;
  2244. }
  2245. void CHThorGroupDedupAllActivity::stop()
  2246. {
  2247. survivors.clear();
  2248. CHThorSimpleActivityBase::stop();
  2249. }
  2250. bool CHThorGroupDedupAllActivity::calcNextDedupAll()
  2251. {
  2252. survivors.clear();
  2253. survivorIndex = 0;
  2254. OwnedRowArray group;
  2255. const void * next;
  2256. while((next = input->nextRow()) != NULL)
  2257. group.append(next);
  2258. if(group.ordinality() == 0)
  2259. return false;
  2260. unsigned max = group.ordinality();
  2261. if (primaryCompare)
  2262. {
  2263. //hard, if not impossible, to hit this code once optimisations in place
  2264. MemoryAttr indexbuff(max*sizeof(void *));
  2265. void ** temp = (void **)indexbuff.bufferBase();
  2266. void ** rows = (void * *)group.getArray();
  2267. msortvecstableinplace(rows, max, *primaryCompare, temp);
  2268. unsigned first = 0;
  2269. for (unsigned idx = 1; idx < max; idx++)
  2270. {
  2271. if (primaryCompare->docompare(rows[first], rows[idx]) != 0)
  2272. {
  2273. dedupRange(first, idx, group);
  2274. first = idx;
  2275. }
  2276. }
  2277. dedupRange(first, max, group);
  2278. for(unsigned idx2=0; idx2<max; ++idx2)
  2279. {
  2280. void * cur = rows[idx2];
  2281. if(cur)
  2282. {
  2283. LinkRoxieRow(cur);
  2284. survivors.append(cur);
  2285. }
  2286. }
  2287. }
  2288. else
  2289. {
  2290. dedupRange(0, max, group);
  2291. for(unsigned idx=0; idx<max; ++idx)
  2292. {
  2293. const void * cur = group.itemClear(idx);
  2294. if(cur)
  2295. survivors.append(cur);
  2296. }
  2297. }
  2298. return true;
  2299. }
  2300. void CHThorGroupDedupAllActivity::dedupRange(unsigned first, unsigned last, OwnedRowArray & group)
  2301. {
  2302. for (unsigned idxL = first; idxL < last; idxL++)
  2303. {
  2304. const void * left = group.item(idxL);
  2305. if (left)
  2306. {
  2307. for (unsigned idxR = first; idxR < last; idxR++)
  2308. {
  2309. const void * right = group.item(idxR);
  2310. if ((idxL != idxR) && right)
  2311. {
  2312. if (helper.matches(left, right))
  2313. {
  2314. if (keepLeft)
  2315. {
  2316. group.replace(NULL, idxR);
  2317. }
  2318. else
  2319. {
  2320. group.replace(NULL, idxL);
  2321. break;
  2322. }
  2323. }
  2324. }
  2325. }
  2326. }
  2327. }
  2328. }
  2329. const void *CHThorGroupDedupAllActivity::nextRow()
  2330. {
  2331. if (!firstDone)
  2332. {
  2333. firstDone = true;
  2334. calcNextDedupAll();
  2335. }
  2336. if(survivors.isItem(survivorIndex))
  2337. {
  2338. processed++;
  2339. return survivors.itemClear(survivorIndex++);
  2340. }
  2341. calcNextDedupAll();
  2342. return NULL;
  2343. }
  2344. //=====================================================================================================
  2345. bool HashDedupTable::insert(const void * row)
  2346. {
  2347. unsigned hash = helper.queryHash()->hash(row);
  2348. RtlDynamicRowBuilder keyRowBuilder(keyRowAllocator, true);
  2349. size32_t thisKeySize = helper.recordToKey(keyRowBuilder, row);
  2350. OwnedConstRoxieRow keyRow = keyRowBuilder.finalizeRowClear(thisKeySize);
  2351. if (find(hash, keyRow.get()))
  2352. return false;
  2353. addNew(new HashDedupElement(hash, keyRow.getClear()), hash);
  2354. return true;
  2355. }
  2356. bool HashDedupTable::insertBest(const void * nextrow)
  2357. {
  2358. unsigned hash = helper.queryHash()->hash(nextrow);
  2359. const void *et = find(hash, nextrow);
  2360. if (et)
  2361. {
  2362. const HashDedupElement *element = reinterpret_cast<const HashDedupElement *>(et);
  2363. const void * row = element->queryRow();
  2364. if (queryBestCompare->docompare(row,nextrow) <= 0)
  2365. return false;
  2366. removeExact( const_cast<void *>(et));
  2367. // drop-through to add new row
  2368. }
  2369. LinkRoxieRow(nextrow);
  2370. addNew(new HashDedupElement(hash, nextrow), hash);
  2371. return true;
  2372. }
  2373. CHThorHashDedupActivity::CHThorHashDedupActivity(IAgentContext & _agent, unsigned _activityId, unsigned _subgraphId, IHThorHashDedupArg & _arg, ThorActivityKind _kind)
  2374. : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg), table(_arg, activityId), hashTableFilled(false), hashDedupTableIter(table)
  2375. {
  2376. keepBest = helper.keepBest();
  2377. }
  2378. void CHThorHashDedupActivity::ready()
  2379. {
  2380. CHThorSimpleActivityBase::ready();
  2381. table.setRowAllocator(agent.queryCodeContext()->getRowAllocator(helper.queryKeySize(), activityId));
  2382. }
  2383. void CHThorHashDedupActivity::stop()
  2384. {
  2385. table.kill();
  2386. CHThorSimpleActivityBase::stop();
  2387. }
  2388. const void * CHThorHashDedupActivity::nextRow()
  2389. {
  2390. if (keepBest)
  2391. {
  2392. // Populate hash table with best rows
  2393. if (!hashTableFilled)
  2394. {
  2395. OwnedConstRoxieRow next(input->nextRow());
  2396. while(next)
  2397. {
  2398. table.insertBest(next);
  2399. next.setown(input->nextRow());
  2400. }
  2401. hashTableFilled = true;
  2402. hashDedupTableIter.first();
  2403. }
  2404. // Iterate through hash table returning rows
  2405. if (hashDedupTableIter.isValid())
  2406. {
  2407. HashDedupElement &el = hashDedupTableIter.query();
  2408. OwnedConstRoxieRow row(el.getRow());
  2409. hashDedupTableIter.next();
  2410. return row.getClear();
  2411. }
  2412. table.kill();
  2413. hashTableFilled = false;
  2414. return NULL;
  2415. }
  2416. else
  2417. {
  2418. while(true)
  2419. {
  2420. OwnedConstRoxieRow next(input->nextRow());
  2421. if(!next)
  2422. {
  2423. table.kill();
  2424. return NULL;
  2425. }
  2426. if(table.insert(next))
  2427. return next.getClear();
  2428. }
  2429. }
  2430. }
  2431. //=====================================================================================================
  2432. CHThorSteppableActivityBase::CHThorSteppableActivityBase(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorArg & _help, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _help, _kind)
  2433. {
  2434. inputStepping = NULL;
  2435. stepCompare = NULL;
  2436. }
  2437. void CHThorSteppableActivityBase::setInput(unsigned index, IHThorInput *_input)
  2438. {
  2439. CHThorSimpleActivityBase::setInput(index, _input);
  2440. if (input && index == 0)
  2441. {
  2442. inputStepping = input->querySteppingMeta();
  2443. if (inputStepping)
  2444. stepCompare = inputStepping->queryCompare();
  2445. }
  2446. }
  2447. IInputSteppingMeta * CHThorSteppableActivityBase::querySteppingMeta()
  2448. {
  2449. return inputStepping;
  2450. }
  2451. //=====================================================================================================
  2452. CHThorFilterActivity::CHThorFilterActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorFilterArg &_arg, ThorActivityKind _kind) : CHThorSteppableActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  2453. {
  2454. }
  2455. void CHThorFilterActivity::ready()
  2456. {
  2457. CHThorSimpleActivityBase::ready();
  2458. anyThisGroup = false;
  2459. eof = !helper.canMatchAny();
  2460. }
  2461. const void * CHThorFilterActivity::nextRow()
  2462. {
  2463. if (eof)
  2464. return NULL;
  2465. for (;;)
  2466. {
  2467. OwnedConstRoxieRow ret(input->nextRow());
  2468. if (!ret)
  2469. {
  2470. //stop returning two NULLs in a row.
  2471. if (anyThisGroup)
  2472. {
  2473. anyThisGroup = false;
  2474. return NULL;
  2475. }
  2476. ret.setown(input->nextRow());
  2477. if (!ret)
  2478. return NULL; // eof...
  2479. }
  2480. if (helper.isValid(ret))
  2481. {
  2482. anyThisGroup = true;
  2483. processed++;
  2484. return ret.getClear();
  2485. }
  2486. }
  2487. }
  2488. const void * CHThorFilterActivity::nextRowGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
  2489. {
  2490. if (eof)
  2491. return NULL;
  2492. OwnedConstRoxieRow ret(input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra));
  2493. if (!ret)
  2494. return NULL;
  2495. if (helper.isValid(ret))
  2496. {
  2497. anyThisGroup = true;
  2498. processed++;
  2499. return ret.getClear();
  2500. }
  2501. return ungroupedNextRow();
  2502. }
  2503. bool CHThorFilterActivity::gatherConjunctions(ISteppedConjunctionCollector & collector)
  2504. {
  2505. return input->gatherConjunctions(collector);
  2506. }
  2507. void CHThorFilterActivity::resetEOF()
  2508. {
  2509. //Sometimes the smart stepping code returns a premature eof indicator (two nulls) and will
  2510. //therefore call resetEOF so the activity can reset its eof without resetting the activity itself.
  2511. //Note that resetEOF only needs to be implemented by activities that implement gatherConjunctions()
  2512. //and that cache eof.
  2513. eof = false;
  2514. anyThisGroup = false;
  2515. input->resetEOF();
  2516. }
  2517. //=====================================================================================================
  2518. CHThorFilterGroupActivity::CHThorFilterGroupActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorFilterGroupArg &_arg, ThorActivityKind _kind) : CHThorSteppableActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  2519. {
  2520. }
  2521. void CHThorFilterGroupActivity::ready()
  2522. {
  2523. CHThorSimpleActivityBase::ready();
  2524. eof = !helper.canMatchAny();
  2525. nextIndex = 0;
  2526. }
  2527. void CHThorFilterGroupActivity::stop()
  2528. {
  2529. CHThorSimpleActivityBase::stop();
  2530. pending.clear();
  2531. }
  2532. const void * CHThorFilterGroupActivity::nextRow()
  2533. {
  2534. for (;;)
  2535. {
  2536. if (eof)
  2537. return NULL;
  2538. if (pending.ordinality())
  2539. {
  2540. if (pending.isItem(nextIndex))
  2541. {
  2542. processed++;
  2543. return pending.itemClear(nextIndex++);
  2544. }
  2545. nextIndex = 0;
  2546. pending.clear();
  2547. return NULL;
  2548. }
  2549. const void * ret = input->nextRow();
  2550. while (ret)
  2551. {
  2552. pending.append(ret);
  2553. ret = input->nextRow();
  2554. }
  2555. unsigned num = pending.ordinality();
  2556. if (num != 0)
  2557. {
  2558. if (!helper.isValid(num, (const void * *)pending.getArray()))
  2559. pending.clear(); // read next group
  2560. }
  2561. else
  2562. eof = true;
  2563. }
  2564. }
  2565. const void * CHThorFilterGroupActivity::nextRowGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
  2566. {
  2567. if (eof)
  2568. return NULL;
  2569. if (pending.ordinality())
  2570. {
  2571. while (pending.isItem(nextIndex))
  2572. {
  2573. OwnedConstRoxieRow ret(pending.itemClear(nextIndex++));
  2574. if (stepCompare->docompare(ret, seek, numFields) >= 0)
  2575. {
  2576. processed++;
  2577. return ret.getClear();
  2578. }
  2579. }
  2580. nextIndex = 0;
  2581. pending.clear();
  2582. }
  2583. const void * ret = input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
  2584. while (ret)
  2585. {
  2586. pending.append(ret);
  2587. ret = input->nextRow();
  2588. }
  2589. unsigned num = pending.ordinality();
  2590. if (num != 0)
  2591. {
  2592. if (!helper.isValid(num, (const void * *)pending.getArray()))
  2593. pending.clear(); // read next group
  2594. }
  2595. else
  2596. eof = true;
  2597. return ungroupedNextRow();
  2598. }
  2599. //=====================================================================================================
  2600. CHThorLimitActivity::CHThorLimitActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorLimitArg &_arg, ThorActivityKind _kind) : CHThorSteppableActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  2601. {
  2602. }
  2603. void CHThorLimitActivity::ready()
  2604. {
  2605. CHThorSimpleActivityBase::ready();
  2606. rowLimit = helper.getRowLimit();
  2607. numGot = 0;
  2608. }
  2609. const void * CHThorLimitActivity::nextRow()
  2610. {
  2611. OwnedConstRoxieRow ret(input->nextRow());
  2612. if (ret)
  2613. {
  2614. if (++numGot > rowLimit)
  2615. {
  2616. if ( agent.queryCodeContext()->queryDebugContext())
  2617. agent.queryCodeContext()->queryDebugContext()->checkBreakpoint(DebugStateLimit, NULL, static_cast<IActivityBase *>(this));
  2618. helper.onLimitExceeded();
  2619. return NULL;
  2620. }
  2621. processed++;
  2622. }
  2623. return ret.getClear();
  2624. }
  2625. const void * CHThorLimitActivity::nextRowGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
  2626. {
  2627. OwnedConstRoxieRow ret(input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra));
  2628. if (ret)
  2629. {
  2630. if (++numGot > rowLimit)
  2631. {
  2632. if ( agent.queryCodeContext()->queryDebugContext())
  2633. agent.queryCodeContext()->queryDebugContext()->checkBreakpoint(DebugStateLimit, NULL, static_cast<IActivityBase *>(this));
  2634. helper.onLimitExceeded();
  2635. return NULL;
  2636. }
  2637. processed++;
  2638. }
  2639. return ret.getClear();
  2640. }
  2641. //=====================================================================================================
  2642. CHThorSkipLimitActivity::CHThorSkipLimitActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorLimitArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  2643. {
  2644. }
  2645. void CHThorSkipLimitActivity::ready()
  2646. {
  2647. CHThorSimpleActivityBase::ready();
  2648. rowLimit = helper.getRowLimit();
  2649. }
  2650. void CHThorSkipLimitActivity::stop()
  2651. {
  2652. CHThorSimpleActivityBase::stop();
  2653. buffer.clear();
  2654. }
  2655. const void * CHThorSkipLimitActivity::nextRow()
  2656. {
  2657. if(!buffer)
  2658. {
  2659. buffer.setown(new CRowBuffer(input->queryOutputMeta(), true));
  2660. if(!buffer->pull(input, rowLimit))
  2661. {
  2662. if ( agent.queryCodeContext()->queryDebugContext())
  2663. agent.queryCodeContext()->queryDebugContext()->checkBreakpoint(DebugStateLimit, NULL, static_cast<IActivityBase *>(this));
  2664. onLimitExceeded();
  2665. }
  2666. }
  2667. const void * next = buffer->next();
  2668. if(next)
  2669. processed++;
  2670. return next;
  2671. }
  2672. //=====================================================================================================
  2673. CHThorCatchActivity::CHThorCatchActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorCatchArg &_arg, ThorActivityKind _kind) : CHThorSteppableActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  2674. {
  2675. }
  2676. const void * CHThorCatchActivity::nextRow()
  2677. {
  2678. try
  2679. {
  2680. OwnedConstRoxieRow ret(input->nextRow());
  2681. if (ret)
  2682. processed++;
  2683. return ret.getClear();
  2684. }
  2685. catch (IException *E)
  2686. {
  2687. E->Release();
  2688. helper.onExceptionCaught();
  2689. }
  2690. catch (...)
  2691. {
  2692. helper.onExceptionCaught();
  2693. }
  2694. throwUnexpected(); // onExceptionCaught should have thrown something
  2695. }
  2696. const void * CHThorCatchActivity::nextRowGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
  2697. {
  2698. try
  2699. {
  2700. OwnedConstRoxieRow ret(input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra));
  2701. if (ret)
  2702. processed++;
  2703. return ret.getClear();
  2704. }
  2705. catch (IException *E)
  2706. {
  2707. E->Release();
  2708. helper.onExceptionCaught();
  2709. }
  2710. catch (...)
  2711. {
  2712. helper.onExceptionCaught();
  2713. }
  2714. throwUnexpected(); // onExceptionCaught should have thrown something
  2715. }
  2716. //=====================================================================================================
  2717. CHThorSkipCatchActivity::CHThorSkipCatchActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorCatchArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  2718. {
  2719. }
  2720. void CHThorSkipCatchActivity::stop()
  2721. {
  2722. CHThorSimpleActivityBase::stop();
  2723. buffer.clear();
  2724. }
  2725. void CHThorSkipCatchActivity::onException(IException *E)
  2726. {
  2727. buffer->clear();
  2728. if (kind == TAKcreaterowcatch)
  2729. {
  2730. createRowAllocator();
  2731. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  2732. size32_t newSize = helper.transformOnExceptionCaught(rowBuilder, E);
  2733. if (newSize)
  2734. buffer->insert(rowBuilder.finalizeRowClear(newSize));
  2735. }
  2736. E->Release();
  2737. }
  2738. const void * CHThorSkipCatchActivity::nextRow()
  2739. {
  2740. if(!buffer)
  2741. {
  2742. buffer.setown(new CRowBuffer(input->queryOutputMeta(), true));
  2743. try
  2744. {
  2745. buffer->pull(input, (unsigned __int64) -1);
  2746. }
  2747. catch (IException *E)
  2748. {
  2749. onException(E);
  2750. }
  2751. catch (...)
  2752. {
  2753. onException(MakeStringException(2, "Unknown exception caught"));
  2754. }
  2755. }
  2756. const void * next = buffer->next();
  2757. if(next)
  2758. processed++;
  2759. return next;
  2760. }
  2761. //=====================================================================================================
  2762. CHThorOnFailLimitActivity::CHThorOnFailLimitActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorLimitArg &_arg, ThorActivityKind _kind) : CHThorSkipLimitActivity(_agent, _activityId, _subgraphId, _arg, _kind)
  2763. {
  2764. }
  2765. void CHThorOnFailLimitActivity::onLimitExceeded()
  2766. {
  2767. buffer->clear();
  2768. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  2769. size32_t newSize = helper.transformOnLimitExceeded(rowBuilder);
  2770. if (newSize)
  2771. buffer->insert(rowBuilder.finalizeRowClear(newSize));
  2772. }
  2773. //=====================================================================================================
  2774. CHThorIfActivity::CHThorIfActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorIfArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  2775. {
  2776. inputTrue = NULL;
  2777. inputFalse = NULL;
  2778. selectedInput = NULL;
  2779. }
  2780. void CHThorIfActivity::stop()
  2781. {
  2782. if (selectedInput)
  2783. selectedInput->stop();
  2784. CHThorSimpleActivityBase::stop();
  2785. }
  2786. void CHThorIfActivity::ready()
  2787. {
  2788. CHThorSimpleActivityBase::ready();
  2789. selectedInput = helper.getCondition() ? inputTrue : inputFalse;
  2790. if (selectedInput)
  2791. selectedInput->ready();
  2792. }
  2793. void CHThorIfActivity::setInput(unsigned index, IHThorInput *_input)
  2794. {
  2795. if (index==0)
  2796. inputTrue = _input;
  2797. else if (index == 1)
  2798. inputFalse = _input;
  2799. else
  2800. CHThorActivityBase::setInput(index, _input);
  2801. }
  2802. const void * CHThorIfActivity::nextRow()
  2803. {
  2804. if (!selectedInput)
  2805. return NULL;
  2806. const void *ret = selectedInput->nextRow();
  2807. if (ret)
  2808. processed++;
  2809. return ret;
  2810. }
  2811. //=====================================================================================================
  2812. CHThorCaseActivity::CHThorCaseActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorCaseArg &_arg, ThorActivityKind _kind) : CHThorMultiInputActivity(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  2813. {
  2814. }
  2815. void CHThorCaseActivity::ready()
  2816. {
  2817. //Evaluate the condition here to avoid calling ready() on the unused branch?
  2818. initialProcessed = processed;
  2819. selectedInput = NULL;
  2820. unsigned whichBranch = helper.getBranch();
  2821. if (whichBranch >= inputs.ordinality())
  2822. whichBranch = inputs.ordinality()-1;
  2823. selectedInput = inputs.item(whichBranch);
  2824. selectedInput->ready();
  2825. }
  2826. void CHThorCaseActivity::stop()
  2827. {
  2828. if (selectedInput)
  2829. selectedInput->stop();
  2830. }
  2831. const void *CHThorCaseActivity::nextRow()
  2832. {
  2833. if (!selectedInput)
  2834. return NULL;
  2835. const void *ret = selectedInput->nextRow();
  2836. if (ret)
  2837. processed++;
  2838. return ret;
  2839. }
  2840. //=====================================================================================================
  2841. CHThorSampleActivity::CHThorSampleActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorSampleArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  2842. {
  2843. }
  2844. void CHThorSampleActivity::ready()
  2845. {
  2846. CHThorSimpleActivityBase::ready();
  2847. numSamples = helper.getProportion();
  2848. whichSample = helper.getSampleNumber();
  2849. numToSkip = (whichSample ? whichSample-1 : 0);
  2850. anyThisGroup = false;
  2851. }
  2852. const void * CHThorSampleActivity::nextRow()
  2853. {
  2854. for (;;)
  2855. {
  2856. OwnedConstRoxieRow ret(input->nextRow());
  2857. if (!ret)
  2858. {
  2859. //this does work with groups - may or may not be useful...
  2860. //reset the sample for each group.... probably best.
  2861. numToSkip = (whichSample ? whichSample-1 : 0);
  2862. if (anyThisGroup)
  2863. {
  2864. anyThisGroup = false;
  2865. return NULL;
  2866. }
  2867. ret.setown(input->nextRow());
  2868. if (!ret)
  2869. return NULL; // eof...
  2870. }
  2871. if (numToSkip == 0)
  2872. {
  2873. anyThisGroup = true;
  2874. numToSkip = numSamples-1;
  2875. processed++;
  2876. return ret.getClear();
  2877. }
  2878. numToSkip--;
  2879. }
  2880. }
  2881. //=====================================================================================================
  2882. CHThorAggregateActivity::CHThorAggregateActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorAggregateArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  2883. {
  2884. }
  2885. void CHThorAggregateActivity::ready()
  2886. {
  2887. CHThorSimpleActivityBase::ready();
  2888. eof = false;
  2889. }
  2890. const void * CHThorAggregateActivity::nextRow()
  2891. {
  2892. if (eof)
  2893. return NULL;
  2894. const void * next = input->nextRow();
  2895. if (!next && input->isGrouped())
  2896. {
  2897. eof = true;
  2898. return NULL;
  2899. }
  2900. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  2901. helper.clearAggregate(rowBuilder);
  2902. if (next)
  2903. {
  2904. helper.processFirst(rowBuilder, next);
  2905. ReleaseRoxieRow(next);
  2906. bool abortEarly = (kind == TAKexistsaggregate) && !input->isGrouped();
  2907. if (!abortEarly)
  2908. {
  2909. for (;;)
  2910. {
  2911. next = input->nextRow();
  2912. if (!next)
  2913. break;
  2914. helper.processNext(rowBuilder, next);
  2915. ReleaseRoxieRow(next);
  2916. }
  2917. }
  2918. }
  2919. if (!input->isGrouped()) // either read all, or aborted early
  2920. eof = true;
  2921. processed++;
  2922. size32_t finalSize = outputMeta.getRecordSize(rowBuilder.getSelf());
  2923. return rowBuilder.finalizeRowClear(finalSize);
  2924. }
  2925. //=====================================================================================================
  2926. CHThorHashAggregateActivity::CHThorHashAggregateActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorHashAggregateArg &_arg, ThorActivityKind _kind, bool _isGroupedAggregate)
  2927. : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg),
  2928. isGroupedAggregate(_isGroupedAggregate),
  2929. aggregated(_arg, _arg)
  2930. {
  2931. }
  2932. void CHThorHashAggregateActivity::ready()
  2933. {
  2934. CHThorSimpleActivityBase::ready();
  2935. eof = false;
  2936. gathered = false;
  2937. }
  2938. void CHThorHashAggregateActivity::stop()
  2939. {
  2940. aggregated.reset();
  2941. CHThorSimpleActivityBase::stop();
  2942. }
  2943. const void * CHThorHashAggregateActivity::nextRow()
  2944. {
  2945. if (eof)
  2946. return NULL;
  2947. if (!gathered)
  2948. {
  2949. bool eog = true;
  2950. aggregated.start(rowAllocator);
  2951. for (;;)
  2952. {
  2953. OwnedConstRoxieRow next(input->nextRow());
  2954. if (!next)
  2955. {
  2956. if (isGroupedAggregate)
  2957. {
  2958. if (eog)
  2959. eof = true;
  2960. break;
  2961. }
  2962. next.setown(input->nextRow());
  2963. if (!next)
  2964. break;
  2965. }
  2966. eog = false;
  2967. try
  2968. {
  2969. aggregated.addRow(next);
  2970. }
  2971. catch(IException * e)
  2972. {
  2973. throw makeWrappedException(e);
  2974. }
  2975. }
  2976. gathered = true;
  2977. }
  2978. Owned<AggregateRowBuilder> next = aggregated.nextResult();
  2979. if (next)
  2980. {
  2981. processed++;
  2982. return next->finalizeRowClear();
  2983. }
  2984. if (!isGroupedAggregate)
  2985. eof = true;
  2986. aggregated.reset();
  2987. gathered = false;
  2988. return NULL;
  2989. }
  2990. //=====================================================================================================
  2991. CHThorSelectNActivity::CHThorSelectNActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorSelectNArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  2992. {
  2993. }
  2994. const void * CHThorSelectNActivity::defaultRow()
  2995. {
  2996. if (!rowAllocator)
  2997. createRowAllocator(); //We delay as often not needed...
  2998. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  2999. size32_t thisSize = helper.createDefault(rowBuilder);
  3000. return rowBuilder.finalizeRowClear(thisSize);
  3001. }
  3002. void CHThorSelectNActivity::ready()
  3003. {
  3004. CHThorSimpleActivityBase::ready();
  3005. finished = false;
  3006. }
  3007. const void * CHThorSelectNActivity::nextRow()
  3008. {
  3009. if (finished)
  3010. return NULL;
  3011. finished = true;
  3012. unsigned __int64 index = helper.getRowToSelect();
  3013. while (--index)
  3014. {
  3015. OwnedConstRoxieRow next(input->nextRow());
  3016. if (!next)
  3017. next.setown(input->nextRow());
  3018. if (!next)
  3019. {
  3020. processed++;
  3021. return defaultRow();
  3022. }
  3023. }
  3024. OwnedConstRoxieRow next(input->nextRow());
  3025. if (!next)
  3026. next.setown(input->nextRow());
  3027. if (!next)
  3028. next.setown(defaultRow());
  3029. processed++;
  3030. return next.getClear();
  3031. }
  3032. //=====================================================================================================
  3033. CHThorFirstNActivity::CHThorFirstNActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorFirstNArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  3034. {
  3035. grouped = outputMeta.isGrouped();
  3036. }
  3037. void CHThorFirstNActivity::ready()
  3038. {
  3039. CHThorSimpleActivityBase::ready();
  3040. skip = helper.numToSkip();
  3041. limit = helper.getLimit();
  3042. doneThisGroup = 0;
  3043. finished = (limit == 0);
  3044. if (limit + skip >= limit)
  3045. limit += skip;
  3046. }
  3047. const void * CHThorFirstNActivity::nextRow()
  3048. {
  3049. if (finished)
  3050. return NULL;
  3051. OwnedConstRoxieRow ret;
  3052. for (;;)
  3053. {
  3054. ret.setown(input->nextRow());
  3055. if (!ret)
  3056. {
  3057. if (grouped)
  3058. {
  3059. if (doneThisGroup > skip)
  3060. {
  3061. doneThisGroup = 0;
  3062. return NULL;
  3063. }
  3064. doneThisGroup = 0;
  3065. }
  3066. ret.setown(input->nextRow());
  3067. if (!ret)
  3068. {
  3069. finished = true;
  3070. return NULL;
  3071. }
  3072. }
  3073. doneThisGroup++;
  3074. if (doneThisGroup > skip)
  3075. break;
  3076. }
  3077. if (doneThisGroup <= limit)
  3078. {
  3079. processed++;
  3080. return ret.getClear();
  3081. }
  3082. if (grouped)
  3083. {
  3084. ret.setown(input->nextRow());
  3085. while (ret)
  3086. ret.setown(input->nextRow());
  3087. doneThisGroup = 0;
  3088. }
  3089. else
  3090. finished = true;
  3091. return NULL;
  3092. }
  3093. //=====================================================================================================
  3094. CHThorChooseSetsActivity::CHThorChooseSetsActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorChooseSetsArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  3095. {
  3096. numSets = helper.getNumSets();
  3097. setCounts = new unsigned[numSets];
  3098. }
  3099. CHThorChooseSetsActivity::~CHThorChooseSetsActivity()
  3100. {
  3101. delete [] setCounts;
  3102. }
  3103. void CHThorChooseSetsActivity::ready()
  3104. {
  3105. CHThorSimpleActivityBase::ready();
  3106. finished = false;
  3107. memset(setCounts, 0, sizeof(unsigned)*numSets);
  3108. helper.setCounts(setCounts);
  3109. }
  3110. const void * CHThorChooseSetsActivity::nextRow()
  3111. {
  3112. if (finished)
  3113. return NULL;
  3114. for (;;)
  3115. {
  3116. OwnedConstRoxieRow ret(input->nextRow());
  3117. if (!ret)
  3118. {
  3119. ret.setown(input->nextRow());
  3120. if (!ret)
  3121. return NULL;
  3122. }
  3123. processed++;
  3124. switch (helper.getRecordAction(ret))
  3125. {
  3126. case 2:
  3127. finished = true;
  3128. return ret.getClear();
  3129. case 1:
  3130. return ret.getClear();
  3131. }
  3132. }
  3133. }
  3134. //=====================================================================================================
  3135. CHThorChooseSetsExActivity::CHThorChooseSetsExActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorChooseSetsExArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  3136. {
  3137. numSets = helper.getNumSets();
  3138. setCounts = new unsigned[numSets];
  3139. memset(setCounts, 0, sizeof(unsigned)*numSets);
  3140. limits = (count_t *)checked_calloc(sizeof(count_t), numSets, "choose sets ex");
  3141. helper.getLimits(limits);
  3142. }
  3143. CHThorChooseSetsExActivity::~CHThorChooseSetsExActivity()
  3144. {
  3145. delete [] setCounts;
  3146. free(limits);
  3147. }
  3148. void CHThorChooseSetsExActivity::ready()
  3149. {
  3150. CHThorSimpleActivityBase::ready();
  3151. finished = false;
  3152. curIndex = 0;
  3153. memset(setCounts, 0, sizeof(unsigned)*numSets);
  3154. }
  3155. void CHThorChooseSetsExActivity::stop()
  3156. {
  3157. gathered.clear();
  3158. CHThorSimpleActivityBase::stop();
  3159. }
  3160. const void * CHThorChooseSetsExActivity::nextRow()
  3161. {
  3162. if (gathered.ordinality() == 0)
  3163. {
  3164. curIndex = 0;
  3165. const void * next = input->nextRow();
  3166. while(next)
  3167. {
  3168. gathered.append(next);
  3169. next = input->nextRow();
  3170. }
  3171. if(gathered.ordinality() == 0)
  3172. {
  3173. finished = true;
  3174. return NULL;
  3175. }
  3176. ForEachItemIn(idx1, gathered)
  3177. {
  3178. unsigned category = helper.getCategory(gathered.item(idx1));
  3179. if (category)
  3180. setCounts[category-1]++;
  3181. }
  3182. calculateSelection();
  3183. }
  3184. while (gathered.isItem(curIndex))
  3185. {
  3186. OwnedConstRoxieRow row(gathered.itemClear(curIndex++));
  3187. if (includeRow(row))
  3188. {
  3189. processed++;
  3190. return row.getClear();
  3191. }
  3192. }
  3193. gathered.clear();
  3194. return NULL;
  3195. }
  3196. //=====================================================================================================
  3197. CHThorChooseSetsLastActivity::CHThorChooseSetsLastActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorChooseSetsExArg &_arg, ThorActivityKind _kind) : CHThorChooseSetsExActivity(_agent, _activityId, _subgraphId, _arg, _kind)
  3198. {
  3199. numToSkip = (unsigned *)checked_calloc(sizeof(unsigned), numSets, "choose sets last");
  3200. }
  3201. CHThorChooseSetsLastActivity::~CHThorChooseSetsLastActivity()
  3202. {
  3203. free(numToSkip);
  3204. }
  3205. void CHThorChooseSetsLastActivity::ready()
  3206. {
  3207. CHThorChooseSetsExActivity::ready();
  3208. memset(numToSkip, 0, sizeof(unsigned) * numSets);
  3209. }
  3210. void CHThorChooseSetsLastActivity::calculateSelection()
  3211. {
  3212. for (unsigned idx=0; idx < numSets; idx++)
  3213. {
  3214. if (setCounts[idx] < limits[idx])
  3215. numToSkip[idx] = 0;
  3216. else
  3217. numToSkip[idx] = (unsigned)(setCounts[idx] - limits[idx]);
  3218. }
  3219. }
  3220. bool CHThorChooseSetsLastActivity::includeRow(const void * row)
  3221. {
  3222. unsigned category = helper.getCategory(row);
  3223. if (category)
  3224. {
  3225. if (numToSkip[category-1] == 0)
  3226. return true;
  3227. numToSkip[category-1]--;
  3228. }
  3229. return false;
  3230. }
  3231. //=====================================================================================================
  3232. CHThorChooseSetsEnthActivity::CHThorChooseSetsEnthActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorChooseSetsExArg &_arg, ThorActivityKind _kind) : CHThorChooseSetsExActivity(_agent, _activityId, _subgraphId, _arg, _kind)
  3233. {
  3234. counter = (unsigned __int64 *)checked_calloc(sizeof(unsigned __int64), numSets, "choose sets enth");
  3235. }
  3236. CHThorChooseSetsEnthActivity::~CHThorChooseSetsEnthActivity()
  3237. {
  3238. free(counter);
  3239. }
  3240. void CHThorChooseSetsEnthActivity::ready()
  3241. {
  3242. CHThorChooseSetsExActivity::ready();
  3243. memset(counter, 0, sizeof(unsigned __int64) * numSets);
  3244. }
  3245. void CHThorChooseSetsEnthActivity::calculateSelection()
  3246. {
  3247. }
  3248. bool CHThorChooseSetsEnthActivity::includeRow(const void * row)
  3249. {
  3250. unsigned category = helper.getCategory(row);
  3251. if (category)
  3252. {
  3253. counter[category-1] += limits[category-1];
  3254. if(counter[category-1] >= setCounts[category-1])
  3255. {
  3256. counter[category-1] -= setCounts[category-1];
  3257. return true;
  3258. }
  3259. }
  3260. return false;
  3261. }
  3262. //=====================================================================================================
  3263. CHThorDegroupActivity::CHThorDegroupActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorDegroupArg &_arg, ThorActivityKind _kind) : CHThorSteppableActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  3264. {
  3265. }
  3266. const void * CHThorDegroupActivity::nextRow()
  3267. {
  3268. const void * ret = input->ungroupedNextRow();
  3269. if (ret)
  3270. processed++;
  3271. return ret;
  3272. }
  3273. const void * CHThorDegroupActivity::nextRowGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
  3274. {
  3275. const void * ret = input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
  3276. if (ret)
  3277. processed++;
  3278. return ret;
  3279. }
  3280. bool CHThorDegroupActivity::isGrouped()
  3281. {
  3282. return false;
  3283. }
  3284. //=====================================================================================================
  3285. CHThorGroupActivity::CHThorGroupActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorGroupArg &_arg, ThorActivityKind _kind) : CHThorSteppableActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  3286. {
  3287. }
  3288. bool CHThorGroupActivity::isGrouped()
  3289. {
  3290. return true;
  3291. }
  3292. void CHThorGroupActivity::ready()
  3293. {
  3294. CHThorSimpleActivityBase::ready();
  3295. next.clear();
  3296. endPending = false;
  3297. firstDone = false;
  3298. }
  3299. void CHThorGroupActivity::stop()
  3300. {
  3301. CHThorSimpleActivityBase::stop();
  3302. next.clear();
  3303. }
  3304. const void *CHThorGroupActivity::nextRow()
  3305. {
  3306. if (!firstDone)
  3307. {
  3308. firstDone = true;
  3309. next.setown(input->nextRow());
  3310. }
  3311. if (endPending)
  3312. {
  3313. endPending = false;
  3314. return NULL;
  3315. }
  3316. OwnedConstRoxieRow prev(next.getClear());
  3317. next.setown(input->nextRow());
  3318. if (!next) // skip incoming groups. (should it sub-group??)
  3319. next.setown(input->nextRow());
  3320. if (next)
  3321. {
  3322. assertex(prev); // If this fails, you have an initial empty group. That is not legal.
  3323. if (!helper.isSameGroup(prev, next))
  3324. endPending = true;
  3325. }
  3326. if (prev)
  3327. processed++;
  3328. return prev.getClear();
  3329. }
  3330. const void * CHThorGroupActivity::nextRowGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
  3331. {
  3332. if (firstDone)
  3333. {
  3334. if (next)
  3335. {
  3336. if (stepCompare->docompare(next, seek, numFields) >= 0)
  3337. return nextRow();
  3338. }
  3339. }
  3340. next.setown(input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra));
  3341. firstDone = true;
  3342. return nextRow();
  3343. }
  3344. //=====================================================================================================
  3345. CHThorGroupSortActivity::CHThorGroupSortActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorSortArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  3346. {
  3347. gotSorted = false;
  3348. }
  3349. void CHThorGroupSortActivity::ready()
  3350. {
  3351. CHThorSimpleActivityBase::ready();
  3352. if(!sorter)
  3353. createSorter();
  3354. }
  3355. void CHThorGroupSortActivity::stop()
  3356. {
  3357. if(sorter)
  3358. {
  3359. if(sorterIsConst)
  3360. sorter->killSorted();
  3361. else
  3362. sorter.clear();
  3363. }
  3364. gotSorted = false;
  3365. diskReader.clear();
  3366. CHThorSimpleActivityBase::stop();
  3367. }
  3368. const void *CHThorGroupSortActivity::nextRow()
  3369. {
  3370. if(!gotSorted)
  3371. getSorted();
  3372. if(diskReader)
  3373. {
  3374. const void *row = diskReader->nextRow();
  3375. if (row)
  3376. return row;
  3377. diskReader.clear();
  3378. }
  3379. else
  3380. {
  3381. const void * ret = sorter->getNextSorted();
  3382. if(ret)
  3383. {
  3384. processed++;
  3385. return ret;
  3386. }
  3387. }
  3388. sorter->killSorted();
  3389. gotSorted = false;
  3390. return NULL;
  3391. }
  3392. void CHThorGroupSortActivity::createSorter()
  3393. {
  3394. unsigned flags = helper.getAlgorithmFlags();
  3395. sorterIsConst = ((flags & TAFconstant) != 0);
  3396. OwnedRoxieString algoname(helper.getAlgorithm());
  3397. if(!algoname)
  3398. {
  3399. if((flags & TAFunstable) != 0)
  3400. sorter.setown(new CQuickSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep));
  3401. else
  3402. sorter.setown(new CHeapSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep));
  3403. return;
  3404. }
  3405. if(stricmp(algoname, "quicksort") == 0)
  3406. {
  3407. if((flags & TAFstable) != 0)
  3408. sorter.setown(new CStableQuickSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep, this));
  3409. else
  3410. sorter.setown(new CQuickSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep));
  3411. }
  3412. else if(stricmp(algoname, "parquicksort") == 0)
  3413. {
  3414. if((flags & TAFstable) != 0)
  3415. sorter.setown(new CParallelStableQuickSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep, this));
  3416. else
  3417. sorter.setown(new CParallelQuickSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep));
  3418. }
  3419. else if(stricmp(algoname, "mergesort") == 0)
  3420. {
  3421. if((flags & TAFparallel) != 0)
  3422. sorter.setown(new CParallelStableMergeSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep, this));
  3423. else
  3424. sorter.setown(new CStableMergeSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep, this));
  3425. }
  3426. else if(stricmp(algoname, "parmergesort") == 0)
  3427. sorter.setown(new CParallelStableMergeSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep, this));
  3428. else if(stricmp(algoname, "heapsort") == 0)
  3429. sorter.setown(new CHeapSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep));
  3430. else if(stricmp(algoname, "insertionsort") == 0)
  3431. {
  3432. if((flags & TAFstable) != 0)
  3433. sorter.setown(new CStableInsertionSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep));
  3434. else
  3435. sorter.setown(new CInsertionSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep));
  3436. }
  3437. else
  3438. {
  3439. StringBuffer sb;
  3440. sb.appendf("Ignoring unsupported sort order algorithm '%s', using default", algoname.get());
  3441. agent.addWuException(sb.str(),WRN_UnsupportedAlgorithm,SeverityWarning,"hthor");
  3442. if((flags & TAFunstable) != 0)
  3443. sorter.setown(new CQuickSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep));
  3444. else
  3445. sorter.setown(new CHeapSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep));
  3446. }
  3447. sorter->setActivityId(activityId);
  3448. }
  3449. void CHThorGroupSortActivity::getSorted()
  3450. {
  3451. diskMerger.clear();
  3452. diskReader.clear();
  3453. queryRowManager()->addRowBuffer(this);//register for OOM callbacks
  3454. const void * next;
  3455. while((next = input->nextRow()) != NULL)
  3456. {
  3457. if (!sorter->addRow(next))
  3458. {
  3459. {
  3460. //Unlikely that this code will ever be executed but added for comfort
  3461. roxiemem::RoxieOutputRowArrayLock block(sorter->getRowArray());
  3462. sorter->flushRows();
  3463. sortAndSpillRows();
  3464. //Ensure new rows are written to the head of the array. It needs to be a separate call because
  3465. //performSort() cannot shift active row pointer since it can be called from any thread
  3466. sorter->flushRows();
  3467. }
  3468. if (!sorter->addRow(next))
  3469. {
  3470. ReleaseRoxieRow(next);
  3471. throw MakeStringException(0, "Insufficient memory to append sort row");
  3472. }
  3473. }
  3474. }
  3475. queryRowManager()->removeRowBuffer(this);//unregister for OOM callbacks
  3476. sorter->flushRows();
  3477. if(diskMerger)
  3478. {
  3479. sortAndSpillRows();
  3480. sorter->killSorted();
  3481. ICompare *compare = helper.queryCompare();
  3482. diskReader.setown(diskMerger->merge(compare));
  3483. }
  3484. else
  3485. {
  3486. sorter->performSort();
  3487. }
  3488. gotSorted = true;
  3489. }
  3490. //interface roxiemem::IBufferedRowCallback
  3491. unsigned CHThorGroupSortActivity::getSpillCost() const
  3492. {
  3493. return 10;
  3494. }
  3495. unsigned CHThorGroupSortActivity::getActivityId() const
  3496. {
  3497. return activityId;
  3498. }
  3499. bool CHThorGroupSortActivity::freeBufferedRows(bool critical)
  3500. {
  3501. roxiemem::RoxieOutputRowArrayLock block(sorter->getRowArray());
  3502. return sortAndSpillRows();
  3503. }
  3504. bool CHThorGroupSortActivity::sortAndSpillRows()
  3505. {
  3506. if (0 == sorter->numCommitted())
  3507. return false;
  3508. if(!diskMerger)
  3509. {
  3510. StringBuffer fbase;
  3511. agent.getTempfileBase(fbase).append(PATHSEPCHAR).appendf("spill_sort_%p", this);
  3512. PROGLOG("SORT: spilling to disk, filename base %s", fbase.str());
  3513. class CHThorRowLinkCounter : implements IRowLinkCounter, public CSimpleInterface
  3514. {
  3515. public:
  3516. IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
  3517. virtual void releaseRow(const void *row)
  3518. {
  3519. ReleaseRoxieRow(row);
  3520. }
  3521. virtual void linkRow(const void *row)
  3522. {
  3523. LinkRoxieRow(row);
  3524. }
  3525. };
  3526. Owned<IRowLinkCounter> linker = new CHThorRowLinkCounter();
  3527. Owned<IRowInterfaces> rowInterfaces = createRowInterfaces(input->queryOutputMeta(), activityId, 0, agent.queryCodeContext());
  3528. diskMerger.setown(createDiskMerger(rowInterfaces, linker, fbase.str()));
  3529. }
  3530. sorter->performSort();
  3531. sorter->spillSortedToDisk(diskMerger);
  3532. return true;
  3533. }
  3534. // Base for Quick sort and both Insertion sorts
  3535. void CSimpleSorterBase::spillSortedToDisk(IDiskMerger * merger)
  3536. {
  3537. Owned<IRowWriter> out = merger->createWriteBlock();
  3538. for (;;)
  3539. {
  3540. const void *row = getNextSorted();
  3541. if (!row)
  3542. break;
  3543. out->putRow(row);
  3544. }
  3545. finger = 0;
  3546. out->flush();
  3547. rowsToSort.noteSpilled(rowsToSort.numCommitted());
  3548. }
  3549. // Quick sort
  3550. void CQuickSorter::performSort()
  3551. {
  3552. size32_t numRows = rowsToSort.numCommitted();
  3553. if (numRows)
  3554. {
  3555. const void * * rows = rowsToSort.getBlock(numRows);
  3556. qsortvec((void * *)rows, numRows, *compare);
  3557. finger = 0;
  3558. }
  3559. }
  3560. // Quick sort
  3561. void CParallelQuickSorter::performSort()
  3562. {
  3563. size32_t numRows = rowsToSort.numCommitted();
  3564. if (numRows)
  3565. {
  3566. const void * * rows = rowsToSort.getBlock(numRows);
  3567. parqsortvec((void * *)rows, numRows, *compare);
  3568. finger = 0;
  3569. }
  3570. }
  3571. // StableQuick sort
  3572. bool CStableSorter::addRow(const void * next)
  3573. {
  3574. roxiemem::rowidx_t nextRowCapacity = rowsToSort.rowCapacity() + 1;//increment capacity for the row we are about to add
  3575. if (nextRowCapacity > indexCapacity)
  3576. {
  3577. void *** newIndex = (void ***)rowManager->allocate(nextRowCapacity * sizeof(void*), activityId);//could force an OOM callback
  3578. if (newIndex)
  3579. {
  3580. roxiemem::RoxieOutputRowArrayLock block(getRowArray());//could force an OOM callback after index is freed but before index,indexCapacity is updated
  3581. ReleaseRoxieRow(index);
  3582. index = newIndex;
  3583. indexCapacity = RoxieRowCapacity(index) / sizeof(void*);
  3584. }
  3585. else
  3586. {
  3587. killSorted();
  3588. ReleaseRoxieRow(next);
  3589. throw MakeStringException(0, "Insufficient memory to allocate StableQuickSorter index");
  3590. }
  3591. }
  3592. return CSimpleSorterBase::addRow(next);
  3593. }
  3594. void CStableSorter::spillSortedToDisk(IDiskMerger * merger)
  3595. {
  3596. CSimpleSorterBase::spillSortedToDisk(merger);
  3597. ReleaseRoxieRow(index);
  3598. index = NULL;
  3599. indexCapacity = 0;
  3600. }
  3601. void CStableSorter::killSorted()
  3602. {
  3603. CSimpleSorterBase::killSorted();
  3604. ReleaseRoxieRow(index);
  3605. index = NULL;
  3606. indexCapacity = 0;
  3607. }
  3608. // StableQuick sort
  3609. void CStableQuickSorter::performSort()
  3610. {
  3611. size32_t numRows = rowsToSort.numCommitted();
  3612. if (numRows)
  3613. {
  3614. const void * * rows = rowsToSort.getBlock(numRows);
  3615. qsortvecstableinplace((void * *)rows, numRows, *compare, (void * *)index);
  3616. finger = 0;
  3617. }
  3618. }
  3619. void CParallelStableQuickSorter::performSort()
  3620. {
  3621. size32_t numRows = rowsToSort.numCommitted();
  3622. if (numRows)
  3623. {
  3624. const void * * rows = rowsToSort.getBlock(numRows);
  3625. parqsortvecstableinplace((void * *)rows, numRows, *compare, (void * *)index);
  3626. finger = 0;
  3627. }
  3628. }
  3629. // StableMerge sort
  3630. void CStableMergeSorter::performSort()
  3631. {
  3632. size32_t numRows = rowsToSort.numCommitted();
  3633. if (numRows)
  3634. {
  3635. const void * * rows = rowsToSort.getBlock(numRows);
  3636. msortvecstableinplace((void * *)rows, numRows, *compare, (void * *)index);
  3637. finger = 0;
  3638. }
  3639. }
  3640. void CParallelStableMergeSorter::performSort()
  3641. {
  3642. size32_t numRows = rowsToSort.numCommitted();
  3643. if (numRows)
  3644. {
  3645. const void * * rows = rowsToSort.getBlock(numRows);
  3646. parmsortvecstableinplace((void * *)rows, numRows, *compare, (void * *)index);
  3647. finger = 0;
  3648. }
  3649. }
  3650. // Heap sort
  3651. void CHeapSorter::performSort()
  3652. {
  3653. size32_t numRows = rowsToSort.numCommitted();
  3654. if (numRows)
  3655. {
  3656. const void * * rows = rowsToSort.getBlock(numRows);
  3657. heapsize = numRows;
  3658. for (unsigned i = 0; i < numRows; i++)
  3659. {
  3660. heap.append(i);
  3661. heap_push_up(i, heap.getArray(), rows, compare);
  3662. }
  3663. }
  3664. }
  3665. void CHeapSorter::spillSortedToDisk(IDiskMerger * merger)
  3666. {
  3667. CSimpleSorterBase::spillSortedToDisk(merger);
  3668. heap.kill();
  3669. heapsize = 0;
  3670. }
  3671. const void * CHeapSorter::getNextSorted()
  3672. {
  3673. if(heapsize)
  3674. {
  3675. size32_t numRows = rowsToSort.numCommitted();
  3676. if (numRows)
  3677. {
  3678. const void * * rows = rowsToSort.getBlock(numRows);
  3679. unsigned top = heap.item(0);
  3680. --heapsize;
  3681. heap.replace(heap.item(heapsize), 0);
  3682. heap_push_down(0, heapsize, heap.getArray(), rows, compare);
  3683. const void * row = rows[top];
  3684. rows[top] = NULL;
  3685. return row;
  3686. }
  3687. }
  3688. return NULL;
  3689. }
  3690. void CHeapSorter::killSorted()
  3691. {
  3692. CSimpleSorterBase::killSorted();
  3693. heap.kill();
  3694. heapsize = 0;
  3695. }
  3696. // Insertion sorts
  3697. void CInsertionSorter::performSort()
  3698. {
  3699. size32_t numRows = rowsToSort.numCommitted();
  3700. if (numRows)
  3701. {
  3702. const void * * rows = rowsToSort.getBlock(numRows);
  3703. for (unsigned i = 0; i < numRows; i++)
  3704. {
  3705. binary_vec_insert(rowsToSort.query(i), rows, i, *compare);
  3706. }
  3707. finger = 0;
  3708. }
  3709. }
  3710. void CStableInsertionSorter::performSort()
  3711. {
  3712. size32_t numRows = rowsToSort.numCommitted();
  3713. if (numRows)
  3714. {
  3715. const void * * rows = rowsToSort.getBlock(numRows);
  3716. for (unsigned i = 0; i < numRows; i++)
  3717. {
  3718. binary_vec_insert_stable(rowsToSort.query(i), rows, i, *compare);
  3719. }
  3720. finger = 0;
  3721. }
  3722. }
  3723. //=====================================================================================================
  3724. CHThorGroupedActivity::CHThorGroupedActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorGroupedArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  3725. {
  3726. }
  3727. void CHThorGroupedActivity::ready()
  3728. {
  3729. CHThorSimpleActivityBase::ready();
  3730. firstDone = false;
  3731. nextRowIndex = 0;
  3732. }
  3733. void CHThorGroupedActivity::stop()
  3734. {
  3735. CHThorSimpleActivityBase::stop();
  3736. next[0].clear();
  3737. next[1].clear();
  3738. next[2].clear();
  3739. }
  3740. const void *CHThorGroupedActivity::nextRow()
  3741. {
  3742. if (!firstDone)
  3743. {
  3744. next[0].setown(input->nextRow());
  3745. next[1].setown(input->nextRow());
  3746. nextRowIndex = 0;
  3747. }
  3748. unsigned nextToCompare = (nextRowIndex + 1) % 3;
  3749. unsigned nextToFill = (nextRowIndex + 2) % 3;
  3750. next[nextToFill].setown(input->nextRow());
  3751. OwnedConstRoxieRow ret(next[nextRowIndex].getClear());
  3752. if (ret)
  3753. {
  3754. if (next[nextToCompare])
  3755. {
  3756. if (!helper.isSameGroup(ret, next[nextToCompare]))
  3757. throw MakeStringException(100, "GROUPED(%u), expected a group break between adjacent rows (rows %" I64F "d, %" I64F "d) ", activityId, processed+1, processed+2);
  3758. }
  3759. else if (next[nextToFill])
  3760. {
  3761. if (helper.isSameGroup(ret, next[nextToFill]))
  3762. throw MakeStringException(100, "GROUPED(%u), unexpected group break found between rows %" I64F "d and %" I64F "d)", activityId, processed+1, processed+2);
  3763. }
  3764. processed++;
  3765. }
  3766. nextRowIndex = nextToCompare;
  3767. return ret.getClear();
  3768. }
  3769. //=====================================================================================================
  3770. CHThorSortedActivity::CHThorSortedActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorSortedArg &_arg, ThorActivityKind _kind) : CHThorSteppableActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  3771. {
  3772. //MORE: Should probably have a inter group and intra group sort functions
  3773. compare = helper.queryCompare();
  3774. }
  3775. void CHThorSortedActivity::ready()
  3776. {
  3777. CHThorSimpleActivityBase::ready();
  3778. firstDone = false;
  3779. }
  3780. void CHThorSortedActivity::stop()
  3781. {
  3782. CHThorSimpleActivityBase::stop();
  3783. next.clear();
  3784. }
  3785. const void *CHThorSortedActivity::nextRow()
  3786. {
  3787. if (!firstDone)
  3788. {
  3789. firstDone = true;
  3790. next.setown(input->nextRow());
  3791. }
  3792. OwnedConstRoxieRow prev(next.getClear());
  3793. next.setown(input->nextRow());
  3794. if (prev && next)
  3795. if (compare->docompare(prev, next) > 0)
  3796. throw MakeStringException(100, "SORTED(%u) detected incorrectly sorted rows (row %" I64F "d, %" I64F "d))", activityId, processed+1, processed+2);
  3797. if (prev)
  3798. processed++;
  3799. return prev.getClear();
  3800. }
  3801. const void * CHThorSortedActivity::nextRowGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
  3802. {
  3803. if (next)
  3804. {
  3805. if (stepCompare->docompare(next, seek, numFields) >= 0)
  3806. return nextRow();
  3807. }
  3808. firstDone = true;
  3809. next.setown(input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra));
  3810. return nextRow();
  3811. }
  3812. //=====================================================================================================
  3813. CHThorTraceActivity::CHThorTraceActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorTraceArg &_arg, ThorActivityKind _kind)
  3814. : CHThorSteppableActivityBase(_agent, _activityId, _subgraphId, _arg, _kind),
  3815. helper(_arg), keepLimit(0), skip(0), sample(0), traceEnabled(false)
  3816. {
  3817. }
  3818. void CHThorTraceActivity::ready()
  3819. {
  3820. CHThorSimpleActivityBase::ready();
  3821. traceEnabled = agent.queryWorkUnit()->getDebugValueBool("traceEnabled", false);
  3822. if (traceEnabled && helper.canMatchAny())
  3823. {
  3824. keepLimit = helper.getKeepLimit();
  3825. if (keepLimit==(unsigned) -1)
  3826. keepLimit = agent.queryWorkUnit()->getDebugValueInt("traceLimit", 10);
  3827. skip = helper.getSkip();
  3828. sample = helper.getSample();
  3829. if (sample)
  3830. sample--;
  3831. name.setown(helper.getName());
  3832. if (!name)
  3833. name.set("Row");
  3834. }
  3835. else
  3836. keepLimit = 0;
  3837. }
  3838. void CHThorTraceActivity::stop()
  3839. {
  3840. CHThorSimpleActivityBase::stop();
  3841. name.clear();
  3842. }
  3843. const void *CHThorTraceActivity::nextRow()
  3844. {
  3845. OwnedConstRoxieRow ret(input->nextRow());
  3846. if (!ret)
  3847. return NULL;
  3848. onTrace(ret);
  3849. processed++;
  3850. return ret.getClear();
  3851. }
  3852. const void * CHThorTraceActivity::nextRowGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
  3853. {
  3854. OwnedConstRoxieRow ret(input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra));
  3855. if (ret)
  3856. {
  3857. onTrace(ret);
  3858. processed++;
  3859. }
  3860. return ret.getClear();
  3861. }
  3862. void CHThorTraceActivity::onTrace(const void *row)
  3863. {
  3864. if (keepLimit && helper.isValid(row))
  3865. {
  3866. if (skip)
  3867. skip--;
  3868. else if (sample)
  3869. sample--;
  3870. else
  3871. {
  3872. CommonXmlWriter xmlwrite(XWFnoindent);
  3873. outputMeta.toXML((const byte *) row, xmlwrite);
  3874. DBGLOG("TRACE: <%s>%s<%s>", name.get(), xmlwrite.str(), name.get());
  3875. keepLimit--;
  3876. sample = helper.getSample();
  3877. if (sample)
  3878. sample--;
  3879. }
  3880. }
  3881. }
  3882. //=====================================================================================================
  3883. void getLimitType(unsigned flags, bool & limitFail, bool & limitOnFail)
  3884. {
  3885. if((flags & JFmatchAbortLimitSkips) != 0)
  3886. {
  3887. limitFail = false;
  3888. limitOnFail = false;
  3889. }
  3890. else
  3891. {
  3892. limitOnFail = ((flags & JFonfail) != 0);
  3893. limitFail = !limitOnFail;
  3894. }
  3895. }
  3896. CHThorJoinActivity::CHThorJoinActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorJoinArg &_arg, ThorActivityKind _kind)
  3897. : CHThorActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg), outBuilder(NULL)
  3898. {
  3899. }
  3900. void CHThorJoinActivity::ready()
  3901. {
  3902. CHThorActivityBase::ready();
  3903. input1->ready();
  3904. bool isStable = (helper.getJoinFlags() & JFunstable) == 0;
  3905. RoxieSortAlgorithm sortAlgorithm = isStable ? stableSpillingQuickSortAlgorithm : spillingQuickSortAlgorithm;
  3906. StringBuffer tempBase;
  3907. agent.getTempfileBase(tempBase);
  3908. if (helper.isLeftAlreadySorted())
  3909. sortedLeftInput.setown(createDegroupedInputReader(&input->queryStream()));
  3910. else
  3911. sortedLeftInput.setown(createSortedInputReader(&input->queryStream(), createSortAlgorithm(sortAlgorithm, helper.queryCompareLeft(), *queryRowManager(), input->queryOutputMeta(), agent.queryCodeContext(), tempBase, activityId)));
  3912. ICompare *compareRight = helper.queryCompareRight();
  3913. if (helper.isRightAlreadySorted())
  3914. groupedSortedRightInput.setown(createGroupedInputReader(&input1->queryStream(), compareRight));
  3915. else
  3916. groupedSortedRightInput.setown(createSortedGroupedInputReader(&input1->queryStream(), compareRight, createSortAlgorithm(sortAlgorithm, compareRight, *queryRowManager(), input1->queryOutputMeta(), agent.queryCodeContext(), tempBase, activityId)));
  3917. outBuilder.setAllocator(rowAllocator);
  3918. leftOuterJoin = (helper.getJoinFlags() & JFleftouter) != 0;
  3919. rightOuterJoin = (helper.getJoinFlags() & JFrightouter) != 0;
  3920. exclude = (helper.getJoinFlags() & JFexclude) != 0;
  3921. getLimitType(helper.getJoinFlags(), limitFail, limitOnFail);
  3922. if (rightOuterJoin && !defaultLeft)
  3923. createDefaultLeft();
  3924. if ((leftOuterJoin || limitOnFail) && !defaultRight)
  3925. createDefaultRight();
  3926. betweenjoin = ((helper.getJoinFlags() & JFslidingmatch) != 0);
  3927. assertex(!(betweenjoin && rightOuterJoin));
  3928. keepLimit = helper.getKeepLimit();
  3929. if (keepLimit == 0)
  3930. keepLimit = (unsigned)-1;
  3931. atmostLimit = helper.getJoinLimit();
  3932. if(atmostLimit == 0)
  3933. atmostLimit = (unsigned)-1;
  3934. else
  3935. assertex(!rightOuterJoin && !betweenjoin);
  3936. abortLimit = helper.getMatchAbortLimit();
  3937. if (abortLimit == 0)
  3938. abortLimit = (unsigned)-1;
  3939. assertex((helper.getJoinFlags() & (JFfirst | JFfirstleft | JFfirstright)) == 0); // no longer supported
  3940. if(betweenjoin)
  3941. {
  3942. collate = helper.queryCompareLeftRightLower();
  3943. collateupper = helper.queryCompareLeftRightUpper();
  3944. }
  3945. else
  3946. {
  3947. collate = collateupper = helper.queryCompareLeftRight();
  3948. }
  3949. rightIndex = 0;
  3950. joinCounter = 0;
  3951. failingLimit.clear();
  3952. state = JSfill;
  3953. if ((helper.getJoinFlags() & JFlimitedprefixjoin) && helper.getJoinLimit())
  3954. { //Limited Match Join (s[1..n])
  3955. limitedhelper.setown(createRHLimitedCompareHelper());
  3956. limitedhelper->init( helper.getJoinLimit(), groupedSortedRightInput, collate, helper.queryPrefixCompare() );
  3957. }
  3958. }
  3959. void CHThorJoinActivity::stop()
  3960. {
  3961. outBuilder.clear();
  3962. right.clear();
  3963. left.clear();
  3964. pendingRight.clear();
  3965. sortedLeftInput.clear();
  3966. groupedSortedRightInput.clear();
  3967. CHThorActivityBase::stop();
  3968. input1->stop();
  3969. }
  3970. void CHThorJoinActivity::setInput(unsigned index, IHThorInput *_input)
  3971. {
  3972. if (index==1)
  3973. input1 = _input;
  3974. else
  3975. CHThorActivityBase::setInput(index, _input);
  3976. }
  3977. void CHThorJoinActivity::createDefaultLeft()
  3978. {
  3979. if (!defaultLeft)
  3980. {
  3981. if (!defaultLeftAllocator)
  3982. defaultLeftAllocator.setown(agent.queryCodeContext()->getRowAllocator(input->queryOutputMeta(), activityId));
  3983. RtlDynamicRowBuilder rowBuilder(defaultLeftAllocator);
  3984. size32_t thisSize = helper.createDefaultLeft(rowBuilder);
  3985. defaultLeft.setown(rowBuilder.finalizeRowClear(thisSize));
  3986. }
  3987. }
  3988. void CHThorJoinActivity::createDefaultRight()
  3989. {
  3990. if (!defaultRight)
  3991. {
  3992. if (!defaultRightAllocator)
  3993. defaultRightAllocator.setown(agent.queryCodeContext()->getRowAllocator(input1->queryOutputMeta(), activityId));
  3994. RtlDynamicRowBuilder rowBuilder(defaultRightAllocator);
  3995. size32_t thisSize = helper.createDefaultRight(rowBuilder);
  3996. defaultRight.setown(rowBuilder.finalizeRowClear(thisSize));
  3997. }
  3998. }
  3999. void CHThorJoinActivity::fillLeft()
  4000. {
  4001. matchedLeft = false;
  4002. left.setown(sortedLeftInput->nextRow()); // NOTE: already degrouped
  4003. if(betweenjoin && left && pendingRight && (collate->docompare(left, pendingRight) >= 0))
  4004. fillRight();
  4005. if (limitedhelper && 0==rightIndex)
  4006. {
  4007. rightIndex = 0;
  4008. joinCounter = 0;
  4009. right.clear();
  4010. matchedRight.kill();
  4011. if (left)
  4012. {
  4013. limitedhelper->getGroup(right,left);
  4014. ForEachItemIn(idx, right)
  4015. matchedRight.append(false);
  4016. }
  4017. }
  4018. }
  4019. void CHThorJoinActivity::fillRight()
  4020. {
  4021. if (limitedhelper)
  4022. return;
  4023. failingLimit.clear();
  4024. if(betweenjoin && left)
  4025. {
  4026. aindex_t start = 0;
  4027. while(right.isItem(start) && (collateupper->docompare(left, right.item(start)) > 0))
  4028. start++;
  4029. if(start>0)
  4030. right.clearPart(0, start);
  4031. }
  4032. else
  4033. right.clear();
  4034. rightIndex = 0;
  4035. joinCounter = 0;
  4036. unsigned groupCount = 0;
  4037. while(true)
  4038. {
  4039. OwnedConstRoxieRow next;
  4040. if(pendingRight)
  4041. {
  4042. next.setown(pendingRight.getClear());
  4043. }
  4044. else
  4045. {
  4046. next.setown(groupedSortedRightInput->nextRow());
  4047. }
  4048. if(!rightOuterJoin && next && (!left || (collateupper->docompare(left, next) > 0))) // if right is less than left, and not right outer, can skip group
  4049. {
  4050. while(next)
  4051. next.setown(groupedSortedRightInput->nextRow());
  4052. continue;
  4053. }
  4054. while(next)
  4055. {
  4056. if(groupCount==abortLimit)
  4057. {
  4058. if(limitFail)
  4059. failLimit();
  4060. if ( agent.queryCodeContext()->queryDebugContext())
  4061. agent.queryCodeContext()->queryDebugContext()->checkBreakpoint(DebugStateLimit, NULL, static_cast<IActivityBase *>(this));
  4062. if(limitOnFail)
  4063. {
  4064. assertex(!failingLimit);
  4065. try
  4066. {
  4067. failLimit();
  4068. }
  4069. catch(IException * except)
  4070. {
  4071. failingLimit.setown(except);
  4072. }
  4073. assertex(failingLimit);
  4074. }
  4075. right.append(next.getClear());
  4076. do
  4077. {
  4078. next.setown(groupedSortedRightInput->nextRow());
  4079. } while(next);
  4080. break;
  4081. }
  4082. else if(groupCount==atmostLimit)
  4083. {
  4084. right.clear();
  4085. groupCount = 0;
  4086. while(next)
  4087. {
  4088. next.setown(groupedSortedRightInput->nextRow());
  4089. }
  4090. }
  4091. else
  4092. {
  4093. right.append(next.getClear());
  4094. groupCount++;
  4095. }
  4096. next.setown(groupedSortedRightInput->nextRow());
  4097. }
  4098. // normally only want to read one right group, but if is between join and next right group is in window for left, need to continue
  4099. if(betweenjoin && left)
  4100. {
  4101. pendingRight.setown(groupedSortedRightInput->nextRow());
  4102. if(!pendingRight || (collate->docompare(left, pendingRight) < 0))
  4103. break;
  4104. }
  4105. else
  4106. break;
  4107. }
  4108. matchedRight.kill();
  4109. ForEachItemIn(idx, right)
  4110. matchedRight.append(false);
  4111. }
  4112. const void * CHThorJoinActivity::joinRecords(const void * curLeft, const void * curRight, unsigned counter, unsigned flags)
  4113. {
  4114. try
  4115. {
  4116. outBuilder.ensureRow();
  4117. size32_t thisSize = helper.transform(outBuilder, curLeft, curRight, counter, flags);
  4118. if(thisSize)
  4119. return outBuilder.finalizeRowClear(thisSize);
  4120. else
  4121. return NULL;
  4122. }
  4123. catch(IException * e)
  4124. {
  4125. throw makeWrappedException(e);
  4126. }
  4127. }
  4128. const void * CHThorJoinActivity::groupDenormalizeRecords(const void * curLeft, ConstPointerArray & rows, unsigned flags)
  4129. {
  4130. try
  4131. {
  4132. outBuilder.ensureRow();
  4133. unsigned numRows = rows.ordinality();
  4134. const void * rhs = numRows ? rows.item(0) : defaultRight.get();
  4135. if (numRows>0)
  4136. flags |= JTFmatchedright;
  4137. memsize_t thisSize = helper.transform(outBuilder, curLeft, rhs, numRows, (const void * *)rows.getArray(), flags);
  4138. if(thisSize)
  4139. return outBuilder.finalizeRowClear(thisSize);
  4140. else
  4141. return NULL;
  4142. }
  4143. catch(IException * e)
  4144. {
  4145. throw makeWrappedException(e);
  4146. }
  4147. }
  4148. const void * CHThorJoinActivity::joinException(const void * curLeft, IException * except)
  4149. {
  4150. try
  4151. {
  4152. outBuilder.ensureRow();
  4153. size32_t thisSize = helper.onFailTransform(outBuilder, curLeft, defaultRight, except, JTFmatchedleft);
  4154. if(thisSize)
  4155. return outBuilder.finalizeRowClear(thisSize);
  4156. else
  4157. return NULL;
  4158. }
  4159. catch(IException * e)
  4160. {
  4161. throw makeWrappedException(e);
  4162. }
  4163. }
  4164. void CHThorJoinActivity::failLimit()
  4165. {
  4166. helper.onMatchAbortLimitExceeded();
  4167. CommonXmlWriter xmlwrite(0);
  4168. if (input->queryOutputMeta() && input->queryOutputMeta()->hasXML())
  4169. {
  4170. input->queryOutputMeta()->toXML((byte *)left.get(), xmlwrite);
  4171. }
  4172. throw MakeStringException(0, "More than %d match candidates in join for row %s", abortLimit, xmlwrite.str());
  4173. }
  4174. const void *CHThorJoinActivity::nextRow()
  4175. {
  4176. for (;;)
  4177. {
  4178. switch (state)
  4179. {
  4180. case JSfill:
  4181. fillLeft();
  4182. state = JSfillright;
  4183. break;
  4184. case JSfillright:
  4185. fillRight();
  4186. state = JScollate;
  4187. break;
  4188. case JSfillleft:
  4189. fillLeft();
  4190. state = JScollate;
  4191. break;
  4192. case JScollate:
  4193. if (right.ordinality() == 0)
  4194. {
  4195. if (!left)
  4196. return NULL;
  4197. state = JSleftonly;
  4198. }
  4199. else
  4200. {
  4201. if (!left)
  4202. state = JSrightonly;
  4203. else
  4204. {
  4205. int diff;
  4206. if(betweenjoin)
  4207. diff = ((collate->docompare(left, right.item(0)) < 0) ? -1 : ((collateupper->docompare(left, right.item(right.ordinality()-1)) > 0) ? +1 : 0));
  4208. else
  4209. diff = collate->docompare(left, right.item(0));
  4210. bool limitExceeded = right.ordinality()>abortLimit;
  4211. if (diff == 0)
  4212. {
  4213. if (limitExceeded)
  4214. {
  4215. const void * ret = NULL;
  4216. if(failingLimit)
  4217. {
  4218. if ( agent.queryCodeContext()->queryDebugContext())
  4219. agent.queryCodeContext()->queryDebugContext()->checkBreakpoint(DebugStateLimit, NULL, static_cast<IActivityBase *>(this));
  4220. ret = joinException(left, failingLimit);
  4221. }
  4222. left.clear();
  4223. state = JSfillleft;
  4224. ForEachItemIn(idx, right)
  4225. matchedRight.replace(true, idx);
  4226. if(ret)
  4227. {
  4228. processed++;
  4229. return ret;
  4230. }
  4231. }
  4232. else
  4233. {
  4234. state = JScompare;
  4235. joinLimit = keepLimit;
  4236. }
  4237. }
  4238. else if (diff < 0)
  4239. state = JSleftonly;
  4240. else if (limitExceeded)
  4241. {
  4242. // MORE - Roxie code seems to think there should be a destroyRowset(right) here....
  4243. state = JSfillright;
  4244. }
  4245. else
  4246. state = JSrightonly;
  4247. }
  4248. }
  4249. break;
  4250. case JSrightonly:
  4251. if (rightOuterJoin)
  4252. {
  4253. switch (kind)
  4254. {
  4255. case TAKjoin:
  4256. {
  4257. while (right.isItem(rightIndex))
  4258. {
  4259. if (!matchedRight.item(rightIndex))
  4260. {
  4261. const void * rhs = right.item(rightIndex++);
  4262. const void * ret = joinRecords(defaultLeft, rhs, 0, JTFmatchedright);
  4263. if (ret)
  4264. {
  4265. processed++;
  4266. return ret;
  4267. }
  4268. }
  4269. else
  4270. rightIndex++;
  4271. }
  4272. break;
  4273. }
  4274. //Probably excessive to implement the following, but possibly useful
  4275. case TAKdenormalize:
  4276. {
  4277. OwnedConstRoxieRow newLeft(defaultLeft.getLink());
  4278. unsigned rowSize = 0;
  4279. unsigned leftCount = 0;
  4280. while (right.isItem(rightIndex))
  4281. {
  4282. if (!matchedRight.item(rightIndex))
  4283. {
  4284. const void * rhs = right.item(rightIndex);
  4285. try
  4286. {
  4287. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  4288. size32_t thisSize = helper.transform(rowBuilder, newLeft, rhs, ++leftCount, JTFmatchedright);
  4289. if (thisSize)
  4290. {
  4291. rowSize = thisSize;
  4292. newLeft.setown(rowBuilder.finalizeRowClear(rowSize));
  4293. }
  4294. }
  4295. catch(IException * e)
  4296. {
  4297. throw makeWrappedException(e);
  4298. }
  4299. }
  4300. rightIndex++;
  4301. }
  4302. state = JSfillright;
  4303. if (rowSize)
  4304. {
  4305. processed++;
  4306. return newLeft.getClear();
  4307. }
  4308. break;
  4309. }
  4310. case TAKdenormalizegroup:
  4311. {
  4312. filteredRight.kill();
  4313. while (right.isItem(rightIndex))
  4314. {
  4315. if (!matchedRight.item(rightIndex))
  4316. filteredRight.append(right.item(rightIndex));
  4317. rightIndex++;
  4318. }
  4319. state = JSfillright;
  4320. if (filteredRight.ordinality())
  4321. {
  4322. const void * ret = groupDenormalizeRecords(defaultLeft, filteredRight, 0);
  4323. filteredRight.kill();
  4324. if (ret)
  4325. {
  4326. processed++;
  4327. return ret;
  4328. }
  4329. }
  4330. break;
  4331. }
  4332. default:
  4333. throwUnexpected();
  4334. }
  4335. }
  4336. state = JSfillright;
  4337. break;
  4338. case JSleftonly:
  4339. {
  4340. const void * ret = NULL;
  4341. if (!matchedLeft && leftOuterJoin)
  4342. {
  4343. switch (kind)
  4344. {
  4345. case TAKjoin:
  4346. ret = joinRecords(left, defaultRight, 0, JTFmatchedleft);
  4347. break;
  4348. case TAKdenormalize:
  4349. ret = left.getClear();
  4350. break;
  4351. case TAKdenormalizegroup:
  4352. filteredRight.kill();
  4353. ret = groupDenormalizeRecords(left, filteredRight, JTFmatchedleft);
  4354. break;
  4355. default:
  4356. throwUnexpected();
  4357. }
  4358. }
  4359. left.clear();
  4360. state = JSfillleft;
  4361. if (ret)
  4362. {
  4363. processed++;
  4364. return ret;
  4365. }
  4366. break;
  4367. }
  4368. case JScompare:
  4369. if (joinLimit != 0)
  4370. {
  4371. switch (kind)
  4372. {
  4373. case TAKjoin:
  4374. {
  4375. while (right.isItem(rightIndex))
  4376. {
  4377. const void * rhs = right.item(rightIndex++);
  4378. if (helper.match(left, rhs))
  4379. {
  4380. matchedRight.replace(true, rightIndex-1);
  4381. matchedLeft = true;
  4382. if (!exclude)
  4383. {
  4384. const void *ret = joinRecords(left, rhs, ++joinCounter, JTFmatchedleft|JTFmatchedright);
  4385. if (ret)
  4386. {
  4387. processed++;
  4388. joinLimit--;
  4389. return ret;
  4390. }
  4391. }
  4392. }
  4393. }
  4394. break;
  4395. }
  4396. case TAKdenormalize:
  4397. {
  4398. OwnedConstRoxieRow newLeft;
  4399. newLeft.set(left);
  4400. unsigned rowSize = 0;
  4401. unsigned leftCount = 0;
  4402. while (right.isItem(rightIndex) && joinLimit)
  4403. {
  4404. const void * rhs = right.item(rightIndex++);
  4405. if (helper.match(left, rhs))
  4406. {
  4407. matchedRight.replace(true, rightIndex-1);
  4408. matchedLeft = true;
  4409. if (!exclude)
  4410. {
  4411. try
  4412. {
  4413. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  4414. unsigned thisSize = helper.transform(rowBuilder, newLeft, rhs, ++leftCount, JTFmatchedleft|JTFmatchedright);
  4415. if (thisSize)
  4416. {
  4417. rowSize = thisSize;
  4418. newLeft.setown(rowBuilder.finalizeRowClear(rowSize));
  4419. joinLimit--;
  4420. }
  4421. }
  4422. catch(IException * e)
  4423. {
  4424. throw makeWrappedException(e);
  4425. }
  4426. }
  4427. }
  4428. }
  4429. state = JSleftonly;
  4430. rightIndex = 0;
  4431. if (rowSize)
  4432. {
  4433. processed++;
  4434. return newLeft.getClear();
  4435. }
  4436. break;
  4437. }
  4438. case TAKdenormalizegroup:
  4439. {
  4440. filteredRight.kill();
  4441. while (right.isItem(rightIndex))
  4442. {
  4443. const void * rhs = right.item(rightIndex++);
  4444. if (helper.match(left, rhs))
  4445. {
  4446. matchedRight.replace(true, rightIndex-1);
  4447. filteredRight.append(rhs);
  4448. matchedLeft = true;
  4449. if (filteredRight.ordinality()==joinLimit)
  4450. break;
  4451. }
  4452. }
  4453. state = JSleftonly;
  4454. rightIndex = 0;
  4455. if (!exclude && filteredRight.ordinality())
  4456. {
  4457. const void * ret = groupDenormalizeRecords(left, filteredRight, JTFmatchedleft);
  4458. filteredRight.kill();
  4459. if (ret)
  4460. {
  4461. processed++;
  4462. return ret;
  4463. }
  4464. }
  4465. break;
  4466. }
  4467. default:
  4468. throwUnexpected();
  4469. }
  4470. }
  4471. state = JSleftonly;
  4472. rightIndex = 0;
  4473. joinCounter = 0;
  4474. break;
  4475. }
  4476. }
  4477. }
  4478. bool CHThorJoinActivity::isGrouped()
  4479. {
  4480. return false;
  4481. }
  4482. //=====================================================================================================
  4483. CHThorSelfJoinActivity::CHThorSelfJoinActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorJoinArg &_arg, ThorActivityKind _kind)
  4484. : CHThorActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg), outBuilder(NULL)
  4485. {
  4486. dualCacheInput = NULL;
  4487. }
  4488. void CHThorSelfJoinActivity::ready()
  4489. {
  4490. CHThorActivityBase::ready();
  4491. outBuilder.setAllocator(rowAllocator);
  4492. ICompare *compareLeft = helper.queryCompareLeft();
  4493. if (helper.isLeftAlreadySorted())
  4494. groupedInput.setown(createGroupedInputReader(&input->queryStream(), compareLeft));
  4495. else
  4496. {
  4497. bool isStable = (helper.getJoinFlags() & JFunstable) == 0;
  4498. RoxieSortAlgorithm sortAlgorithm = isStable ? stableSpillingQuickSortAlgorithm : spillingQuickSortAlgorithm;
  4499. StringBuffer tempBase;
  4500. agent.getTempfileBase(tempBase);
  4501. groupedInput.setown(createSortedGroupedInputReader(&input->queryStream(), compareLeft, createSortAlgorithm(sortAlgorithm, compareLeft, *queryRowManager(), input->queryOutputMeta(), agent.queryCodeContext(), tempBase, activityId)));
  4502. }
  4503. leftOuterJoin = (helper.getJoinFlags() & JFleftouter) != 0;
  4504. rightOuterJoin = (helper.getJoinFlags() & JFrightouter) != 0;
  4505. exclude = (helper.getJoinFlags() & JFexclude) != 0;
  4506. getLimitType(helper.getJoinFlags(), limitFail, limitOnFail);
  4507. if (rightOuterJoin && !defaultLeft)
  4508. {
  4509. if (!defaultAllocator)
  4510. defaultAllocator.setown(agent.queryCodeContext()->getRowAllocator(input->queryOutputMeta(), activityId));
  4511. RtlDynamicRowBuilder rowBuilder(defaultAllocator);
  4512. size32_t thisSize = helper.createDefaultLeft(rowBuilder);
  4513. defaultLeft.setown(rowBuilder.finalizeRowClear(thisSize));
  4514. }
  4515. if ((leftOuterJoin || limitOnFail) && !defaultRight)
  4516. {
  4517. if (!defaultAllocator)
  4518. defaultAllocator.setown(agent.queryCodeContext()->getRowAllocator(input->queryOutputMeta(), activityId));
  4519. RtlDynamicRowBuilder rowBuilder(defaultAllocator);
  4520. size32_t thisSize = helper.createDefaultRight(rowBuilder);
  4521. defaultRight.setown(rowBuilder.finalizeRowClear(thisSize));
  4522. }
  4523. if((helper.getJoinFlags() & JFslidingmatch) != 0)
  4524. throw MakeStringException(99, "Sliding self join not supported");
  4525. keepLimit = helper.getKeepLimit();
  4526. if(keepLimit == 0)
  4527. keepLimit = (unsigned)-1;
  4528. atmostLimit = helper.getJoinLimit();
  4529. if(atmostLimit == 0)
  4530. atmostLimit = (unsigned)-1;
  4531. else
  4532. assertex(!rightOuterJoin);
  4533. abortLimit = helper.getMatchAbortLimit();
  4534. if (abortLimit == 0)
  4535. abortLimit = (unsigned)-1;
  4536. assertex((helper.getJoinFlags() & (JFfirst | JFfirstleft | JFfirstright)) == 0); // no longer supported
  4537. collate = helper.queryCompareLeftRight();
  4538. eof = false;
  4539. doneFirstFill = false;
  4540. failingLimit.clear();
  4541. if ((helper.getJoinFlags() & JFlimitedprefixjoin) && helper.getJoinLimit())
  4542. { //Limited Match Join (s[1..n])
  4543. dualcache.setown(new CRHDualCache());
  4544. dualcache->init(groupedInput);
  4545. dualCacheInput = dualcache->queryOut1();
  4546. failingOuterAtmost = false;
  4547. matchedLeft = false;
  4548. leftIndex = 0;
  4549. rightOuterIndex = 0;
  4550. limitedhelper.setown(createRHLimitedCompareHelper());
  4551. limitedhelper->init( helper.getJoinLimit(), dualcache->queryOut2(), collate, helper.queryPrefixCompare() );
  4552. }
  4553. joinCounter = 0;
  4554. }
  4555. void CHThorSelfJoinActivity::stop()
  4556. {
  4557. outBuilder.clear();
  4558. group.clear();
  4559. groupedInput.clear();
  4560. CHThorActivityBase::stop();
  4561. }
  4562. bool CHThorSelfJoinActivity::fillGroup()
  4563. {
  4564. group.clear();
  4565. matchedLeft = false;
  4566. matchedRight.kill();
  4567. failingOuterAtmost = false;
  4568. OwnedConstRoxieRow next;
  4569. unsigned groupCount = 0;
  4570. next.setown(groupedInput->nextRow());
  4571. while(next)
  4572. {
  4573. if(groupCount==abortLimit)
  4574. {
  4575. if(limitFail)
  4576. failLimit(next);
  4577. if ( agent.queryCodeContext()->queryDebugContext())
  4578. agent.queryCodeContext()->queryDebugContext()->checkBreakpoint(DebugStateLimit, NULL, static_cast<IActivityBase *>(this));
  4579. if(limitOnFail)
  4580. {
  4581. assertex(!failingLimit);
  4582. try
  4583. {
  4584. failLimit(next);
  4585. }
  4586. catch(IException * except)
  4587. {
  4588. failingLimit.setown(except);
  4589. }
  4590. assertex(failingLimit);
  4591. group.append(next.getClear());
  4592. groupCount++;
  4593. break;
  4594. }
  4595. group.clear();
  4596. groupCount = 0;
  4597. while(next)
  4598. next.setown(groupedInput->nextRow());
  4599. }
  4600. else if(groupCount==atmostLimit)
  4601. {
  4602. if(leftOuterJoin)
  4603. {
  4604. group.append(next.getClear());
  4605. groupCount++;
  4606. failingOuterAtmost = true;
  4607. break;
  4608. }
  4609. else
  4610. {
  4611. group.clear();
  4612. groupCount = 0;
  4613. while(next)
  4614. next.setown(groupedInput->nextRow());
  4615. }
  4616. }
  4617. else
  4618. {
  4619. group.append(next.getClear());
  4620. groupCount++;
  4621. }
  4622. next.setown(groupedInput->nextRow());
  4623. }
  4624. if(group.ordinality()==0)
  4625. {
  4626. eof = true;
  4627. return false;
  4628. }
  4629. leftIndex = 0;
  4630. rightIndex = 0;
  4631. joinCounter = 0;
  4632. rightOuterIndex = 0;
  4633. joinLimit = keepLimit;
  4634. ForEachItemIn(idx, group)
  4635. matchedRight.append(false);
  4636. return true;
  4637. }
  4638. const void * CHThorSelfJoinActivity::nextRow()
  4639. {
  4640. if (limitedhelper) {
  4641. while(!eof) //limited match join
  4642. {
  4643. if (!group.isItem(rightIndex))
  4644. {
  4645. lhs.setown(dualCacheInput->nextRow());
  4646. if (lhs)
  4647. {
  4648. rightIndex = 0;
  4649. joinCounter = 0;
  4650. group.clear();
  4651. limitedhelper->getGroup(group,lhs);
  4652. }
  4653. else
  4654. eof = true;
  4655. }
  4656. if (group.isItem(rightIndex))
  4657. {
  4658. const void * rhs = group.item(rightIndex++);
  4659. if(helper.match(lhs, rhs))
  4660. {
  4661. const void * ret = joinRecords(lhs, rhs, ++joinCounter, JTFmatchedleft|JTFmatchedright, NULL);
  4662. if(ret)
  4663. {
  4664. processed++;
  4665. return ret;
  4666. }
  4667. }
  4668. }
  4669. }
  4670. return NULL;
  4671. }
  4672. if(!doneFirstFill)
  4673. {
  4674. fillGroup();
  4675. doneFirstFill = true;
  4676. }
  4677. while(!eof)
  4678. {
  4679. if(failingOuterAtmost)
  4680. while(group.isItem(leftIndex))
  4681. {
  4682. const void * ret = joinRecords(group.item(leftIndex++), defaultRight, 0, JTFmatchedleft, NULL);
  4683. if(ret)
  4684. {
  4685. processed++;
  4686. return ret;
  4687. }
  4688. }
  4689. if((joinLimit == 0) || !group.isItem(rightIndex))
  4690. {
  4691. if(leftOuterJoin && !matchedLeft && !failingLimit)
  4692. {
  4693. const void * ret = joinRecords(group.item(leftIndex), defaultRight, 0, JTFmatchedleft, NULL);
  4694. if(ret)
  4695. {
  4696. matchedLeft = true;
  4697. processed++;
  4698. return ret;
  4699. }
  4700. }
  4701. leftIndex++;
  4702. matchedLeft = false;
  4703. rightIndex = 0;
  4704. joinCounter = 0;
  4705. joinLimit = keepLimit;
  4706. }
  4707. if(!group.isItem(leftIndex))
  4708. {
  4709. if(failingLimit || failingOuterAtmost)
  4710. {
  4711. OwnedConstRoxieRow lhs(groupedInput->nextRow()); // dualCache never active here
  4712. while(lhs)
  4713. {
  4714. const void * ret = joinRecords(lhs, defaultRight, 0, JTFmatchedleft, failingLimit);
  4715. if(ret)
  4716. {
  4717. processed++;
  4718. return ret;
  4719. }
  4720. lhs.setown(groupedInput->nextRow());
  4721. }
  4722. failingLimit.clear();
  4723. }
  4724. if(rightOuterJoin && !failingLimit)
  4725. while(group.isItem(rightOuterIndex))
  4726. if(!matchedRight.item(rightOuterIndex++))
  4727. {
  4728. const void * ret = joinRecords(defaultLeft, group.item(rightOuterIndex-1), 0, JTFmatchedright, NULL);
  4729. if(ret)
  4730. {
  4731. processed++;
  4732. return ret;
  4733. }
  4734. }
  4735. if(!fillGroup())
  4736. return NULL;
  4737. continue;
  4738. }
  4739. const void * lhs = group.item(leftIndex);
  4740. if(failingLimit)
  4741. {
  4742. leftIndex++;
  4743. const void * ret = joinRecords(lhs, defaultRight, 0, JTFmatchedleft, failingLimit);
  4744. if(ret)
  4745. {
  4746. processed++;
  4747. return ret;
  4748. }
  4749. }
  4750. else
  4751. {
  4752. const void * rhs = group.item(rightIndex++);
  4753. if(helper.match(lhs, rhs))
  4754. {
  4755. matchedLeft = true;
  4756. matchedRight.replace(true, rightIndex-1);
  4757. if(!exclude)
  4758. {
  4759. const void * ret = joinRecords(lhs, rhs, ++joinCounter, JTFmatchedleft|JTFmatchedright, NULL);
  4760. if(ret)
  4761. {
  4762. processed++;
  4763. joinLimit--;
  4764. return ret;
  4765. }
  4766. }
  4767. }
  4768. }
  4769. }
  4770. return NULL;
  4771. }
  4772. const void * CHThorSelfJoinActivity::joinRecords(const void * curLeft, const void * curRight, unsigned counter, unsigned flags, IException * except)
  4773. {
  4774. outBuilder.ensureRow();
  4775. try
  4776. {
  4777. size32_t thisSize = (except ? helper.onFailTransform(outBuilder, curLeft, curRight, except, flags) : helper.transform(outBuilder, curLeft, curRight, counter, flags));
  4778. if(thisSize){
  4779. return outBuilder.finalizeRowClear(thisSize);
  4780. }
  4781. else
  4782. return NULL;
  4783. }
  4784. catch(IException * e)
  4785. {
  4786. throw makeWrappedException(e);
  4787. }
  4788. }
  4789. void CHThorSelfJoinActivity::failLimit(const void * next)
  4790. {
  4791. helper.onMatchAbortLimitExceeded();
  4792. CommonXmlWriter xmlwrite(0);
  4793. if (input->queryOutputMeta() && input->queryOutputMeta()->hasXML())
  4794. {
  4795. input->queryOutputMeta()->toXML((byte *) next, xmlwrite);
  4796. }
  4797. throw MakeStringException(0, "More than %d match candidates in self-join for row %s", abortLimit, xmlwrite.str());
  4798. }
  4799. bool CHThorSelfJoinActivity::isGrouped()
  4800. {
  4801. return false;
  4802. }
  4803. //=====================================================================================================
  4804. CHThorLookupJoinActivity::LookupTable::LookupTable(unsigned _size, ICompare * _leftRightCompare, ICompare * _rightCompare, IHash * _leftHash, IHash * _rightHash, bool _dedupOnAdd)
  4805. : leftRightCompare(_leftRightCompare), rightCompare(_rightCompare), leftHash(_leftHash), rightHash(_rightHash), dedupOnAdd(_dedupOnAdd)
  4806. {
  4807. unsigned minsize = (4*_size)/3;
  4808. size = 2;
  4809. while((minsize >>= 1) > 0)
  4810. size <<= 1;
  4811. mask = size - 1;
  4812. table = new OwnedConstRoxieRow[size];
  4813. findex = BadIndex;
  4814. }
  4815. CHThorLookupJoinActivity::LookupTable::~LookupTable()
  4816. {
  4817. delete [] table;
  4818. }
  4819. bool CHThorLookupJoinActivity::LookupTable::add(const void * _right)
  4820. {
  4821. OwnedConstRoxieRow right(_right);
  4822. findex = BadIndex;
  4823. unsigned start = rightHash->hash(right) & mask;
  4824. unsigned index = start;
  4825. while(table[index])
  4826. {
  4827. if(dedupOnAdd && (rightCompare->docompare(table[index], right) == 0))
  4828. return false;
  4829. index++;
  4830. if(index==size)
  4831. index = 0;
  4832. if(index==start)
  4833. return false; //table is full, should never happen
  4834. }
  4835. table[index].setown(right.getClear());
  4836. return true;
  4837. }
  4838. const void * CHThorLookupJoinActivity::LookupTable::find(const void * left) const
  4839. {
  4840. fstart = leftHash->hash(left) & mask;
  4841. findex = fstart;
  4842. return doFind(left);
  4843. }
  4844. const void * CHThorLookupJoinActivity::LookupTable::findNext(const void * left) const
  4845. {
  4846. if(findex == BadIndex)
  4847. return NULL;
  4848. advance();
  4849. return doFind(left);
  4850. }
  4851. void CHThorLookupJoinActivity::LookupTable::advance() const
  4852. {
  4853. findex++;
  4854. if(findex==size)
  4855. findex = 0;
  4856. if(findex==fstart)
  4857. throw MakeStringException(0, "Internal error hthor lookup join activity (hash table full on lookup)");
  4858. }
  4859. const void * CHThorLookupJoinActivity::LookupTable::doFind(const void * left) const
  4860. {
  4861. while(table[findex])
  4862. {
  4863. if(leftRightCompare->docompare(left, table[findex]) == 0)
  4864. return table[findex];
  4865. advance();
  4866. }
  4867. findex = BadIndex;
  4868. return NULL;
  4869. }
  4870. CHThorLookupJoinActivity::CHThorLookupJoinActivity(IAgentContext & _agent, unsigned _activityId, unsigned _subgraphId, IHThorHashJoinArg &_arg, ThorActivityKind _kind) : CHThorActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg), table(0), outBuilder(NULL)
  4871. {
  4872. }
  4873. void CHThorLookupJoinActivity::ready()
  4874. {
  4875. CHThorActivityBase::ready();
  4876. input1->ready();
  4877. outBuilder.setAllocator(rowAllocator);
  4878. leftOuterJoin = (helper.getJoinFlags() & JFleftouter) != 0;
  4879. assertex((helper.getJoinFlags() & JFrightouter) == 0);
  4880. exclude = (helper.getJoinFlags() & JFexclude) != 0;
  4881. many = (helper.getJoinFlags() & JFmanylookup) != 0;
  4882. dedupRHS = (helper.getJoinFlags() & (JFmanylookup | JFmatchrequired | JFtransformMaySkip)) == 0; // optimisation: can implicitly dedup RHS unless is many lookup, or match required, or transform may skip
  4883. if((helper.getJoinFlags() & (JFfirst | JFfirstleft | JFfirstright | JFslidingmatch)) != 0)
  4884. throwUnexpected(); // compiler should have rejected
  4885. keepLimit = helper.getKeepLimit();
  4886. if(keepLimit==0)
  4887. keepLimit = static_cast<unsigned>(-1);
  4888. atmostLimit = helper.getJoinLimit();
  4889. limitLimit = helper.getMatchAbortLimit();
  4890. hasGroupLimit = ((atmostLimit > 0) || (limitLimit > 0));
  4891. if(atmostLimit==0)
  4892. atmostLimit = static_cast<unsigned>(-1);
  4893. if(limitLimit==0)
  4894. limitLimit = static_cast<unsigned>(-1);
  4895. isSmartJoin = (helper.getJoinFlags() & JFsmart) != 0;
  4896. getLimitType(helper.getJoinFlags(), limitFail, limitOnFail);
  4897. if((leftOuterJoin || limitOnFail) && !defaultRight)
  4898. createDefaultRight();
  4899. eog = false;
  4900. matchedGroup = false;
  4901. joinCounter = 0;
  4902. }
  4903. void CHThorLookupJoinActivity::stop()
  4904. {
  4905. outBuilder.clear();
  4906. left.clear();
  4907. table.clear();
  4908. CHThorActivityBase::stop();
  4909. input1->stop();
  4910. }
  4911. void CHThorLookupJoinActivity::createDefaultRight()
  4912. {
  4913. if (!defaultRight)
  4914. {
  4915. if (!defaultRightAllocator)
  4916. defaultRightAllocator.setown(agent.queryCodeContext()->getRowAllocator(input1->queryOutputMeta(), activityId));
  4917. RtlDynamicRowBuilder rowBuilder(defaultRightAllocator);
  4918. size32_t thisSize = helper.createDefaultRight(rowBuilder);
  4919. defaultRight.setown(rowBuilder.finalizeRowClear(thisSize));
  4920. }
  4921. }
  4922. void CHThorLookupJoinActivity::loadRight()
  4923. {
  4924. OwnedRowArray rightset;
  4925. const void * next;
  4926. while(true)
  4927. {
  4928. next = input1->nextRow();
  4929. if(!next)
  4930. next = input1->nextRow();
  4931. if(!next)
  4932. break;
  4933. rightset.append(next);
  4934. }
  4935. unsigned rightord = rightset.ordinality();
  4936. table.setown(new LookupTable(rightord, helper.queryCompareLeftRight(), helper.queryCompareRight(), helper.queryHashLeft(), helper.queryHashRight(), dedupRHS));
  4937. unsigned i;
  4938. for(i=0; i<rightord; i++)
  4939. table->add(rightset.itemClear(i));
  4940. };
  4941. void CHThorLookupJoinActivity::setInput(unsigned index, IHThorInput * _input)
  4942. {
  4943. if (index==1)
  4944. input1 = _input;
  4945. else
  4946. CHThorActivityBase::setInput(index, _input);
  4947. }
  4948. //following are all copied from CHThorJoinActivity - should common up.
  4949. const void * CHThorLookupJoinActivity::joinRecords(const void * left, const void * right, unsigned counter, unsigned flags)
  4950. {
  4951. try
  4952. {
  4953. outBuilder.ensureRow();
  4954. size32_t thisSize = helper.transform(outBuilder, left, right, counter, flags);
  4955. if(thisSize)
  4956. return outBuilder.finalizeRowClear(thisSize);
  4957. else
  4958. return NULL;
  4959. }
  4960. catch(IException * e)
  4961. {
  4962. throw makeWrappedException(e);
  4963. }
  4964. }
  4965. const void * CHThorLookupJoinActivity::joinException(const void * left, IException * except)
  4966. {
  4967. try
  4968. {
  4969. outBuilder.ensureRow();
  4970. memsize_t thisSize = helper.onFailTransform(outBuilder, left, defaultRight, except, JTFmatchedleft);
  4971. if(thisSize)
  4972. return outBuilder.finalizeRowClear(thisSize);
  4973. else
  4974. return NULL;
  4975. }
  4976. catch(IException * e)
  4977. {
  4978. throw makeWrappedException(e);
  4979. }
  4980. }
  4981. const void * CHThorLookupJoinActivity::groupDenormalizeRecords(const void * left, ConstPointerArray & rows, unsigned flags)
  4982. {
  4983. try
  4984. {
  4985. outBuilder.ensureRow();
  4986. unsigned numRows = rows.ordinality();
  4987. const void * right = numRows ? rows.item(0) : defaultRight.get();
  4988. if (numRows>0)
  4989. flags |= JTFmatchedright;
  4990. memsize_t thisSize = helper.transform(outBuilder, left, right, numRows, (const void * *)rows.getArray(), flags);
  4991. if(thisSize)
  4992. return outBuilder.finalizeRowClear(thisSize);
  4993. else
  4994. return NULL;
  4995. }
  4996. catch(IException * e)
  4997. {
  4998. throw makeWrappedException(e);
  4999. }
  5000. }
  5001. const void * CHThorLookupJoinActivity::nextRow()
  5002. {
  5003. if(!table)
  5004. loadRight();
  5005. switch (kind)
  5006. {
  5007. case TAKlookupjoin:
  5008. case TAKsmartjoin:
  5009. return nextRowJoin();
  5010. case TAKlookupdenormalize:
  5011. case TAKlookupdenormalizegroup:
  5012. case TAKsmartdenormalize:
  5013. case TAKsmartdenormalizegroup:
  5014. return nextRowDenormalize();
  5015. }
  5016. throwUnexpected();
  5017. }
  5018. const void * CHThorLookupJoinActivity::nextRowJoin()
  5019. {
  5020. while(true)
  5021. {
  5022. const void * right = NULL;
  5023. if(!left)
  5024. {
  5025. left.setown(input->nextRow());
  5026. keepCount = keepLimit;
  5027. if(!left)
  5028. {
  5029. if (isSmartJoin)
  5030. left.setown(input->nextRow());
  5031. if(!left)
  5032. {
  5033. if(matchedGroup || eog)
  5034. {
  5035. matchedGroup = false;
  5036. eog = true;
  5037. return NULL;
  5038. }
  5039. eog = true;
  5040. continue;
  5041. }
  5042. }
  5043. eog = false;
  5044. gotMatch = false;
  5045. right = getRightFirst();
  5046. }
  5047. else
  5048. right = getRightNext();
  5049. const void * ret = NULL;
  5050. if(failingLimit)
  5051. {
  5052. ret = joinException(left, failingLimit);
  5053. }
  5054. else
  5055. {
  5056. while(right)
  5057. {
  5058. if(helper.match(left, right))
  5059. {
  5060. gotMatch = true;
  5061. if(exclude)
  5062. break;
  5063. ret = joinRecords(left, right, ++joinCounter, JTFmatchedleft|JTFmatchedright);
  5064. if(ret)
  5065. {
  5066. processed++;
  5067. break;
  5068. }
  5069. }
  5070. right = getRightNext();
  5071. ret = NULL;
  5072. }
  5073. if(leftOuterJoin && !gotMatch)
  5074. {
  5075. ret = joinRecords(left, defaultRight, 0, JTFmatchedleft);
  5076. gotMatch = true;
  5077. }
  5078. }
  5079. if(ret)
  5080. {
  5081. matchedGroup = true;
  5082. processed++;
  5083. if(!many || (--keepCount == 0) || failingLimit)
  5084. {
  5085. left.clear();
  5086. joinCounter = 0;
  5087. failingLimit.clear();
  5088. }
  5089. return ret;
  5090. }
  5091. left.clear();
  5092. joinCounter = 0;
  5093. }
  5094. }
  5095. const void * CHThorLookupJoinActivity::nextRowDenormalize()
  5096. {
  5097. while(true)
  5098. {
  5099. left.setown(input->nextRow());
  5100. if(!left)
  5101. {
  5102. if (!matchedGroup || isSmartJoin)
  5103. left.setown(input->nextRow());
  5104. if (!left)
  5105. {
  5106. matchedGroup = false;
  5107. return NULL;
  5108. }
  5109. }
  5110. gotMatch = false;
  5111. const void * right = getRightFirst();
  5112. const void * ret = NULL;
  5113. if (failingLimit)
  5114. ret = joinException(left, failingLimit);
  5115. else if (kind == TAKlookupdenormalize || kind == TAKsmartdenormalize)
  5116. {
  5117. OwnedConstRoxieRow newLeft(left.getLink());
  5118. unsigned rowSize = 0;
  5119. unsigned leftCount = 0;
  5120. keepCount = keepLimit;
  5121. while (right)
  5122. {
  5123. if (helper.match(left, right))
  5124. {
  5125. gotMatch = true;
  5126. if (exclude)
  5127. break;
  5128. try
  5129. {
  5130. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  5131. unsigned thisSize = helper.transform(rowBuilder, newLeft, right, ++leftCount, JTFmatchedleft|JTFmatchedright);
  5132. if (thisSize)
  5133. {
  5134. rowSize = thisSize;
  5135. newLeft.setown(rowBuilder.finalizeRowClear(rowSize));
  5136. }
  5137. }
  5138. catch(IException * e)
  5139. {
  5140. throw makeWrappedException(e);
  5141. }
  5142. if(!many || (--keepCount == 0))
  5143. break;
  5144. }
  5145. right = getRightNext();
  5146. }
  5147. //Is this rowSize test correct?? Is there any situation where it shouldn't just return newLeft?
  5148. if (rowSize)
  5149. ret = newLeft.getClear();
  5150. else if (leftOuterJoin && !gotMatch)
  5151. ret = left.getClear();
  5152. }
  5153. else
  5154. {
  5155. filteredRight.kill();
  5156. keepCount = keepLimit;
  5157. while (right)
  5158. {
  5159. if (helper.match(left, right))
  5160. {
  5161. gotMatch = true;
  5162. if(exclude)
  5163. break;
  5164. filteredRight.append(right);
  5165. if(!many || (--keepCount == 0))
  5166. break;
  5167. }
  5168. right = getRightNext();
  5169. }
  5170. if((filteredRight.ordinality() > 0) || (leftOuterJoin && !gotMatch))
  5171. ret = groupDenormalizeRecords(left, filteredRight, JTFmatchedleft);
  5172. filteredRight.kill();
  5173. }
  5174. left.clear();
  5175. failingLimit.clear();
  5176. if(ret)
  5177. {
  5178. matchedGroup = true;
  5179. processed++;
  5180. return ret;
  5181. }
  5182. }
  5183. }
  5184. bool CHThorLookupJoinActivity::isGrouped()
  5185. {
  5186. return input ? input->isGrouped() : false;
  5187. }
  5188. const void * CHThorLookupJoinActivity::fillRightGroup()
  5189. {
  5190. rightGroup.kill();
  5191. for(const void * right = table->find(left); right; right = table->findNext(left))
  5192. {
  5193. rightGroup.append(right);
  5194. if(rightGroup.ordinality() > limitLimit)
  5195. {
  5196. if(limitFail)
  5197. failLimit();
  5198. if ( agent.queryCodeContext()->queryDebugContext())
  5199. agent.queryCodeContext()->queryDebugContext()->checkBreakpoint(DebugStateLimit, NULL, static_cast<IActivityBase *>(this));
  5200. gotMatch = true;
  5201. if(limitOnFail)
  5202. {
  5203. assertex(!failingLimit);
  5204. try
  5205. {
  5206. failLimit();
  5207. }
  5208. catch(IException * e)
  5209. {
  5210. failingLimit.setown(e);
  5211. }
  5212. assertex(failingLimit);
  5213. }
  5214. else
  5215. {
  5216. rightGroup.kill();
  5217. }
  5218. break;
  5219. }
  5220. if(rightGroup.ordinality() > atmostLimit)
  5221. {
  5222. rightGroup.kill();
  5223. break;
  5224. }
  5225. }
  5226. rightGroupIndex = 0;
  5227. return readRightGroup();
  5228. }
  5229. void CHThorLookupJoinActivity::failLimit()
  5230. {
  5231. helper.onMatchAbortLimitExceeded();
  5232. CommonXmlWriter xmlwrite(0);
  5233. if(input->queryOutputMeta() && input->queryOutputMeta()->hasXML())
  5234. {
  5235. input->queryOutputMeta()->toXML(static_cast<const unsigned char *>(left.get()), xmlwrite);
  5236. }
  5237. throw MakeStringException(0, "More than %u match candidates in join for row %s", limitLimit, xmlwrite.str());
  5238. }
  5239. unsigned const CHThorLookupJoinActivity::LookupTable::BadIndex(static_cast<unsigned>(-1));
  5240. //=====================================================================================================
  5241. CHThorAllJoinActivity::CHThorAllJoinActivity(IAgentContext & _agent, unsigned _activityId, unsigned _subgraphId, IHThorAllJoinArg &_arg, ThorActivityKind _kind) : CHThorActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg), outBuilder(NULL)
  5242. {
  5243. }
  5244. void CHThorAllJoinActivity::ready()
  5245. {
  5246. CHThorActivityBase::ready();
  5247. input1->ready();
  5248. outBuilder.setAllocator(rowAllocator);
  5249. leftOuterJoin = (helper.getJoinFlags() & JFleftouter) != 0;
  5250. exclude = (helper.getJoinFlags() & JFexclude) != 0;
  5251. if(leftOuterJoin && !defaultRight)
  5252. createDefaultRight();
  5253. if((helper.getJoinFlags() & (JFrightouter | JFfirst | JFfirstleft | JFfirstright)) != 0)
  5254. throwUnexpected();
  5255. keepLimit = helper.getKeepLimit();
  5256. if(keepLimit==0)
  5257. keepLimit = (unsigned)-1;
  5258. started = false;
  5259. countForLeft = keepLimit;
  5260. matchedLeft = false;
  5261. matchedGroup = false;
  5262. eog = false;
  5263. eos = false;
  5264. }
  5265. void CHThorAllJoinActivity::stop()
  5266. {
  5267. outBuilder.clear();
  5268. left.clear();
  5269. rightset.clear();
  5270. matchedRight.kill();
  5271. CHThorActivityBase::stop();
  5272. input1->stop();
  5273. }
  5274. void CHThorAllJoinActivity::createDefaultRight()
  5275. {
  5276. if (!defaultRight)
  5277. {
  5278. if (!defaultRightAllocator)
  5279. defaultRightAllocator.setown(agent.queryCodeContext()->getRowAllocator(input1->queryOutputMeta(), activityId));
  5280. RtlDynamicRowBuilder rowBuilder(defaultRightAllocator);
  5281. size32_t thisSize = helper.createDefaultRight(rowBuilder);
  5282. defaultRight.setown(rowBuilder.finalizeRowClear(thisSize));
  5283. }
  5284. }
  5285. void CHThorAllJoinActivity::loadRight()
  5286. {
  5287. const void * next;
  5288. while(true)
  5289. {
  5290. next = input1->nextRow();
  5291. if(!next)
  5292. next = input1->nextRow();
  5293. if(!next)
  5294. break;
  5295. rightset.append(next);
  5296. matchedRight.append(false);
  5297. }
  5298. rightIndex = 0;
  5299. joinCounter = 0;
  5300. rightOrdinality = rightset.ordinality();
  5301. }
  5302. const void * CHThorAllJoinActivity::joinRecords(const void * left, const void * right, unsigned counter, unsigned flags)
  5303. {
  5304. try
  5305. {
  5306. outBuilder.ensureRow();
  5307. memsize_t thisSize = helper.transform(outBuilder, left, right, counter, flags);
  5308. if(thisSize)
  5309. return outBuilder.finalizeRowClear(thisSize);
  5310. else
  5311. return NULL;
  5312. }
  5313. catch(IException * e)
  5314. {
  5315. throw makeWrappedException(e);
  5316. }
  5317. }
  5318. const void * CHThorAllJoinActivity::groupDenormalizeRecords(const void * curLeft, ConstPointerArray & rows, unsigned flags)
  5319. {
  5320. try
  5321. {
  5322. outBuilder.ensureRow();
  5323. unsigned numRows = rows.ordinality();
  5324. const void * right = numRows ? rows.item(0) : defaultRight.get();
  5325. if (numRows>0)
  5326. flags |= JTFmatchedright;
  5327. memsize_t thisSize = helper.transform(outBuilder, curLeft, right, numRows, (const void * *)rows.getArray(), flags);
  5328. if(thisSize)
  5329. return outBuilder.finalizeRowClear(thisSize);
  5330. else
  5331. return NULL;
  5332. }
  5333. catch(IException * e)
  5334. {
  5335. throw makeWrappedException(e);
  5336. }
  5337. }
  5338. void CHThorAllJoinActivity::setInput(unsigned index, IHThorInput * _input)
  5339. {
  5340. if (index==1)
  5341. input1 = _input;
  5342. else
  5343. {
  5344. CHThorActivityBase::setInput(index, _input);
  5345. leftIsGrouped = true; // input->isGrouped() is unreliable and it is just as good to always behave as if input is grouped
  5346. }
  5347. }
  5348. const void * CHThorAllJoinActivity::nextRow()
  5349. {
  5350. if(!started)
  5351. {
  5352. started = true;
  5353. left.setown(input->nextRow());
  5354. matchedLeft = false;
  5355. countForLeft = keepLimit;
  5356. if(!left)
  5357. {
  5358. eos = true;
  5359. return NULL;
  5360. }
  5361. loadRight();
  5362. }
  5363. const void * ret;
  5364. const void * right;
  5365. if(eos)
  5366. return NULL;
  5367. while(true)
  5368. {
  5369. ret = NULL;
  5370. if((rightIndex == rightOrdinality) || (countForLeft==0))
  5371. {
  5372. if(leftOuterJoin && left && !matchedLeft)
  5373. {
  5374. switch(kind)
  5375. {
  5376. case TAKalljoin:
  5377. ret = joinRecords(left, defaultRight, 0, JTFmatchedleft);
  5378. break;
  5379. case TAKalldenormalize:
  5380. ret = left.getClear();
  5381. break;
  5382. case TAKalldenormalizegroup:
  5383. filteredRight.kill();
  5384. ret = groupDenormalizeRecords(left, filteredRight, JTFmatchedleft);
  5385. break;
  5386. default:
  5387. throwUnexpected();
  5388. }
  5389. }
  5390. rightIndex = 0;
  5391. joinCounter = 0;
  5392. left.clear();
  5393. if(ret)
  5394. {
  5395. matchedGroup = true;
  5396. processed++;
  5397. return ret;
  5398. }
  5399. }
  5400. if(!left)
  5401. {
  5402. left.setown(input->nextRow());
  5403. matchedLeft = false;
  5404. countForLeft = keepLimit;
  5405. }
  5406. if(!left)
  5407. {
  5408. if(eog)
  5409. {
  5410. eos = true;
  5411. matchedGroup = false;
  5412. return NULL;
  5413. }
  5414. eog = true;
  5415. if(matchedGroup && leftIsGrouped)
  5416. {
  5417. matchedGroup = false;
  5418. return NULL;
  5419. }
  5420. matchedGroup = false;
  5421. continue;
  5422. }
  5423. eog = false;
  5424. switch(kind)
  5425. {
  5426. case TAKalljoin:
  5427. while(rightIndex < rightOrdinality)
  5428. {
  5429. right = rightset.item(rightIndex);
  5430. if(helper.match(left, right))
  5431. {
  5432. matchedLeft = true;
  5433. matchedRight.replace(true, rightIndex);
  5434. if(!exclude)
  5435. ret = joinRecords(left, right, ++joinCounter, JTFmatchedleft|JTFmatchedright);
  5436. }
  5437. rightIndex++;
  5438. if(ret)
  5439. {
  5440. countForLeft--;
  5441. matchedGroup = true;
  5442. processed++;
  5443. return ret;
  5444. }
  5445. }
  5446. case TAKalldenormalize:
  5447. {
  5448. OwnedConstRoxieRow newLeft;
  5449. newLeft.set(left);
  5450. unsigned rowSize = 0;
  5451. unsigned leftCount = 0;
  5452. while((rightIndex < rightOrdinality) && countForLeft)
  5453. {
  5454. right = rightset.item(rightIndex);
  5455. if(helper.match(left, right))
  5456. {
  5457. matchedLeft = true;
  5458. matchedRight.replace(true, rightIndex);
  5459. if(!exclude)
  5460. {
  5461. try
  5462. {
  5463. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  5464. unsigned thisSize = helper.transform(rowBuilder, newLeft, right, ++leftCount, JTFmatchedleft|JTFmatchedright);
  5465. if(thisSize)
  5466. {
  5467. rowSize = thisSize;
  5468. newLeft.setown(rowBuilder.finalizeRowClear(rowSize));
  5469. --countForLeft;
  5470. }
  5471. }
  5472. catch(IException * e)
  5473. {
  5474. throw makeWrappedException(e);
  5475. }
  5476. }
  5477. }
  5478. rightIndex++;
  5479. }
  5480. if(rowSize)
  5481. {
  5482. processed++;
  5483. return newLeft.getClear();
  5484. }
  5485. }
  5486. break;
  5487. case TAKalldenormalizegroup:
  5488. filteredRight.kill();
  5489. while((rightIndex < rightOrdinality) && countForLeft)
  5490. {
  5491. right = rightset.item(rightIndex);
  5492. if(helper.match(left, right))
  5493. {
  5494. matchedLeft = true;
  5495. matchedRight.replace(true, rightIndex);
  5496. filteredRight.append(right);
  5497. --countForLeft;
  5498. }
  5499. ++rightIndex;
  5500. }
  5501. if(!exclude && filteredRight.ordinality())
  5502. {
  5503. const void * ret = groupDenormalizeRecords(left, filteredRight, JTFmatchedleft);
  5504. filteredRight.kill();
  5505. if(ret)
  5506. {
  5507. processed++;
  5508. return ret;
  5509. }
  5510. }
  5511. break;
  5512. default:
  5513. throwUnexpected();
  5514. }
  5515. }
  5516. }
  5517. bool CHThorAllJoinActivity::isGrouped()
  5518. {
  5519. return input ? input->isGrouped() : false;
  5520. }
  5521. //=====================================================================================================
  5522. //=====================================================================================================
  5523. CHThorWorkUnitWriteActivity::CHThorWorkUnitWriteActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorWorkUnitWriteArg &_arg, ThorActivityKind _kind)
  5524. : CHThorActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  5525. {
  5526. }
  5527. void CHThorWorkUnitWriteActivity::execute()
  5528. {
  5529. unsigned flags = helper.getFlags();
  5530. grouped = (POFgrouped & flags) != 0;
  5531. // In absense of OPT_OUTPUTLIMIT check pre 5.2 legacy name OPT_OUTPUTLIMIT_LEGACY
  5532. size32_t outputLimit = agent.queryWorkUnit()->getDebugValueInt(OPT_OUTPUTLIMIT, agent.queryWorkUnit()->getDebugValueInt(OPT_OUTPUTLIMIT_LEGACY, defaultDaliResultLimit));
  5533. if (flags & POFmaxsize)
  5534. outputLimit = helper.getMaxSize();
  5535. if (outputLimit>defaultDaliResultOutputMax)
  5536. throw MakeStringException(0, "Dali result outputs are restricted to a maximum of %d MB, the current limit is %d MB. A huge dali result usually indicates the ECL needs altering.", defaultDaliResultOutputMax, defaultDaliResultLimit);
  5537. assertex(outputLimit<=0x1000); // 32bit limit because MemoryBuffer/CMessageBuffers involved etc.
  5538. outputLimit *= 0x100000;
  5539. MemoryBuffer rowdata;
  5540. __int64 rows = 0;
  5541. IRecordSize * inputMeta = input->queryOutputMeta();
  5542. if (0 != (POFextend & helper.getFlags()))
  5543. {
  5544. WorkunitUpdate w = agent.updateWorkUnit();
  5545. Owned<IWUResult> result = updateWorkUnitResult(w, helper.queryName(), helper.getSequence());
  5546. rows = result->getResultRowCount();
  5547. }
  5548. __int64 initialRows = rows;
  5549. Owned<IOutputRowSerializer> rowSerializer;
  5550. if (input->queryOutputMeta()->getMetaFlags() & MDFneedserializedisk)
  5551. rowSerializer.setown( input->queryOutputMeta()->createDiskSerializer(agent.queryCodeContext(), activityId) );
  5552. int seq = helper.getSequence();
  5553. bool toStdout = (seq >= 0) && agent.queryWriteResultsToStdout();
  5554. Owned<SimpleOutputWriter> writer;
  5555. if (toStdout)
  5556. writer.setown(new SimpleOutputWriter);
  5557. if (agent.queryOutputFmt() == ofXML && seq >= 0)
  5558. {
  5559. StringBuffer sb;
  5560. const char *name = helper.queryName();
  5561. if (name && *name)
  5562. sb.appendf("<Dataset name='%s'>\n", name);
  5563. else
  5564. sb.appendf("<Dataset name='Result %d'>\n", seq+1);
  5565. agent.queryOutputSerializer()->fwrite(seq, (const void*)sb.str(), 1, sb.length());
  5566. }
  5567. for (;;)
  5568. {
  5569. if ((unsigned __int64)rows >= agent.queryStopAfter())
  5570. break;
  5571. OwnedConstRoxieRow nextrec(input->nextRow());
  5572. if (grouped && (rows != initialRows))
  5573. rowdata.append(nextrec == NULL);
  5574. if (!nextrec)
  5575. {
  5576. nextrec.setown(input->nextRow());
  5577. if (!nextrec)
  5578. break;
  5579. }
  5580. size32_t thisSize = inputMeta->getRecordSize(nextrec);
  5581. if(outputLimit && ((rowdata.length() + thisSize) > outputLimit))
  5582. {
  5583. StringBuffer errMsg("Dataset too large to output to workunit (limit ");
  5584. errMsg.append(outputLimit/0x100000).append(" megabytes), in result (");
  5585. const char *name = helper.queryName();
  5586. if (name)
  5587. errMsg.append("name=").append(name);
  5588. else
  5589. errMsg.append("sequence=").append(helper.getSequence());
  5590. errMsg.append(")");
  5591. throw MakeStringExceptionDirect(0, errMsg.str());
  5592. }
  5593. if (rowSerializer)
  5594. {
  5595. CThorDemoRowSerializer serializerTarget(rowdata);
  5596. rowSerializer->serialize(serializerTarget, (const byte *) nextrec.get() );
  5597. }
  5598. else
  5599. rowdata.append(thisSize, nextrec);
  5600. if (toStdout && seq >= 0)
  5601. {
  5602. if (agent.queryOutputFmt() == ofSTD)
  5603. {
  5604. helper.serializeXml((byte *) nextrec.get(), *writer);
  5605. writer->newline();
  5606. agent.queryOutputSerializer()->fwrite(seq, (const void*)writer->str(), 1, writer->length());
  5607. writer->clear();
  5608. }
  5609. else if (agent.queryOutputFmt() == ofXML)
  5610. {
  5611. CommonXmlWriter xmlwrite(0,1);
  5612. xmlwrite.outputBeginNested(DEFAULTXMLROWTAG, false);
  5613. helper.serializeXml((byte *) nextrec.get(), xmlwrite);
  5614. xmlwrite.outputEndNested(DEFAULTXMLROWTAG);
  5615. agent.queryOutputSerializer()->fwrite(seq, (const void*)xmlwrite.str(), 1, xmlwrite.length());
  5616. }
  5617. }
  5618. rows++;
  5619. }
  5620. WorkunitUpdate w = agent.updateWorkUnit();
  5621. Owned<IWUResult> result = updateWorkUnitResult(w, helper.queryName(), helper.getSequence());
  5622. if (0 != (POFextend & helper.getFlags()))
  5623. result->addResultRaw(rowdata.length(), rowdata.toByteArray(), ResultFormatRaw);
  5624. else
  5625. result->setResultRaw(rowdata.length(), rowdata.toByteArray(), ResultFormatRaw);
  5626. result->setResultStatus(ResultStatusCalculated);
  5627. result->setResultRowCount(rows);
  5628. result->setResultTotalRowCount(rows); // Is this right??
  5629. if (toStdout && seq >= 0)
  5630. {
  5631. if (agent.queryOutputFmt() == ofXML)
  5632. {
  5633. StringBuffer sb;
  5634. sb.appendf(DEFAULTXMLFOOTER).newline();
  5635. agent.queryOutputSerializer()->fwrite(seq, (const void*)sb.str(), 1, sb.length());
  5636. }
  5637. else if (agent.queryOutputFmt() != ofSTD)
  5638. agent.outputFormattedResult(helper.queryName(), seq, false);
  5639. if (!(POFextend & helper.getFlags()))//POextend results will never get closed, so wont flush until serializer dtor
  5640. agent.queryOutputSerializer()->close(seq, false);
  5641. }
  5642. }
  5643. //=====================================================================================================
  5644. CHThorDictionaryWorkUnitWriteActivity::CHThorDictionaryWorkUnitWriteActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorDictionaryWorkUnitWriteArg &_arg, ThorActivityKind _kind)
  5645. : CHThorActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  5646. {
  5647. }
  5648. void CHThorDictionaryWorkUnitWriteActivity::execute()
  5649. {
  5650. int sequence = helper.getSequence();
  5651. const char *storedName = helper.queryName();
  5652. assertex(storedName && *storedName);
  5653. assertex(sequence < 0);
  5654. RtlLinkedDictionaryBuilder builder(rowAllocator, helper.queryHashLookupInfo());
  5655. for (;;)
  5656. {
  5657. const void *row = input->nextRow();
  5658. if (!row)
  5659. {
  5660. row = input->nextRow();
  5661. if (!row)
  5662. break;
  5663. }
  5664. builder.appendOwn(row);
  5665. processed++;
  5666. }
  5667. unsigned __int64 usedCount = rtlDictionaryCount(builder.getcount(), builder.queryrows());
  5668. // In absense of OPT_OUTPUTLIMIT check pre 5.2 legacy name OPT_OUTPUTLIMIT_LEGACY
  5669. size32_t outputLimit = agent.queryWorkUnit()->getDebugValueInt(OPT_OUTPUTLIMIT, agent.queryWorkUnit()->getDebugValueInt(OPT_OUTPUTLIMIT_LEGACY, defaultDaliResultLimit)) * 0x100000;
  5670. MemoryBuffer rowdata;
  5671. CThorDemoRowSerializer out(rowdata);
  5672. Owned<IOutputRowSerializer> serializer = input->queryOutputMeta()->createDiskSerializer(agent.queryCodeContext(), activityId);
  5673. rtlSerializeDictionary(out, serializer, builder.getcount(), builder.queryrows());
  5674. if(outputLimit && (rowdata.length() > outputLimit))
  5675. {
  5676. StringBuffer errMsg("Dictionary too large to output to workunit (limit ");
  5677. errMsg.append(outputLimit/0x100000).append(" megabytes), in result (");
  5678. const char *name = helper.queryName();
  5679. if (name)
  5680. errMsg.append("name=").append(name);
  5681. else
  5682. errMsg.append("sequence=").append(helper.getSequence());
  5683. errMsg.append(")");
  5684. throw MakeStringExceptionDirect(0, errMsg.str());
  5685. }
  5686. WorkunitUpdate w = agent.updateWorkUnit();
  5687. Owned<IWUResult> result = updateWorkUnitResult(w, helper.queryName(), helper.getSequence());
  5688. result->setResultRaw(rowdata.length(), rowdata.toByteArray(), ResultFormatRaw);
  5689. result->setResultStatus(ResultStatusCalculated);
  5690. result->setResultRowCount(usedCount);
  5691. result->setResultTotalRowCount(usedCount); // Is this right??
  5692. }
  5693. //=====================================================================================================
  5694. CHThorRemoteResultActivity::CHThorRemoteResultActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorRemoteResultArg &_arg, ThorActivityKind _kind)
  5695. : CHThorActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  5696. {
  5697. }
  5698. void CHThorRemoteResultActivity::execute()
  5699. {
  5700. OwnedConstRoxieRow result(input->nextRow());
  5701. helper.sendResult(result);
  5702. }
  5703. //=====================================================================================================
  5704. CHThorInlineTableActivity::CHThorInlineTableActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorInlineTableArg &_arg, ThorActivityKind _kind) :
  5705. CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  5706. {
  5707. }
  5708. void CHThorInlineTableActivity::ready()
  5709. {
  5710. CHThorSimpleActivityBase::ready();
  5711. curRow = 0;
  5712. numRows = helper.numRows();
  5713. }
  5714. const void *CHThorInlineTableActivity::nextRow()
  5715. {
  5716. // Filtering empty rows, returns the next valid row
  5717. while (curRow < numRows)
  5718. {
  5719. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  5720. size32_t size = helper.getRow(rowBuilder, curRow++);
  5721. if (size)
  5722. {
  5723. processed++;
  5724. return rowBuilder.finalizeRowClear(size);
  5725. }
  5726. }
  5727. return NULL;
  5728. }
  5729. //=====================================================================================================
  5730. CHThorNullActivity::CHThorNullActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  5731. {
  5732. }
  5733. const void *CHThorNullActivity::nextRow()
  5734. {
  5735. return NULL;
  5736. }
  5737. //=====================================================================================================
  5738. CHThorActionActivity::CHThorActionActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorActionArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  5739. {
  5740. }
  5741. void CHThorActionActivity::execute()
  5742. {
  5743. helper.action();
  5744. }
  5745. const void *CHThorActionActivity::nextRow()
  5746. {
  5747. return NULL;
  5748. }
  5749. //=====================================================================================================
  5750. CHThorSideEffectActivity::CHThorSideEffectActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorSideEffectArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  5751. {
  5752. }
  5753. const void *CHThorSideEffectActivity::nextRow()
  5754. {
  5755. try
  5756. {
  5757. helper.action();
  5758. }
  5759. catch(IException * e)
  5760. {
  5761. throw makeWrappedException(e);
  5762. }
  5763. return NULL;
  5764. }
  5765. //=====================================================================================================
  5766. CHThorDummyActivity::CHThorDummyActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind)
  5767. {
  5768. }
  5769. void CHThorDummyActivity::execute()
  5770. {
  5771. }
  5772. const void *CHThorDummyActivity::nextRow()
  5773. {
  5774. return input ? input->nextRow() : NULL;
  5775. }
  5776. //=====================================================================================================
  5777. CHThorWhenActionActivity::CHThorWhenActionActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorArg &_arg, ThorActivityKind _kind, EclGraphElement * _graphElement)
  5778. : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), graphElement(_graphElement)
  5779. {
  5780. }
  5781. void CHThorWhenActionActivity::ready()
  5782. {
  5783. CHThorSimpleActivityBase::ready();
  5784. graphElement->executeDependentActions(agent, NULL, WhenBeforeId);
  5785. graphElement->executeDependentActions(agent, NULL, WhenParallelId);
  5786. }
  5787. void CHThorWhenActionActivity::execute()
  5788. {
  5789. graphElement->executeDependentActions(agent, NULL, 1);
  5790. }
  5791. const void * CHThorWhenActionActivity::nextRow()
  5792. {
  5793. return input->nextRow();
  5794. }
  5795. void CHThorWhenActionActivity::stop()
  5796. {
  5797. graphElement->executeDependentActions(agent, NULL, WhenSuccessId);
  5798. CHThorSimpleActivityBase::stop();
  5799. }
  5800. //=====================================================================================================
  5801. CHThorMultiInputActivity::CHThorMultiInputActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind)
  5802. {
  5803. }
  5804. void CHThorMultiInputActivity::ready()
  5805. {
  5806. CHThorSimpleActivityBase::ready();
  5807. ForEachItemIn(idx, inputs)
  5808. inputs.item(idx)->ready();
  5809. }
  5810. void CHThorMultiInputActivity::stop()
  5811. {
  5812. CHThorSimpleActivityBase::stop();
  5813. ForEachItemIn(idx, inputs)
  5814. inputs.item(idx)->stop();
  5815. }
  5816. void CHThorMultiInputActivity::resetEOF()
  5817. {
  5818. CHThorSimpleActivityBase::resetEOF();
  5819. ForEachItemIn(idx, inputs)
  5820. inputs.item(idx)->resetEOF();
  5821. }
  5822. void CHThorMultiInputActivity::setInput(unsigned index, IHThorInput *_input)
  5823. {
  5824. if (index==inputs.length())
  5825. {
  5826. inputs.append(_input);
  5827. }
  5828. else
  5829. {
  5830. while (!inputs.isItem(index))
  5831. inputs.append(NULL);
  5832. inputs.replace(_input, index);
  5833. }
  5834. }
  5835. void CHThorMultiInputActivity::updateProgress(IStatisticGatherer &progress) const
  5836. {
  5837. CHThorSimpleActivityBase::updateProgress(progress);
  5838. ForEachItemIn(idx, inputs)
  5839. {
  5840. IHThorInput *i = inputs.item(idx);
  5841. if (i)
  5842. i->updateProgress(progress);
  5843. }
  5844. }
  5845. //=====================================================================================================
  5846. CHThorConcatActivity::CHThorConcatActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorFunnelArg &_arg, ThorActivityKind _kind) : CHThorMultiInputActivity(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  5847. {
  5848. }
  5849. void CHThorConcatActivity::ready()
  5850. {
  5851. grouped = helper.queryOutputMeta()->isGrouped();
  5852. inputIdx = 0;
  5853. curInput = inputs.item(inputIdx);
  5854. eogSeen = false;
  5855. anyThisGroup = false;
  5856. CHThorMultiInputActivity::ready();
  5857. }
  5858. const void *CHThorConcatActivity::nextRow()
  5859. {
  5860. if (!curInput)
  5861. return NULL; // eof
  5862. const void * next = curInput->nextRow();
  5863. if (next)
  5864. {
  5865. anyThisGroup = true;
  5866. eogSeen = false;
  5867. processed++;
  5868. return next;
  5869. }
  5870. else if (!eogSeen)
  5871. {
  5872. eogSeen = true;
  5873. if (grouped)
  5874. {
  5875. if (anyThisGroup)
  5876. {
  5877. anyThisGroup = false;
  5878. return NULL;
  5879. }
  5880. else
  5881. return nextRow();
  5882. }
  5883. else
  5884. return nextRow();
  5885. }
  5886. else if (inputIdx < inputs.length()-1)
  5887. {
  5888. inputIdx++;
  5889. curInput = inputs.item(inputIdx);
  5890. eogSeen = false;
  5891. anyThisGroup = false;
  5892. return nextRow();
  5893. }
  5894. else
  5895. {
  5896. curInput = NULL;
  5897. return NULL;
  5898. }
  5899. }
  5900. //=====================================================================================================
  5901. CHThorNonEmptyActivity::CHThorNonEmptyActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorNonEmptyArg &_arg, ThorActivityKind _kind) : CHThorMultiInputActivity(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  5902. {
  5903. }
  5904. void CHThorNonEmptyActivity::ready()
  5905. {
  5906. grouped = helper.queryOutputMeta()->isGrouped();
  5907. selectedInput = NULL;
  5908. CHThorMultiInputActivity::ready();
  5909. }
  5910. const void *CHThorNonEmptyActivity::nextRow()
  5911. {
  5912. if (!selectedInput)
  5913. {
  5914. ForEachItemIn(i, inputs)
  5915. {
  5916. IHThorInput * cur = inputs.item(i);
  5917. const void * next = cur->nextRow();
  5918. if (next)
  5919. {
  5920. selectedInput = cur;
  5921. processed++;
  5922. return next;
  5923. }
  5924. }
  5925. return NULL;
  5926. }
  5927. const void * next = selectedInput->nextRow();
  5928. if (next)
  5929. processed++;
  5930. return next;
  5931. }
  5932. //=====================================================================================================
  5933. CHThorRegroupActivity::CHThorRegroupActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorRegroupArg &_arg, ThorActivityKind _kind) : CHThorMultiInputActivity(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  5934. {
  5935. }
  5936. void CHThorRegroupActivity::ready()
  5937. {
  5938. inputIndex = 0;
  5939. eof = false;
  5940. numProcessedLastGroup = processed;
  5941. CHThorMultiInputActivity::ready();
  5942. }
  5943. const void * CHThorRegroupActivity::nextFromInputs()
  5944. {
  5945. unsigned initialInput = inputIndex;
  5946. while (inputs.isItem(inputIndex))
  5947. {
  5948. OwnedConstRoxieRow next(inputs.item(inputIndex)->nextRow());
  5949. if (next)
  5950. {
  5951. if ((inputIndex != initialInput) && (inputIndex != initialInput+1))
  5952. {
  5953. throw MakeStringException(100, "Mismatched groups supplied to regroup %u", activityId);
  5954. }
  5955. return next.getClear();
  5956. }
  5957. inputIndex++;
  5958. }
  5959. if ((initialInput != 0) && (initialInput+1 != inputs.ordinality()))
  5960. throw MakeStringException(100, "Mismatched groups supplied to Regroup Activity(%u)", activityId);
  5961. inputIndex = 0;
  5962. return NULL;
  5963. }
  5964. const void * CHThorRegroupActivity::nextRow()
  5965. {
  5966. if (eof)
  5967. return NULL;
  5968. const void * ret = nextFromInputs();
  5969. if (ret)
  5970. {
  5971. processed++;
  5972. return ret;
  5973. }
  5974. if (numProcessedLastGroup != processed)
  5975. {
  5976. numProcessedLastGroup = processed;
  5977. return NULL;
  5978. }
  5979. eof = true;
  5980. return NULL;
  5981. }
  5982. //=====================================================================================================
  5983. CHThorRollupGroupActivity::CHThorRollupGroupActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorRollupGroupArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  5984. {
  5985. }
  5986. void CHThorRollupGroupActivity::ready()
  5987. {
  5988. CHThorSimpleActivityBase::ready();
  5989. eof = false;
  5990. }
  5991. const void * CHThorRollupGroupActivity::nextRow()
  5992. {
  5993. if (eof)
  5994. return NULL;
  5995. for (;;)
  5996. {
  5997. OwnedRowArray group;
  5998. for (;;)
  5999. {
  6000. const void * in = input->nextRow();
  6001. if (!in)
  6002. break;
  6003. group.append(in);
  6004. }
  6005. if (group.ordinality() == 0)
  6006. {
  6007. eof = true;
  6008. return NULL;
  6009. }
  6010. try
  6011. {
  6012. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  6013. size32_t outSize = helper.transform(rowBuilder, group.ordinality(), (const void * *)group.getArray());
  6014. if (outSize)
  6015. {
  6016. processed++;
  6017. return rowBuilder.finalizeRowClear(outSize);
  6018. }
  6019. }
  6020. catch(IException * e)
  6021. {
  6022. throw makeWrappedException(e);
  6023. }
  6024. }
  6025. }
  6026. //=====================================================================================================
  6027. CHThorCombineActivity::CHThorCombineActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorCombineArg &_arg, ThorActivityKind _kind) : CHThorMultiInputActivity(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  6028. {
  6029. }
  6030. void CHThorCombineActivity::ready()
  6031. {
  6032. numProcessedLastGroup = processed;
  6033. CHThorMultiInputActivity::ready();
  6034. }
  6035. void CHThorCombineActivity::nextInputs(OwnedRowArray & out)
  6036. {
  6037. ForEachItemIn(i, inputs)
  6038. {
  6039. const void * next = inputs.item(i)->nextRow();
  6040. if (next)
  6041. out.append(next);
  6042. }
  6043. }
  6044. const void *CHThorCombineActivity::nextRow()
  6045. {
  6046. for (;;)
  6047. {
  6048. OwnedRowArray group;
  6049. nextInputs(group);
  6050. if ((group.ordinality() == 0) && (numProcessedLastGroup == processed))
  6051. nextInputs(group);
  6052. if (group.ordinality() == 0)
  6053. {
  6054. numProcessedLastGroup = processed;
  6055. return NULL;
  6056. }
  6057. else if (group.ordinality() != inputs.ordinality())
  6058. {
  6059. throw MakeStringException(101, "Mismatched group input for Combine Activity(%u)", activityId);
  6060. }
  6061. try
  6062. {
  6063. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  6064. size32_t outSize = helper.transform(rowBuilder, group.ordinality(), (const void * *)group.getArray());
  6065. if (outSize)
  6066. {
  6067. processed++;
  6068. return rowBuilder.finalizeRowClear(outSize);
  6069. }
  6070. }
  6071. catch(IException * e)
  6072. {
  6073. throw makeWrappedException(e);
  6074. }
  6075. }
  6076. }
  6077. //=====================================================================================================
  6078. CHThorCombineGroupActivity::CHThorCombineGroupActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorCombineGroupArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  6079. {
  6080. }
  6081. void CHThorCombineGroupActivity::ready()
  6082. {
  6083. numProcessedLastGroup = processed;
  6084. CHThorSimpleActivityBase::ready();
  6085. input1->ready();
  6086. }
  6087. void CHThorCombineGroupActivity::stop()
  6088. {
  6089. CHThorSimpleActivityBase::stop();
  6090. input1->stop();
  6091. }
  6092. void CHThorCombineGroupActivity::setInput(unsigned index, IHThorInput *_input)
  6093. {
  6094. if (index==1)
  6095. input1 = _input;
  6096. else
  6097. CHThorSimpleActivityBase::setInput(index, _input);
  6098. }
  6099. const void *CHThorCombineGroupActivity::nextRow()
  6100. {
  6101. for (;;)
  6102. {
  6103. OwnedConstRoxieRow left(input->nextRow());
  6104. if (!left && (numProcessedLastGroup == processed))
  6105. left.setown(input->nextRow());
  6106. if (!left)
  6107. {
  6108. if (numProcessedLastGroup == processed)
  6109. {
  6110. OwnedConstRoxieRow nextRight(input1->nextRow());
  6111. if (nextRight)
  6112. throw MakeStringException(101, "Missing LEFT record for Combine group Activity(%u)", activityId);
  6113. }
  6114. else
  6115. numProcessedLastGroup = processed;
  6116. return NULL;
  6117. }
  6118. OwnedRowArray group;
  6119. for (;;)
  6120. {
  6121. const void * in = input1->nextRow();
  6122. if (!in)
  6123. break;
  6124. group.append(in);
  6125. }
  6126. if (group.ordinality() == 0)
  6127. {
  6128. throw MakeStringException(101, "Missing RIGHT group for Combine Group Activity(%u)", activityId);
  6129. }
  6130. try
  6131. {
  6132. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  6133. size32_t outSize = helper.transform(rowBuilder, left, group.ordinality(), (const void * *)group.getArray());
  6134. if (outSize)
  6135. {
  6136. processed++;
  6137. return rowBuilder.finalizeRowClear(outSize);
  6138. }
  6139. }
  6140. catch(IException * e)
  6141. {
  6142. throw makeWrappedException(e);
  6143. }
  6144. }
  6145. }
  6146. //=====================================================================================================
  6147. CHThorApplyActivity::CHThorApplyActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorApplyArg &_arg, ThorActivityKind _kind) : CHThorActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  6148. {
  6149. }
  6150. void CHThorApplyActivity::execute()
  6151. {
  6152. try
  6153. {
  6154. helper.start();
  6155. for (;;)
  6156. {
  6157. OwnedConstRoxieRow next(input->nextRow());
  6158. if (!next)
  6159. {
  6160. next.setown(input->nextRow());
  6161. if (!next)
  6162. break;
  6163. }
  6164. helper.apply(next);
  6165. }
  6166. helper.end();
  6167. }
  6168. catch (IException *e)
  6169. {
  6170. throw makeWrappedException(e);
  6171. }
  6172. }
  6173. //=====================================================================================================
  6174. CHThorDistributionActivity::CHThorDistributionActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorDistributionArg &_arg, ThorActivityKind _kind)
  6175. : CHThorActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  6176. {
  6177. }
  6178. void CHThorDistributionActivity::execute()
  6179. {
  6180. MemoryAttr ma;
  6181. IDistributionTable * * accumulator = (IDistributionTable * *)ma.allocate(helper.queryInternalRecordSize()->getMinRecordSize());
  6182. helper.clearAggregate(accumulator);
  6183. OwnedConstRoxieRow nextrec(input->nextRow());
  6184. for (;;)
  6185. {
  6186. if (!nextrec)
  6187. {
  6188. nextrec.setown(input->nextRow());
  6189. if (!nextrec)
  6190. break;
  6191. }
  6192. helper.process(accumulator, nextrec);
  6193. nextrec.setown(input->nextRow());
  6194. }
  6195. StringBuffer result;
  6196. result.append("<XML>");
  6197. helper.gatherResult(accumulator, result);
  6198. result.append("</XML>");
  6199. helper.sendResult(result.length(), result.str());
  6200. helper.destruct(accumulator);
  6201. }
  6202. //---------------------------------------------------------------------------
  6203. CHThorWorkunitReadActivity::CHThorWorkunitReadActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorWorkunitReadArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  6204. {
  6205. first = true;
  6206. bufferStream.setown(createMemoryBufferSerialStream(resultBuffer));
  6207. deserializer.setStream(bufferStream);
  6208. }
  6209. CHThorWorkunitReadActivity::~CHThorWorkunitReadActivity()
  6210. {
  6211. }
  6212. void CHThorWorkunitReadActivity::ready()
  6213. {
  6214. CHThorSimpleActivityBase::ready();
  6215. rowDeserializer.setown(rowAllocator->createDiskDeserializer(agent.queryCodeContext()));
  6216. if(first)
  6217. {
  6218. checkForDiskRead();
  6219. first = false;
  6220. }
  6221. if(diskread)
  6222. {
  6223. diskread->ready();
  6224. return;
  6225. }
  6226. grouped = outputMeta.isGrouped();
  6227. unsigned lenData;
  6228. void * tempData;
  6229. OwnedRoxieString fromWuid(helper.getWUID());
  6230. ICsvToRowTransformer * csvTransformer = helper.queryCsvTransformer();
  6231. IXmlToRowTransformer * xmlTransformer = helper.queryXmlTransformer();
  6232. if (fromWuid)
  6233. agent.queryCodeContext()->getExternalResultRaw(lenData, tempData, fromWuid, helper.queryName(), helper.querySequence(), xmlTransformer, csvTransformer);
  6234. else
  6235. agent.queryCodeContext()->getResultRaw(lenData, tempData, helper.queryName(), helper.querySequence(), xmlTransformer, csvTransformer);
  6236. resultBuffer.setBuffer(lenData, tempData, true);
  6237. eogPending = false;
  6238. }
  6239. void CHThorWorkunitReadActivity::checkForDiskRead()
  6240. {
  6241. StringBuffer diskFilename;
  6242. OwnedRoxieString fromWuid(helper.getWUID());
  6243. if (agent.getWorkunitResultFilename(diskFilename, fromWuid, helper.queryName(), helper.querySequence()))
  6244. {
  6245. diskreadHelper.setown(createWorkUnitReadArg(diskFilename.str(), &helper));
  6246. try
  6247. {
  6248. diskreadHelper->onCreate(agent.queryCodeContext(), NULL, NULL);
  6249. }
  6250. catch(IException * e)
  6251. {
  6252. throw makeWrappedException(e);
  6253. }
  6254. diskread.setown(new CHThorDiskReadActivity(agent, activityId, subgraphId, *diskreadHelper, TAKdiskread));
  6255. }
  6256. }
  6257. void CHThorWorkunitReadActivity::stop()
  6258. {
  6259. if(diskread)
  6260. diskread->stop();
  6261. resultBuffer.resetBuffer();
  6262. CHThorSimpleActivityBase::stop();
  6263. }
  6264. const void *CHThorWorkunitReadActivity::nextRow()
  6265. {
  6266. if(diskread)
  6267. {
  6268. const void * ret = diskread->nextRow();
  6269. processed = diskread->queryProcessed();
  6270. return ret;
  6271. }
  6272. if (deserializer.eos())
  6273. return NULL;
  6274. if (eogPending)
  6275. {
  6276. eogPending = false;
  6277. return NULL;
  6278. }
  6279. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  6280. size32_t newSize = rowDeserializer->deserialize(rowBuilder, deserializer);
  6281. if (grouped)
  6282. deserializer.read(sizeof(bool), &eogPending);
  6283. processed++;
  6284. return rowBuilder.finalizeRowClear(newSize);
  6285. }
  6286. //=====================================================================================================
  6287. CHThorParseActivity::CHThorParseActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorParseArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  6288. {
  6289. //DebugBreak();
  6290. anyThisGroup = false;
  6291. curSearchTextLen = 0;
  6292. curSearchText = NULL;
  6293. algorithm = createThorParser(agent.queryCodeContext(), helper);
  6294. parser = algorithm->createParser(agent.queryCodeContext(), activityId, helper.queryHelper(), &helper);
  6295. rowIter = parser->queryResultIter();
  6296. }
  6297. CHThorParseActivity::~CHThorParseActivity()
  6298. {
  6299. if (curSearchText && helper.searchTextNeedsFree())
  6300. rtlFree(curSearchText);
  6301. parser->Release();
  6302. algorithm->Release();
  6303. }
  6304. void CHThorParseActivity::ready()
  6305. {
  6306. CHThorSimpleActivityBase::ready();
  6307. anyThisGroup = false;
  6308. parser->reset();
  6309. }
  6310. void CHThorParseActivity::stop()
  6311. {
  6312. CHThorSimpleActivityBase::stop();
  6313. if (curSearchText && helper.searchTextNeedsFree())
  6314. rtlFree(curSearchText);
  6315. curSearchText = NULL;
  6316. in.clear();
  6317. }
  6318. bool CHThorParseActivity::processRecord(const void * in)
  6319. {
  6320. if (curSearchText && helper.searchTextNeedsFree())
  6321. rtlFree(curSearchText);
  6322. curSearchTextLen = 0;
  6323. curSearchText = NULL;
  6324. helper.getSearchText(curSearchTextLen, curSearchText, in);
  6325. return parser->performMatch(*this, in, curSearchTextLen, curSearchText);
  6326. }
  6327. unsigned CHThorParseActivity::onMatch(ARowBuilder & self, const void * curRecord, IMatchedResults * results, IMatchWalker * walker)
  6328. {
  6329. try
  6330. {
  6331. return helper.transform(self, curRecord, results, walker);
  6332. }
  6333. catch(IException * e)
  6334. {
  6335. throw makeWrappedException(e);
  6336. }
  6337. }
  6338. const void * CHThorParseActivity::nextRow()
  6339. {
  6340. for (;;)
  6341. {
  6342. if (rowIter->isValid())
  6343. {
  6344. anyThisGroup = true;
  6345. OwnedConstRoxieRow out = rowIter->getRow();
  6346. rowIter->next();
  6347. processed++;
  6348. return out.getClear();
  6349. }
  6350. in.setown(input->nextRow());
  6351. if (!in)
  6352. {
  6353. if (anyThisGroup)
  6354. {
  6355. anyThisGroup = false;
  6356. return NULL;
  6357. }
  6358. in.setown(input->nextRow());
  6359. if (!in)
  6360. return NULL;
  6361. }
  6362. processRecord(in);
  6363. rowIter->first();
  6364. }
  6365. }
  6366. //=====================================================================================================
  6367. CHThorEnthActivity::CHThorEnthActivity(IAgentContext & _agent, unsigned _activityId, unsigned _subgraphId, IHThorEnthArg & _arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg), outBuilder(NULL)
  6368. {
  6369. }
  6370. void CHThorEnthActivity::ready()
  6371. {
  6372. CHThorSimpleActivityBase::ready();
  6373. outBuilder.setAllocator(rowAllocator);
  6374. numerator = helper.getProportionNumerator();
  6375. denominator = helper.getProportionDenominator();
  6376. started = false;
  6377. }
  6378. void CHThorEnthActivity::stop()
  6379. {
  6380. outBuilder.clear();
  6381. }
  6382. void CHThorEnthActivity::start()
  6383. {
  6384. if(denominator == 0) denominator = 1;
  6385. counter = (helper.getSampleNumber()-1) * greatestCommonDivisor(numerator, denominator);
  6386. if (counter >= denominator)
  6387. counter %= denominator;
  6388. started = true;
  6389. }
  6390. const void * CHThorEnthActivity::nextRow()
  6391. {
  6392. if(!started)
  6393. start();
  6394. OwnedConstRoxieRow ret;
  6395. for (;;)
  6396. {
  6397. ret.setown(input->nextRow());
  6398. if(!ret) //end of group
  6399. ret.setown(input->nextRow());
  6400. if(!ret) //eof
  6401. return NULL;
  6402. if (wanted())
  6403. {
  6404. processed++;
  6405. return ret.getClear();
  6406. }
  6407. }
  6408. }
  6409. //=====================================================================================================
  6410. CHThorTopNActivity::CHThorTopNActivity(IAgentContext & _agent, unsigned _activityId, unsigned _subgraphId, IHThorTopNArg & _arg, ThorActivityKind _kind)
  6411. : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg), compare(*helper.queryCompare())
  6412. {
  6413. hasBest = helper.hasBest();
  6414. grouped = outputMeta.isGrouped();
  6415. curIndex = 0;
  6416. sortedCount = 0;
  6417. limit = 0;
  6418. sorted = NULL;
  6419. }
  6420. CHThorTopNActivity::~CHThorTopNActivity()
  6421. {
  6422. roxiemem::ReleaseRoxieRowRange(sorted, curIndex, sortedCount);
  6423. free(sorted);
  6424. }
  6425. void CHThorTopNActivity::ready()
  6426. {
  6427. CHThorSimpleActivityBase::ready();
  6428. limit = helper.getLimit();
  6429. assertex(limit == (size_t)limit);
  6430. sorted = (const void * *)checked_calloc((size_t)(limit+1), sizeof(void *), "topn");
  6431. sortedCount = 0;
  6432. curIndex = 0;
  6433. eof = false;
  6434. eoi = false;
  6435. }
  6436. void CHThorTopNActivity::stop()
  6437. {
  6438. CHThorSimpleActivityBase::stop();
  6439. roxiemem::ReleaseRoxieRowRange(sorted, curIndex, sortedCount);
  6440. free(sorted);
  6441. sorted = NULL;
  6442. curIndex = 0;
  6443. sortedCount = 0;
  6444. }
  6445. const void * CHThorTopNActivity::nextRow()
  6446. {
  6447. if(eof)
  6448. return NULL;
  6449. if(curIndex >= sortedCount)
  6450. {
  6451. bool eog = sortedCount != 0;
  6452. getSorted();
  6453. if(sortedCount == 0)
  6454. {
  6455. eof = true;
  6456. return NULL;
  6457. }
  6458. if (eog)
  6459. return NULL;
  6460. }
  6461. processed++;
  6462. return sorted[curIndex++];
  6463. }
  6464. bool CHThorTopNActivity::abortEarly()
  6465. {
  6466. if (hasBest && (sortedCount == limit))
  6467. {
  6468. int compare = helper.compareBest(sorted[sortedCount-1]);
  6469. if (compare == 0)
  6470. {
  6471. if (grouped)
  6472. {
  6473. //MORE: This would be more efficient if we had a away of skipping to the end of the incomming group.
  6474. OwnedConstRoxieRow next;
  6475. do
  6476. {
  6477. next.setown(input->nextRow());
  6478. } while(next);
  6479. }
  6480. else
  6481. eoi = true;
  6482. return true;
  6483. }
  6484. //This only checks the lowest element - we could check all elements inserted, but it would increase the number of compares
  6485. if (compare < 0)
  6486. throw MakeStringException(0, "TOPN: row found that exceeds the best value");
  6487. }
  6488. return false;
  6489. }
  6490. void CHThorTopNActivity::getSorted()
  6491. {
  6492. curIndex = 0;
  6493. sortedCount = 0;
  6494. if (eoi)
  6495. return;
  6496. OwnedConstRoxieRow next(input->nextRow());
  6497. while(next)
  6498. {
  6499. if(sortedCount < limit)
  6500. {
  6501. binary_vec_insert_stable(next.getClear(), sorted, sortedCount, compare);
  6502. sortedCount++;
  6503. if (abortEarly())
  6504. return;
  6505. }
  6506. else
  6507. {
  6508. // do not bother with insertion sort if we know next will fall off the end
  6509. if(limit && compare.docompare(sorted[sortedCount-1], next) > 0)
  6510. {
  6511. binary_vec_insert_stable(next.getClear(), sorted, sortedCount, compare);
  6512. ReleaseRoxieRow(sorted[sortedCount]);
  6513. if (abortEarly())
  6514. return;
  6515. }
  6516. }
  6517. next.setown(input->nextRow());
  6518. }
  6519. }
  6520. //=====================================================================================================
  6521. CHThorXmlParseActivity::CHThorXmlParseActivity(IAgentContext & _agent, unsigned _activityId, unsigned _subgraphId, IHThorXmlParseArg & _arg, ThorActivityKind _kind)
  6522. : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  6523. {
  6524. srchStrNeedsFree = helper.searchTextNeedsFree();
  6525. srchStr = NULL;
  6526. }
  6527. CHThorXmlParseActivity::~CHThorXmlParseActivity()
  6528. {
  6529. if(srchStrNeedsFree) rtlFree(srchStr);
  6530. }
  6531. void CHThorXmlParseActivity::ready()
  6532. {
  6533. CHThorSimpleActivityBase::ready();
  6534. numProcessedLastGroup = processed;
  6535. }
  6536. void CHThorXmlParseActivity::stop()
  6537. {
  6538. CHThorSimpleActivityBase::stop();
  6539. if(srchStrNeedsFree) rtlFree(srchStr);
  6540. srchStr = NULL;
  6541. in.clear();
  6542. }
  6543. const void * CHThorXmlParseActivity::nextRow()
  6544. {
  6545. for (;;)
  6546. {
  6547. if(xmlParser)
  6548. {
  6549. for (;;)
  6550. {
  6551. bool gotNext = false;
  6552. try
  6553. {
  6554. gotNext = xmlParser->next();
  6555. }
  6556. catch(IException * e)
  6557. {
  6558. throw makeWrappedException(e);
  6559. }
  6560. if(!gotNext)
  6561. {
  6562. if(srchStrNeedsFree)
  6563. {
  6564. rtlFree(srchStr);
  6565. srchStr = NULL;
  6566. }
  6567. xmlParser.clear();
  6568. break;
  6569. }
  6570. if(lastMatch)
  6571. {
  6572. try
  6573. {
  6574. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  6575. unsigned sizeGot = helper.transform(rowBuilder, in, lastMatch);
  6576. lastMatch.clear();
  6577. if (sizeGot)
  6578. {
  6579. processed++;
  6580. return rowBuilder.finalizeRowClear(sizeGot);
  6581. }
  6582. }
  6583. catch(IException * e)
  6584. {
  6585. throw makeWrappedException(e);
  6586. }
  6587. }
  6588. }
  6589. }
  6590. in.setown(input->nextRow());
  6591. if(!in)
  6592. {
  6593. if(numProcessedLastGroup == processed)
  6594. in.setown(input->nextRow());
  6595. if(!in)
  6596. {
  6597. numProcessedLastGroup = processed;
  6598. return NULL;
  6599. }
  6600. }
  6601. size32_t srchLen;
  6602. helper.getSearchText(srchLen, srchStr, in);
  6603. OwnedRoxieString xmlIteratorPath(helper.getXmlIteratorPath());
  6604. xmlParser.setown(createXMLParse(srchStr, srchLen, xmlIteratorPath, *this, ptr_noRoot, helper.requiresContents()));
  6605. }
  6606. }
  6607. //=====================================================================================================
  6608. class CHThorMergeActivity : public CHThorMultiInputActivity
  6609. {
  6610. protected:
  6611. IHThorMergeArg &helper;
  6612. CHThorStreamMerger merger;
  6613. public:
  6614. CHThorMergeActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorMergeArg &_arg, ThorActivityKind _kind) : CHThorMultiInputActivity(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  6615. {
  6616. merger.init(helper.queryCompare(), helper.dedup(), NULL); // can mass null for range because merger.nextGE() never called
  6617. }
  6618. ~CHThorMergeActivity()
  6619. {
  6620. merger.cleanup();
  6621. }
  6622. virtual void ready()
  6623. {
  6624. CHThorMultiInputActivity::ready();
  6625. merger.initInputs(inputs.length(), inputs.getArray());
  6626. }
  6627. virtual void stop()
  6628. {
  6629. merger.done();
  6630. CHThorMultiInputActivity::stop();
  6631. }
  6632. virtual const void * nextRow()
  6633. {
  6634. const void * ret = merger.nextRow();
  6635. if (ret)
  6636. processed++;
  6637. return ret;
  6638. }
  6639. };
  6640. //=====================================================================================================
  6641. //Web Service Call base
  6642. CHThorWSCBaseActivity::CHThorWSCBaseActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorWebServiceCallArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  6643. {
  6644. callHelper = &_arg;
  6645. init();
  6646. }
  6647. CHThorWSCBaseActivity::CHThorWSCBaseActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorWebServiceCallActionArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  6648. {
  6649. callHelper = NULL;
  6650. init();
  6651. }
  6652. void CHThorWSCBaseActivity::stop()
  6653. {
  6654. WSChelper.clear();//doesn't return until helper threads terminate
  6655. CHThorSimpleActivityBase::stop();
  6656. }
  6657. void CHThorWSCBaseActivity::init()
  6658. {
  6659. // Build authentication token
  6660. StringBuffer uidpair;
  6661. IUserDescriptor *userDesc = agent.queryCodeContext()->queryUserDescriptor();
  6662. if (userDesc)//NULL if standalone
  6663. {
  6664. userDesc->getUserName(uidpair);
  6665. uidpair.append(":");
  6666. userDesc->getPassword(uidpair);
  6667. JBASE64_Encode(uidpair.str(), uidpair.length(), authToken);
  6668. }
  6669. soapTraceLevel = agent.queryWorkUnit()->getDebugValueInt("soapTraceLevel", 1);
  6670. }
  6671. //---------------------------------------------------------------------------
  6672. CHThorWSCRowCallActivity::CHThorWSCRowCallActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorWebServiceCallArg &_arg, ThorActivityKind _kind) : CHThorWSCBaseActivity(_agent, _activityId, _subgraphId, _arg, _kind)
  6673. {
  6674. }
  6675. const void *CHThorWSCRowCallActivity::nextRow()
  6676. {
  6677. try
  6678. {
  6679. assertex(WSChelper);
  6680. OwnedConstRoxieRow ret = WSChelper->getRow();
  6681. if (!ret)
  6682. return NULL;
  6683. ++processed;
  6684. return ret.getClear();
  6685. }
  6686. catch(IException * e)
  6687. {
  6688. throw makeWrappedException(e);
  6689. }
  6690. }
  6691. //---------------------------------------------------------------------------
  6692. const void *CHThorHttpRowCallActivity::nextRow()
  6693. {
  6694. try
  6695. {
  6696. if (WSChelper == NULL)
  6697. {
  6698. WSChelper.setown(createHttpCallHelper(this, rowAllocator, authToken.str(), SCrow, NULL, queryDummyContextLogger(),NULL));
  6699. WSChelper->start();
  6700. }
  6701. return CHThorWSCRowCallActivity::nextRow();
  6702. }
  6703. catch(IException * e)
  6704. {
  6705. throw makeWrappedException(e);
  6706. }
  6707. }
  6708. //---------------------------------------------------------------------------
  6709. const void *CHThorSoapRowCallActivity::nextRow()
  6710. {
  6711. try
  6712. {
  6713. if (WSChelper == NULL)
  6714. {
  6715. WSChelper.setown(createSoapCallHelper(this, rowAllocator, authToken.str(), SCrow, NULL, queryDummyContextLogger(),NULL));
  6716. WSChelper->start();
  6717. }
  6718. return CHThorWSCRowCallActivity::nextRow();
  6719. }
  6720. catch(IException * e)
  6721. {
  6722. throw makeWrappedException(e);
  6723. }
  6724. }
  6725. //---------------------------------------------------------------------------
  6726. //---------------------------------------------------------------------------
  6727. //---------------------------------------------------------------------------
  6728. CHThorSoapRowActionActivity::CHThorSoapRowActionActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorSoapActionArg &_arg, ThorActivityKind _kind) : CHThorWSCBaseActivity(_agent, _activityId, _subgraphId, _arg, _kind)
  6729. {
  6730. }
  6731. void CHThorSoapRowActionActivity::execute()
  6732. {
  6733. try
  6734. {
  6735. WSChelper.setown(createSoapCallHelper(this, NULL, authToken.str(), SCrow, NULL, queryDummyContextLogger(),NULL));
  6736. WSChelper->start();
  6737. WSChelper->waitUntilDone();
  6738. }
  6739. catch(IException * e)
  6740. {
  6741. throw makeWrappedException(e);
  6742. }
  6743. IException *e = WSChelper->getError();
  6744. if(e)
  6745. throw makeWrappedException(e);
  6746. }
  6747. //---------------------------------------------------------------------------
  6748. CHThorSoapDatasetCallActivity::CHThorSoapDatasetCallActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorSoapCallArg &_arg, ThorActivityKind _kind) : CHThorWSCBaseActivity(_agent, _activityId, _subgraphId, _arg, _kind)
  6749. {
  6750. }
  6751. const void * CHThorSoapDatasetCallActivity::nextRow()
  6752. {
  6753. try
  6754. {
  6755. if (WSChelper == NULL)
  6756. {
  6757. WSChelper.setown(createSoapCallHelper(this, rowAllocator, authToken.str(), SCdataset, NULL, queryDummyContextLogger(),NULL));
  6758. WSChelper->start();
  6759. }
  6760. OwnedConstRoxieRow ret = WSChelper->getRow();
  6761. if (!ret)
  6762. return NULL;
  6763. ++processed;
  6764. return ret.getClear();
  6765. }
  6766. catch(IException * e)
  6767. {
  6768. throw makeWrappedException(e);
  6769. }
  6770. }
  6771. const void * CHThorSoapDatasetCallActivity::getNextRow()
  6772. {
  6773. CriticalBlock b(crit);
  6774. const void *nextrec = input->nextRow();
  6775. if (!nextrec)
  6776. {
  6777. nextrec = input->nextRow();
  6778. }
  6779. return nextrec;
  6780. };
  6781. //---------------------------------------------------------------------------
  6782. CHThorSoapDatasetActionActivity::CHThorSoapDatasetActionActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorSoapActionArg &_arg, ThorActivityKind _kind) : CHThorWSCBaseActivity(_agent, _activityId, _subgraphId, _arg, _kind)
  6783. {
  6784. }
  6785. void CHThorSoapDatasetActionActivity::execute()
  6786. {
  6787. try
  6788. {
  6789. WSChelper.setown(createSoapCallHelper(this, NULL, authToken.str(), SCdataset, NULL, queryDummyContextLogger(),NULL));
  6790. WSChelper->start();
  6791. WSChelper->waitUntilDone();
  6792. }
  6793. catch(IException * e)
  6794. {
  6795. throw makeWrappedException(e);
  6796. }
  6797. IException *e = WSChelper->getError();
  6798. if(e)
  6799. throw makeWrappedException(e);
  6800. }
  6801. const void * CHThorSoapDatasetActionActivity::getNextRow()
  6802. {
  6803. CriticalBlock b(crit);
  6804. const void *nextrec = input->nextRow();
  6805. if (!nextrec)
  6806. {
  6807. nextrec = input->nextRow();
  6808. }
  6809. if (nextrec)
  6810. {
  6811. processed++;
  6812. }
  6813. return nextrec;
  6814. };
  6815. //=====================================================================================================
  6816. CHThorResultActivity::CHThorResultActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorArg &_arg, ThorActivityKind _kind)
  6817. : CHThorActivityBase(_agent, _activityId, _subgraphId, _arg, _kind)
  6818. {
  6819. }
  6820. void CHThorResultActivity::extractResult(unsigned & retSize, void * & ret)
  6821. {
  6822. unsigned len = rowdata.length();
  6823. retSize = len;
  6824. if (len)
  6825. {
  6826. void * temp = rtlMalloc(len);
  6827. memcpy(temp, rowdata.toByteArray(), len);
  6828. ret = temp;
  6829. }
  6830. else
  6831. ret = NULL;
  6832. }
  6833. //=====================================================================================================
  6834. CHThorDatasetResultActivity::CHThorDatasetResultActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorDatasetResultArg &_arg, ThorActivityKind _kind)
  6835. : CHThorResultActivity(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  6836. {
  6837. }
  6838. void CHThorDatasetResultActivity::execute()
  6839. {
  6840. rowdata.clear();
  6841. IRecordSize * inputMeta = input->queryOutputMeta();
  6842. for (;;)
  6843. {
  6844. OwnedConstRoxieRow nextrec(input->nextRow());
  6845. if (!nextrec)
  6846. {
  6847. nextrec.setown(input->nextRow());
  6848. if (!nextrec)
  6849. break;
  6850. }
  6851. rowdata.append(inputMeta->getRecordSize(nextrec), nextrec);
  6852. }
  6853. }
  6854. //=====================================================================================================
  6855. CHThorRowResultActivity::CHThorRowResultActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorRowResultArg &_arg, ThorActivityKind _kind)
  6856. : CHThorResultActivity(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  6857. {
  6858. }
  6859. void CHThorRowResultActivity::execute()
  6860. {
  6861. OwnedConstRoxieRow nextrec(input->nextRow());
  6862. assertex(nextrec);
  6863. IRecordSize * inputMeta = input->queryOutputMeta();
  6864. unsigned length = inputMeta->getRecordSize(nextrec);
  6865. rowdata.clear().append(length, nextrec);
  6866. }
  6867. //=====================================================================================================
  6868. CHThorChildIteratorActivity::CHThorChildIteratorActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorChildIteratorArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  6869. {
  6870. }
  6871. const void *CHThorChildIteratorActivity::nextRow()
  6872. {
  6873. if (eof)
  6874. return NULL;
  6875. bool ok;
  6876. if (!started)
  6877. {
  6878. ok = helper.first();
  6879. started = true;
  6880. }
  6881. else
  6882. ok = helper.next();
  6883. try
  6884. {
  6885. while(ok)
  6886. {
  6887. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  6888. size32_t outSize = helper.transform(rowBuilder);
  6889. if(outSize)
  6890. {
  6891. processed++;
  6892. return rowBuilder.finalizeRowClear(outSize);
  6893. }
  6894. ok = helper.next();
  6895. }
  6896. }
  6897. catch(IException * e)
  6898. {
  6899. throw makeWrappedException(e);
  6900. }
  6901. eof = true;
  6902. return NULL;
  6903. }
  6904. void CHThorChildIteratorActivity::ready()
  6905. {
  6906. CHThorSimpleActivityBase::ready();
  6907. started = false;
  6908. eof = false;
  6909. }
  6910. //=====================================================================================================
  6911. CHThorLinkedRawIteratorActivity::CHThorLinkedRawIteratorActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorLinkedRawIteratorArg &_arg, ThorActivityKind _kind)
  6912. : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  6913. {
  6914. }
  6915. const void *CHThorLinkedRawIteratorActivity::nextRow()
  6916. {
  6917. const void *ret =helper.next();
  6918. if (ret)
  6919. {
  6920. LinkRoxieRow(ret);
  6921. processed++;
  6922. }
  6923. return ret;
  6924. }
  6925. //=====================================================================================================
  6926. //=====================================================================================================
  6927. //== New implementations - none are currently used, created or tested =================================
  6928. //=====================================================================================================
  6929. CHThorChildNormalizeActivity::CHThorChildNormalizeActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorChildNormalizeArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  6930. {
  6931. }
  6932. const void *CHThorChildNormalizeActivity::nextRow()
  6933. {
  6934. if (eof)
  6935. return NULL;
  6936. bool ok;
  6937. if (!started)
  6938. {
  6939. ok = helper.first();
  6940. started = true;
  6941. }
  6942. else
  6943. ok = helper.next();
  6944. try
  6945. {
  6946. if (ok)
  6947. {
  6948. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  6949. do {
  6950. unsigned thisSize = helper.transform(rowBuilder);
  6951. if (thisSize)
  6952. {
  6953. processed++;
  6954. return rowBuilder.finalizeRowClear(thisSize);
  6955. }
  6956. ok = helper.next();
  6957. }
  6958. while (ok);
  6959. }
  6960. }
  6961. catch(IException * e)
  6962. {
  6963. throw makeWrappedException(e);
  6964. }
  6965. eof = true;
  6966. return NULL;
  6967. }
  6968. void CHThorChildNormalizeActivity::ready()
  6969. {
  6970. CHThorSimpleActivityBase::ready();
  6971. started = false;
  6972. eof = false;
  6973. }
  6974. //=====================================================================================================
  6975. CHThorChildAggregateActivity::CHThorChildAggregateActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorChildAggregateArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  6976. {
  6977. }
  6978. const void *CHThorChildAggregateActivity::nextRow()
  6979. {
  6980. if (eof)
  6981. return NULL;
  6982. eof = true;
  6983. processed++;
  6984. try
  6985. {
  6986. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  6987. helper.clearAggregate(rowBuilder);
  6988. helper.processRows(rowBuilder);
  6989. size32_t finalSize = outputMeta.getRecordSize(rowBuilder.getSelf());
  6990. return rowBuilder.finalizeRowClear(finalSize);
  6991. }
  6992. catch(IException * e)
  6993. {
  6994. throw makeWrappedException(e);
  6995. }
  6996. }
  6997. void CHThorChildAggregateActivity::ready()
  6998. {
  6999. CHThorSimpleActivityBase::ready();
  7000. eof = false;
  7001. }
  7002. //=====================================================================================================
  7003. CHThorChildGroupAggregateActivity::CHThorChildGroupAggregateActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorChildGroupAggregateArg &_arg, ThorActivityKind _kind)
  7004. : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind),
  7005. helper(_arg),
  7006. aggregated(_arg, _arg)
  7007. {
  7008. }
  7009. void CHThorChildGroupAggregateActivity::ready()
  7010. {
  7011. CHThorSimpleActivityBase::ready();
  7012. eof = false;
  7013. gathered = false;
  7014. aggregated.start(rowAllocator);
  7015. }
  7016. void CHThorChildGroupAggregateActivity::stop()
  7017. {
  7018. aggregated.reset();
  7019. CHThorSimpleActivityBase::stop();
  7020. }
  7021. void CHThorChildGroupAggregateActivity::processRow(const void * next)
  7022. {
  7023. aggregated.addRow(next);
  7024. }
  7025. const void * CHThorChildGroupAggregateActivity::nextRow()
  7026. {
  7027. if (eof)
  7028. return NULL;
  7029. if (!gathered)
  7030. {
  7031. helper.processRows(this);
  7032. gathered = true;
  7033. }
  7034. Owned<AggregateRowBuilder> next = aggregated.nextResult();
  7035. if (next)
  7036. {
  7037. processed++;
  7038. return next->finalizeRowClear();
  7039. }
  7040. eof = true;
  7041. return NULL;
  7042. }
  7043. //=====================================================================================================
  7044. CHThorChildThroughNormalizeActivity::CHThorChildThroughNormalizeActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorChildThroughNormalizeArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg), outBuilder(NULL)
  7045. {
  7046. }
  7047. void CHThorChildThroughNormalizeActivity::stop()
  7048. {
  7049. outBuilder.clear();
  7050. lastInput.clear();
  7051. CHThorSimpleActivityBase::stop();
  7052. }
  7053. void CHThorChildThroughNormalizeActivity::ready()
  7054. {
  7055. CHThorSimpleActivityBase::ready();
  7056. outBuilder.setAllocator(rowAllocator);
  7057. numProcessedLastGroup = processed;
  7058. ok = false;
  7059. }
  7060. const void *CHThorChildThroughNormalizeActivity::nextRow()
  7061. {
  7062. try
  7063. {
  7064. for (;;)
  7065. {
  7066. if (ok)
  7067. ok = helper.next();
  7068. while (!ok)
  7069. {
  7070. lastInput.setown(input->nextRow());
  7071. if (!lastInput)
  7072. {
  7073. if (numProcessedLastGroup != processed)
  7074. {
  7075. numProcessedLastGroup = processed;
  7076. return NULL;
  7077. }
  7078. lastInput.setown(input->nextRow());
  7079. if (!lastInput)
  7080. return NULL;
  7081. }
  7082. ok = helper.first(lastInput);
  7083. }
  7084. outBuilder.ensureRow();
  7085. do
  7086. {
  7087. size32_t thisSize = helper.transform(outBuilder);
  7088. if (thisSize)
  7089. {
  7090. processed++;
  7091. return outBuilder.finalizeRowClear(thisSize);
  7092. }
  7093. ok = helper.next();
  7094. } while (ok);
  7095. }
  7096. }
  7097. catch(IException * e)
  7098. {
  7099. throw makeWrappedException(e);
  7100. }
  7101. }
  7102. //=====================================================================================================
  7103. CHThorDiskReadBaseActivity::CHThorDiskReadBaseActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorDiskReadBaseArg &_arg, ThorActivityKind _kind) : CHThorActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  7104. {
  7105. helper.setCallback(this);
  7106. }
  7107. CHThorDiskReadBaseActivity::~CHThorDiskReadBaseActivity()
  7108. {
  7109. close();
  7110. }
  7111. void CHThorDiskReadBaseActivity::ready()
  7112. {
  7113. CHThorActivityBase::ready();
  7114. grouped = false;
  7115. recordsize = 0;
  7116. fixedDiskRecordSize = 0;
  7117. eofseen = false;
  7118. opened = false;
  7119. compressed = false;
  7120. rowcompressed = false;
  7121. blockcompressed = false;
  7122. persistent = false;
  7123. localOffset = 0;
  7124. offsetOfPart = 0;
  7125. partNum = (unsigned)-1;
  7126. resolve();
  7127. }
  7128. void CHThorDiskReadBaseActivity::stop()
  7129. {
  7130. close();
  7131. CHThorActivityBase::stop();
  7132. }
  7133. void CHThorDiskReadBaseActivity::resolve()
  7134. {
  7135. OwnedRoxieString fileName(helper.getFileName());
  7136. mangleHelperFileName(mangledHelperFileName, fileName, agent.queryWuid(), helper.getFlags());
  7137. if (helper.getFlags() & (TDXtemporary | TDXjobtemp))
  7138. {
  7139. StringBuffer mangledFilename;
  7140. mangleLocalTempFilename(mangledFilename, mangledHelperFileName.str());
  7141. tempFileName.set(agent.queryTemporaryFile(mangledFilename.str()));
  7142. logicalFileName.set(tempFileName);
  7143. gatherInfo(NULL);
  7144. }
  7145. else
  7146. {
  7147. ldFile.setown(agent.resolveLFN(mangledHelperFileName.str(), "Read", 0 != (helper.getFlags() & TDRoptional)));
  7148. if ( mangledHelperFileName.charAt(0) == '~')
  7149. logicalFileName.set(mangledHelperFileName.str()+1);
  7150. else
  7151. logicalFileName.set(mangledHelperFileName.str());
  7152. if (ldFile)
  7153. {
  7154. Owned<IFileDescriptor> fdesc;
  7155. fdesc.setown(ldFile->getFileDescriptor());
  7156. gatherInfo(fdesc);
  7157. IDistributedFile *dFile = ldFile->queryDistributedFile();
  7158. if (dFile) //only makes sense for distributed (non local) files
  7159. {
  7160. persistent = dFile->queryAttributes().getPropBool("@persistent");
  7161. dfsParts.setown(dFile->getIterator());
  7162. if (helper.getFlags() & TDRfilenamecallback)
  7163. {
  7164. IDistributedSuperFile *super = dFile->querySuperFile();
  7165. if (super)
  7166. {
  7167. unsigned numsubs = super->numSubFiles(true);
  7168. unsigned s=0;
  7169. for (; s<numsubs; s++)
  7170. {
  7171. IDistributedFile &subfile = super->querySubFile(s, true);
  7172. subfileLogicalFilenames.append(subfile.queryLogicalName());
  7173. }
  7174. assertex(fdesc);
  7175. superfile.set(fdesc->querySuperFileDescriptor());
  7176. if (!superfile && numsubs>0)
  7177. logicalFileName.set(subfileLogicalFilenames.item(0));
  7178. }
  7179. }
  7180. if((helper.getFlags() & (TDXtemporary | TDXjobtemp)) == 0)
  7181. agent.logFileAccess(dFile, "HThor", "READ");
  7182. if(!agent.queryWorkUnit()->getDebugValueBool("skipFileFormatCrcCheck", false) && !(helper.getFlags() & TDRnocrccheck))
  7183. verifyRecordFormatCrc();
  7184. }
  7185. }
  7186. if (!ldFile)
  7187. {
  7188. StringBuffer buff;
  7189. buff.appendf("Input file '%s' was missing but declared optional", mangledHelperFileName.str());
  7190. WARNLOG("%s", buff.str());
  7191. agent.addWuException(buff.str(), WRN_SkipMissingOptFile, SeverityInformation, "hthor");
  7192. }
  7193. }
  7194. }
  7195. void CHThorDiskReadBaseActivity::gatherInfo(IFileDescriptor * fileDesc)
  7196. {
  7197. if(fileDesc)
  7198. {
  7199. if (!agent.queryResolveFilesLocally())
  7200. {
  7201. grouped = fileDesc->isGrouped();
  7202. if(grouped != ((helper.getFlags() & TDXgrouped) != 0))
  7203. {
  7204. StringBuffer msg;
  7205. msg.append("DFS and code generated group info. differs: DFS(").append(grouped ? "grouped" : "ungrouped").append("), CodeGen(").append(grouped ? "ungrouped" : "grouped").append("), using DFS info");
  7206. WARNLOG("%s", msg.str());
  7207. agent.addWuException(msg.str(), WRN_MismatchGroupInfo, SeverityError, "hthor");
  7208. }
  7209. }
  7210. else
  7211. grouped = ((helper.getFlags() & TDXgrouped) != 0);
  7212. }
  7213. else
  7214. {
  7215. grouped = ((helper.getFlags() & TDXgrouped) != 0);
  7216. }
  7217. diskMeta.set(helper.queryDiskRecordSize()->querySerializedDiskMeta());
  7218. if (grouped)
  7219. diskMeta.setown(new CSuffixedOutputMeta(+1, diskMeta));
  7220. if (outputMeta.isFixedSize())
  7221. {
  7222. recordsize = outputMeta.getFixedSize();
  7223. if (grouped)
  7224. recordsize++;
  7225. }
  7226. else
  7227. recordsize = 0;
  7228. calcFixedDiskRecordSize();
  7229. if(fileDesc)
  7230. {
  7231. compressed = fileDesc->isCompressed(&blockcompressed); //try new decompression, fall back to old unless marked as block
  7232. if(fixedDiskRecordSize)
  7233. {
  7234. size32_t dfsSize = fileDesc->queryProperties().getPropInt("@recordSize");
  7235. if(!((dfsSize == 0) || (dfsSize == fixedDiskRecordSize) || (grouped && (dfsSize+1 == fixedDiskRecordSize)))) //third option for backwards compatibility, as hthor used to publish @recordSize not including the grouping byte
  7236. throw MakeStringException(0, "Published record size %d for file %s does not match coded record size %d", dfsSize, mangledHelperFileName.str(), fixedDiskRecordSize);
  7237. if (!compressed && (((helper.getFlags() & TDXcompress) != 0) && (fixedDiskRecordSize >= MIN_ROWCOMPRESS_RECSIZE)))
  7238. {
  7239. StringBuffer msg;
  7240. msg.append("Ignoring compression attribute on file ").append(mangledHelperFileName.str()).append(", which is not published as compressed");
  7241. WARNLOG("%s", msg.str());
  7242. agent.addWuException(msg.str(), WRN_MismatchCompressInfo, SeverityWarning, "hthor");
  7243. compressed = true;
  7244. }
  7245. }
  7246. }
  7247. else
  7248. {
  7249. compressed = checkIsCompressed(helper.getFlags(), fixedDiskRecordSize, false);//grouped=FALSE because fixedDiskRecordSize already includes grouped
  7250. }
  7251. void *k;
  7252. size32_t kl;
  7253. helper.getEncryptKey(kl,k);
  7254. encryptionkey.setOwn(kl,k);
  7255. if (encryptionkey.length()!=0)
  7256. {
  7257. blockcompressed = true;
  7258. compressed = true;
  7259. }
  7260. }
  7261. void CHThorDiskReadBaseActivity::close()
  7262. {
  7263. closepart();
  7264. tempFileName.clear();
  7265. dfsParts.clear();
  7266. if(ldFile)
  7267. {
  7268. IDistributedFile * dFile = ldFile->queryDistributedFile();
  7269. if(dFile)
  7270. dFile->setAccessed();
  7271. ldFile.clear();
  7272. }
  7273. }
  7274. unsigned __int64 CHThorDiskReadBaseActivity::getFilePosition(const void * row)
  7275. {
  7276. return localOffset + offsetOfPart;
  7277. }
  7278. unsigned __int64 CHThorDiskReadBaseActivity::getLocalFilePosition(const void * row)
  7279. {
  7280. return makeLocalFposOffset(partNum-1, localOffset);
  7281. }
  7282. void CHThorDiskReadBaseActivity::closepart()
  7283. {
  7284. inputstream.clear();
  7285. inputfileio.clear();
  7286. inputfile.clear();
  7287. }
  7288. bool CHThorDiskReadBaseActivity::openNext()
  7289. {
  7290. offsetOfPart += localOffset;
  7291. localOffset = 0;
  7292. saveOpenExc.clear();
  7293. if (dfsParts||ldFile)
  7294. {
  7295. // open next part of a multipart, if there is one
  7296. while ((dfsParts&&dfsParts->isValid())||
  7297. (!dfsParts&&(partNum<ldFile->numParts())))
  7298. {
  7299. IDistributedFilePart * curPart = dfsParts?&dfsParts->query():NULL;
  7300. unsigned numCopies = curPart?curPart->numCopies():ldFile->numPartCopies(partNum);
  7301. //MORE: Order of copies should be optimized at this point....
  7302. StringBuffer file, filelist;
  7303. closepart();
  7304. if (dfsParts && superfile && curPart)
  7305. {
  7306. unsigned subfile;
  7307. unsigned lnum;
  7308. if (superfile->mapSubPart(partNum, subfile, lnum))
  7309. logicalFileName.set(subfileLogicalFilenames.item(subfile));
  7310. }
  7311. for (unsigned copy=0; copy < numCopies; copy++)
  7312. {
  7313. RemoteFilename rfilename;
  7314. if (curPart)
  7315. curPart->getFilename(rfilename,copy);
  7316. else
  7317. ldFile->getPartFilename(rfilename,partNum,copy);
  7318. rfilename.getPath(file.clear());
  7319. filelist.append('\n').append(file);
  7320. try
  7321. {
  7322. inputfile.setown(createIFile(rfilename));
  7323. if(compressed)
  7324. {
  7325. Owned<IExpander> eexp;
  7326. if (encryptionkey.length()!=0)
  7327. eexp.setown(createAESExpander256((size32_t)encryptionkey.length(),encryptionkey.bufferBase()));
  7328. inputfileio.setown(createCompressedFileReader(inputfile,eexp));
  7329. if(!inputfileio && !blockcompressed) //fall back to old decompression, unless dfs marked as new
  7330. {
  7331. inputfileio.setown(inputfile->open(IFOread));
  7332. if(inputfileio)
  7333. rowcompressed = true;
  7334. }
  7335. }
  7336. else
  7337. inputfileio.setown(inputfile->open(IFOread));
  7338. if (inputfileio)
  7339. break;
  7340. }
  7341. catch (IException *E)
  7342. {
  7343. if (saveOpenExc.get())
  7344. E->Release();
  7345. else
  7346. saveOpenExc.setown(E);
  7347. }
  7348. closepart();
  7349. }
  7350. if (dfsParts)
  7351. dfsParts->next();
  7352. partNum++;
  7353. if (checkOpenedFile(file.str(), filelist.str()))
  7354. {
  7355. opened = true;
  7356. return true;
  7357. }
  7358. }
  7359. return false;
  7360. }
  7361. else if (!tempFileName.isEmpty())
  7362. {
  7363. StringBuffer file(tempFileName.get());
  7364. tempFileName.clear();
  7365. closepart();
  7366. try
  7367. {
  7368. inputfile.setown(createIFile(file.str()));
  7369. if(compressed)
  7370. {
  7371. Owned<IExpander> eexp;
  7372. if (encryptionkey.length())
  7373. eexp.setown(createAESExpander256((size32_t) encryptionkey.length(),encryptionkey.bufferBase()));
  7374. inputfileio.setown(createCompressedFileReader(inputfile,eexp));
  7375. if(!inputfileio && !blockcompressed) //fall back to old decompression, unless dfs marked as new
  7376. {
  7377. inputfileio.setown(inputfile->open(IFOread));
  7378. if(inputfileio)
  7379. rowcompressed = true;
  7380. }
  7381. }
  7382. else
  7383. inputfileio.setown(inputfile->open(IFOread));
  7384. }
  7385. catch (IException *E)
  7386. {
  7387. closepart();
  7388. StringBuffer msg;
  7389. WARNLOG("%s", E->errorMessage(msg).str());
  7390. if (saveOpenExc.get())
  7391. E->Release();
  7392. else
  7393. saveOpenExc.setown(E);
  7394. }
  7395. partNum++;
  7396. if (checkOpenedFile(file.str(), NULL))
  7397. {
  7398. opened = true;
  7399. return true;
  7400. }
  7401. }
  7402. return false;
  7403. }
  7404. bool CHThorDiskReadBaseActivity::checkOpenedFile(char const * filename, char const * filenamelist)
  7405. {
  7406. unsigned __int64 filesize = 0;
  7407. if (!inputfileio)
  7408. {
  7409. if (!(helper.getFlags() & TDRoptional))
  7410. {
  7411. StringBuffer s;
  7412. if(filenamelist) {
  7413. if (saveOpenExc.get())
  7414. {
  7415. if (strstr(mangledHelperFileName.str(),"::>")!=NULL) // if a 'special' filename just use saved exception
  7416. saveOpenExc->errorMessage(s);
  7417. else
  7418. {
  7419. s.append("Could not open logical file ").append(mangledHelperFileName.str()).append(" in any of these locations:").append(filenamelist).append(" (");
  7420. saveOpenExc->errorMessage(s).append(")");
  7421. }
  7422. }
  7423. else
  7424. s.append("Could not open logical file ").append(mangledHelperFileName.str()).append(" in any of these locations:").append(filenamelist).append(" (").append((unsigned)GetLastError()).append(")");
  7425. }
  7426. else
  7427. s.append("Could not open local physical file ").append(filename).append(" (").append((unsigned)GetLastError()).append(")");
  7428. agent.fail(1, s.str());
  7429. }
  7430. }
  7431. else
  7432. filesize = inputfileio->size();
  7433. saveOpenExc.clear();
  7434. if (filesize)
  7435. {
  7436. if (!compressed && fixedDiskRecordSize && (filesize % fixedDiskRecordSize) != 0)
  7437. {
  7438. StringBuffer s;
  7439. s.append("File ").append(filename).append(" size is ").append(filesize).append(" which is not a multiple of ").append(fixedDiskRecordSize);
  7440. agent.fail(1, s.str());
  7441. }
  7442. unsigned readBufferSize = queryReadBufferSize();
  7443. inputstream.setown(createFileSerialStream(inputfileio, 0, filesize, readBufferSize));
  7444. StringBuffer report("Reading file ");
  7445. report.append(inputfile->queryFilename());
  7446. agent.reportProgress(report.str());
  7447. }
  7448. return (filesize != 0);
  7449. }
  7450. void CHThorDiskReadBaseActivity::open()
  7451. {
  7452. assertex(!opened);
  7453. partNum = 0;
  7454. if (dfsParts)
  7455. eofseen = !dfsParts->first() || !openNext();
  7456. else if (ldFile||tempFileName.length())
  7457. eofseen = !openNext();
  7458. else
  7459. eofseen = true;
  7460. opened = true;
  7461. }
  7462. //=====================================================================================================
  7463. CHThorBinaryDiskReadBase::CHThorBinaryDiskReadBase(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorDiskReadBaseArg &_arg, IHThorCompoundBaseArg & _segHelper, ThorActivityKind _kind)
  7464. : CHThorDiskReadBaseActivity(_agent, _activityId, _subgraphId, _arg, _kind),
  7465. segHelper(_segHelper), prefetchBuffer(NULL),
  7466. recInfo(outputMeta.queryRecordAccessor(true)), // MORE - is this right - should it be diskMeta->queryRecordAccessor() ?
  7467. rowInfo(recInfo)
  7468. {
  7469. }
  7470. void CHThorBinaryDiskReadBase::calcFixedDiskRecordSize()
  7471. {
  7472. fixedDiskRecordSize = diskMeta->getFixedSize();
  7473. }
  7474. void CHThorBinaryDiskReadBase::append(IKeySegmentMonitor *segment)
  7475. {
  7476. if (segment->isWild())
  7477. segment->Release();
  7478. else
  7479. {
  7480. segMonitors.append(*segment);
  7481. if (segment->numFieldsRequired() > numFieldsRequired)
  7482. numFieldsRequired = segment->numFieldsRequired();
  7483. }
  7484. }
  7485. unsigned CHThorBinaryDiskReadBase::ordinality() const
  7486. {
  7487. return segMonitors.length();
  7488. }
  7489. IKeySegmentMonitor *CHThorBinaryDiskReadBase::item(unsigned idx) const
  7490. {
  7491. if (segMonitors.isItem(idx))
  7492. return &segMonitors.item(idx);
  7493. else
  7494. return NULL;
  7495. }
  7496. void CHThorBinaryDiskReadBase::ready()
  7497. {
  7498. CHThorDiskReadBaseActivity::ready();
  7499. if (!diskMeta)
  7500. diskMeta.set(outputMeta);
  7501. segMonitors.kill();
  7502. numFieldsRequired = 0;
  7503. segHelper.createSegmentMonitors(this);
  7504. prefetcher.setown(diskMeta->createDiskPrefetcher(agent.queryCodeContext(), activityId));
  7505. deserializer.setown(diskMeta->createDiskDeserializer(agent.queryCodeContext(), activityId));
  7506. }
  7507. bool CHThorBinaryDiskReadBase::openNext()
  7508. {
  7509. if (CHThorDiskReadBaseActivity::openNext())
  7510. {
  7511. if(rowcompressed && fixedDiskRecordSize)
  7512. {
  7513. throwUnexpected();
  7514. //MORE: What happens here
  7515. PROGLOG("Disk read falling back to legacy decompression routine");
  7516. //in.setown(createRowCompReadSeq(*inputfileiostream, 0, fixedDiskRecordSize));
  7517. }
  7518. //Only one of these will actually be used.
  7519. prefetchBuffer.setStream(inputstream);
  7520. deserializeSource.setStream(inputstream);
  7521. return true;
  7522. }
  7523. return false;
  7524. }
  7525. void CHThorBinaryDiskReadBase::closepart()
  7526. {
  7527. prefetchBuffer.clearStream();
  7528. deserializeSource.clearStream();
  7529. CHThorDiskReadBaseActivity::closepart();
  7530. }
  7531. unsigned CHThorBinaryDiskReadBase::queryReadBufferSize()
  7532. {
  7533. return hthorReadBufferSize;
  7534. }
  7535. void CHThorBinaryDiskReadBase::open()
  7536. {
  7537. if (!segHelper.canMatchAny())
  7538. {
  7539. eofseen = true;
  7540. opened = true;
  7541. }
  7542. else
  7543. CHThorDiskReadBaseActivity::open();
  7544. }
  7545. //=====================================================================================================
  7546. CHThorDiskReadActivity::CHThorDiskReadActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorDiskReadArg &_arg, ThorActivityKind _kind) : CHThorBinaryDiskReadBase(_agent, _activityId, _subgraphId, _arg, _arg, _kind), helper(_arg), outBuilder(NULL)
  7547. {
  7548. needTransform = false;
  7549. eogPending = 0;
  7550. lastGroupProcessed = 0;
  7551. }
  7552. void CHThorDiskReadActivity::ready()
  7553. {
  7554. PARENT::ready();
  7555. outBuilder.setAllocator(rowAllocator);
  7556. eogPending = false;
  7557. lastGroupProcessed = processed;
  7558. needTransform = helper.needTransform() || segMonitors.length();
  7559. limit = helper.getRowLimit();
  7560. if (helper.getFlags() & TDRlimitskips)
  7561. limit = (unsigned __int64) -1;
  7562. stopAfter = helper.getChooseNLimit();
  7563. }
  7564. void CHThorDiskReadActivity::stop()
  7565. {
  7566. outBuilder.clear();
  7567. PARENT::stop();
  7568. }
  7569. const void *CHThorDiskReadActivity::nextRow()
  7570. {
  7571. if (!opened) open();
  7572. if (eogPending && (lastGroupProcessed != processed))
  7573. {
  7574. eogPending = false;
  7575. lastGroupProcessed = processed;
  7576. return NULL;
  7577. }
  7578. try
  7579. {
  7580. if (needTransform || grouped)
  7581. {
  7582. while (!eofseen && ((stopAfter == 0) || ((processed - initialProcessed) < stopAfter)))
  7583. {
  7584. queryUpdateProgress();
  7585. while (!prefetchBuffer.eos())
  7586. {
  7587. queryUpdateProgress();
  7588. prefetcher->readAhead(prefetchBuffer);
  7589. const byte * next = prefetchBuffer.queryRow();
  7590. size32_t sizeRead = prefetchBuffer.queryRowSize();
  7591. size32_t thisSize;
  7592. if (segMonitorsMatch(next))
  7593. {
  7594. thisSize = helper.transform(outBuilder.ensureRow(), next);
  7595. }
  7596. else
  7597. thisSize = 0;
  7598. bool eog = grouped && next[sizeRead-1];
  7599. prefetchBuffer.finishedRow();
  7600. localOffset += sizeRead;
  7601. if (thisSize)
  7602. {
  7603. if (grouped)
  7604. eogPending = eog;
  7605. if ((processed - initialProcessed) >=limit)
  7606. {
  7607. outBuilder.clear();
  7608. if ( agent.queryCodeContext()->queryDebugContext())
  7609. agent.queryCodeContext()->queryDebugContext()->checkBreakpoint(DebugStateLimit, NULL, static_cast<IActivityBase *>(this));
  7610. helper.onLimitExceeded();
  7611. return NULL;
  7612. }
  7613. processed++;
  7614. return outBuilder.finalizeRowClear(thisSize);
  7615. }
  7616. if (eog && (lastGroupProcessed != processed))
  7617. {
  7618. lastGroupProcessed = processed;
  7619. return NULL;
  7620. }
  7621. }
  7622. eofseen = !openNext();
  7623. }
  7624. }
  7625. else
  7626. {
  7627. assertex(outputMeta == diskMeta);
  7628. while(!eofseen && ((stopAfter == 0) || (processed - initialProcessed) < stopAfter))
  7629. {
  7630. queryUpdateProgress();
  7631. if (!inputstream->eos())
  7632. {
  7633. size32_t sizeRead = deserializer->deserialize(outBuilder.ensureRow(), deserializeSource);
  7634. //In this case size read from disk == size created in memory
  7635. localOffset += sizeRead;
  7636. if ((processed - initialProcessed)>=limit)
  7637. {
  7638. outBuilder.clear();
  7639. if ( agent.queryCodeContext()->queryDebugContext())
  7640. agent.queryCodeContext()->queryDebugContext()->checkBreakpoint(DebugStateLimit, NULL, static_cast<IActivityBase *>(this));
  7641. helper.onLimitExceeded();
  7642. return NULL;
  7643. }
  7644. processed++;
  7645. return outBuilder.finalizeRowClear(sizeRead);
  7646. }
  7647. eofseen = !openNext();
  7648. }
  7649. }
  7650. close();
  7651. }
  7652. catch(IException * e)
  7653. {
  7654. throw makeWrappedException(e);
  7655. }
  7656. return NULL;
  7657. }
  7658. //=====================================================================================================
  7659. CHThorDiskNormalizeActivity::CHThorDiskNormalizeActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorDiskNormalizeArg &_arg, ThorActivityKind _kind) : CHThorBinaryDiskReadBase(_agent, _activityId, _subgraphId, _arg, _arg, _kind), helper(_arg), outBuilder(NULL)
  7660. {
  7661. }
  7662. void CHThorDiskNormalizeActivity::stop()
  7663. {
  7664. outBuilder.clear();
  7665. PARENT::stop();
  7666. }
  7667. void CHThorDiskNormalizeActivity::ready()
  7668. {
  7669. PARENT::ready();
  7670. outBuilder.setAllocator(rowAllocator);
  7671. limit = helper.getRowLimit();
  7672. if (helper.getFlags() & TDRlimitskips)
  7673. limit = (unsigned __int64) -1;
  7674. stopAfter = helper.getChooseNLimit();
  7675. lastSizeRead = 0;
  7676. expanding = false;
  7677. }
  7678. void CHThorDiskNormalizeActivity::gatherInfo(IFileDescriptor * fd)
  7679. {
  7680. PARENT::gatherInfo(fd);
  7681. assertex(!grouped);
  7682. }
  7683. const void *CHThorDiskNormalizeActivity::nextRow()
  7684. {
  7685. if (!opened) open();
  7686. for (;;)
  7687. {
  7688. if (eofseen || (stopAfter && (processed - initialProcessed) >= stopAfter))
  7689. break;
  7690. for (;;)
  7691. {
  7692. if (expanding)
  7693. {
  7694. for (;;)
  7695. {
  7696. expanding = helper.next();
  7697. if (!expanding)
  7698. break;
  7699. const void * ret = createNextRow();
  7700. if (ret)
  7701. return ret;
  7702. }
  7703. }
  7704. localOffset += lastSizeRead;
  7705. prefetchBuffer.finishedRow();
  7706. if (inputstream->eos())
  7707. {
  7708. lastSizeRead = 0;
  7709. break;
  7710. }
  7711. prefetcher->readAhead(prefetchBuffer);
  7712. const byte * next = prefetchBuffer.queryRow();
  7713. lastSizeRead = prefetchBuffer.queryRowSize();
  7714. queryUpdateProgress();
  7715. if (segMonitorsMatch(next))
  7716. {
  7717. try
  7718. {
  7719. expanding = helper.first(next);
  7720. }
  7721. catch(IException * e)
  7722. {
  7723. throw makeWrappedException(e);
  7724. }
  7725. if (expanding)
  7726. {
  7727. const void * ret = createNextRow();
  7728. if (ret)
  7729. return ret;
  7730. }
  7731. }
  7732. }
  7733. eofseen = !openNext();
  7734. }
  7735. close();
  7736. return NULL;
  7737. }
  7738. const void * CHThorDiskNormalizeActivity::createNextRow()
  7739. {
  7740. try
  7741. {
  7742. size32_t thisSize = helper.transform(outBuilder.ensureRow());
  7743. if (thisSize == 0)
  7744. return NULL;
  7745. if ((processed - initialProcessed) >=limit)
  7746. {
  7747. outBuilder.clear();
  7748. if ( agent.queryCodeContext()->queryDebugContext())
  7749. agent.queryCodeContext()->queryDebugContext()->checkBreakpoint(DebugStateLimit, NULL, static_cast<IActivityBase *>(this));
  7750. helper.onLimitExceeded();
  7751. return NULL;
  7752. }
  7753. processed++;
  7754. return outBuilder.finalizeRowClear(thisSize);
  7755. }
  7756. catch(IException * e)
  7757. {
  7758. throw makeWrappedException(e);
  7759. }
  7760. }
  7761. //=====================================================================================================
  7762. CHThorDiskAggregateActivity::CHThorDiskAggregateActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorDiskAggregateArg &_arg, ThorActivityKind _kind) : CHThorBinaryDiskReadBase(_agent, _activityId, _subgraphId, _arg, _arg, _kind), helper(_arg), outBuilder(NULL)
  7763. {
  7764. }
  7765. void CHThorDiskAggregateActivity::stop()
  7766. {
  7767. outBuilder.clear();
  7768. PARENT::stop();
  7769. }
  7770. void CHThorDiskAggregateActivity::ready()
  7771. {
  7772. PARENT::ready();
  7773. outBuilder.setAllocator(rowAllocator);
  7774. finished = false;
  7775. }
  7776. void CHThorDiskAggregateActivity::gatherInfo(IFileDescriptor * fd)
  7777. {
  7778. PARENT::gatherInfo(fd);
  7779. assertex(!grouped);
  7780. }
  7781. const void *CHThorDiskAggregateActivity::nextRow()
  7782. {
  7783. if (finished) return NULL;
  7784. try
  7785. {
  7786. if (!opened) open();
  7787. outBuilder.ensureRow();
  7788. helper.clearAggregate(outBuilder);
  7789. while (!eofseen)
  7790. {
  7791. while (!inputstream->eos())
  7792. {
  7793. queryUpdateProgress();
  7794. prefetcher->readAhead(prefetchBuffer);
  7795. const byte * next = prefetchBuffer.queryRow();
  7796. size32_t sizeRead = prefetchBuffer.queryRowSize();
  7797. if (segMonitorsMatch(next))
  7798. helper.processRow(outBuilder, next);
  7799. prefetchBuffer.finishedRow();
  7800. localOffset += sizeRead;
  7801. }
  7802. eofseen = !openNext();
  7803. }
  7804. close();
  7805. processed++;
  7806. finished = true;
  7807. unsigned retSize = outputMeta.getRecordSize(outBuilder.getSelf());
  7808. return outBuilder.finalizeRowClear(retSize);
  7809. }
  7810. catch(IException * e)
  7811. {
  7812. throw makeWrappedException(e);
  7813. }
  7814. }
  7815. //=====================================================================================================
  7816. CHThorDiskCountActivity::CHThorDiskCountActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorDiskCountArg &_arg, ThorActivityKind _kind) : CHThorBinaryDiskReadBase(_agent, _activityId, _subgraphId, _arg, _arg, _kind), helper(_arg)
  7817. {
  7818. finished = true;
  7819. }
  7820. CHThorDiskCountActivity::~CHThorDiskCountActivity()
  7821. {
  7822. }
  7823. void CHThorDiskCountActivity::ready()
  7824. {
  7825. PARENT::ready();
  7826. finished = false;
  7827. stopAfter = helper.getChooseNLimit();
  7828. }
  7829. void CHThorDiskCountActivity::gatherInfo(IFileDescriptor * fd)
  7830. {
  7831. PARENT::gatherInfo(fd);
  7832. assertex(!grouped);
  7833. }
  7834. const void *CHThorDiskCountActivity::nextRow()
  7835. {
  7836. if (finished) return NULL;
  7837. unsigned __int64 totalCount = 0;
  7838. if ((segMonitors.ordinality() == 0) && !helper.hasFilter() && (fixedDiskRecordSize != 0) && !(helper.getFlags() & (TDXtemporary | TDXjobtemp))
  7839. && !((helper.getFlags() & TDXcompress) && agent.queryResolveFilesLocally()) )
  7840. {
  7841. resolve();
  7842. if (segHelper.canMatchAny() && ldFile)
  7843. {
  7844. unsigned __int64 size = ldFile->getFileSize();
  7845. if (size % fixedDiskRecordSize)
  7846. throw MakeStringException(0, "Physical file %s has size %" I64F "d which is not a multiple of record size %d", ldFile->queryLogicalName(), size, fixedDiskRecordSize);
  7847. totalCount = size / fixedDiskRecordSize;
  7848. }
  7849. }
  7850. else
  7851. {
  7852. if (!opened) open();
  7853. for (;;)
  7854. {
  7855. if (eofseen)
  7856. break;
  7857. while (!inputstream->eos())
  7858. {
  7859. queryUpdateProgress();
  7860. prefetcher->readAhead(prefetchBuffer);
  7861. const byte * next = prefetchBuffer.queryRow();
  7862. size32_t sizeRead = prefetchBuffer.queryRowSize();
  7863. if (segMonitorsMatch(next))
  7864. totalCount += helper.numValid(next);
  7865. prefetchBuffer.finishedRow();
  7866. localOffset += sizeRead;
  7867. if (totalCount > stopAfter)
  7868. break;
  7869. }
  7870. if (totalCount > stopAfter)
  7871. break;
  7872. eofseen = !openNext();
  7873. }
  7874. close();
  7875. }
  7876. if (totalCount > stopAfter)
  7877. totalCount = stopAfter;
  7878. finished = true;
  7879. processed++;
  7880. size32_t outSize = outputMeta.getFixedSize();
  7881. void * ret = rowAllocator->createRow();
  7882. if (outSize == 1)
  7883. {
  7884. assertex(stopAfter == 1);
  7885. *(byte *)ret = (byte)totalCount;
  7886. }
  7887. else
  7888. {
  7889. assertex(outSize == sizeof(unsigned __int64));
  7890. *(unsigned __int64 *)ret = totalCount;
  7891. }
  7892. return rowAllocator->finalizeRow(outSize, ret, outSize);
  7893. }
  7894. //=====================================================================================================
  7895. CHThorDiskGroupAggregateActivity::CHThorDiskGroupAggregateActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorDiskGroupAggregateArg &_arg, ThorActivityKind _kind)
  7896. : CHThorBinaryDiskReadBase(_agent, _activityId, _subgraphId, _arg, _arg, _kind),
  7897. helper(_arg),
  7898. aggregated(_arg, _arg)
  7899. {
  7900. }
  7901. void CHThorDiskGroupAggregateActivity::ready()
  7902. {
  7903. PARENT::ready();
  7904. eof = false;
  7905. gathered = false;
  7906. }
  7907. void CHThorDiskGroupAggregateActivity::gatherInfo(IFileDescriptor * fd)
  7908. {
  7909. PARENT::gatherInfo(fd);
  7910. assertex(!grouped);
  7911. aggregated.start(rowAllocator);
  7912. }
  7913. void CHThorDiskGroupAggregateActivity::processRow(const void * next)
  7914. {
  7915. aggregated.addRow(next);
  7916. }
  7917. const void *CHThorDiskGroupAggregateActivity::nextRow()
  7918. {
  7919. if (eof)
  7920. return NULL;
  7921. try
  7922. {
  7923. if (!gathered)
  7924. {
  7925. if (!opened) open();
  7926. while (!eofseen)
  7927. {
  7928. while (!inputstream->eos())
  7929. {
  7930. queryUpdateProgress();
  7931. prefetcher->readAhead(prefetchBuffer);
  7932. const byte * next = prefetchBuffer.queryRow();
  7933. size32_t sizeRead = prefetchBuffer.queryRowSize();
  7934. //helper.processRows(sizeRead, next, this);
  7935. helper.processRow(next, this);
  7936. prefetchBuffer.finishedRow();
  7937. localOffset += sizeRead;
  7938. }
  7939. eofseen = !openNext();
  7940. }
  7941. close();
  7942. gathered = true;
  7943. }
  7944. }
  7945. catch(IException * e)
  7946. {
  7947. throw makeWrappedException(e);
  7948. }
  7949. Owned<AggregateRowBuilder> next = aggregated.nextResult();
  7950. if (next)
  7951. {
  7952. processed++;
  7953. return next->finalizeRowClear();
  7954. }
  7955. eof = true;
  7956. return NULL;
  7957. }
  7958. //=====================================================================================================
  7959. CHThorCsvReadActivity::CHThorCsvReadActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorCsvReadArg &_arg, ThorActivityKind _kind) : CHThorDiskReadBaseActivity(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  7960. {
  7961. maxRowSize = agent.queryWorkUnit()->getDebugValueInt(OPT_MAXCSVROWSIZE, defaultMaxCsvRowSize) * 1024 * 1024;
  7962. }
  7963. CHThorCsvReadActivity::~CHThorCsvReadActivity()
  7964. {
  7965. }
  7966. void CHThorCsvReadActivity::ready()
  7967. {
  7968. PARENT::ready();
  7969. }
  7970. void CHThorCsvReadActivity::stop()
  7971. {
  7972. csvSplitter.reset();
  7973. PARENT::stop();
  7974. }
  7975. void CHThorCsvReadActivity::gatherInfo(IFileDescriptor * fd)
  7976. {
  7977. PARENT::gatherInfo(fd);
  7978. ICsvParameters * csvInfo = helper.queryCsvParameters();
  7979. headerLines = csvInfo->queryHeaderLen();
  7980. maxDiskSize = csvInfo->queryMaxSize();
  7981. limit = helper.getRowLimit();
  7982. if (helper.getFlags() & TDRlimitskips)
  7983. limit = (unsigned __int64) -1;
  7984. stopAfter = helper.getChooseNLimit();
  7985. const char * quotes = NULL;
  7986. const char * separators = NULL;
  7987. const char * terminators = NULL;
  7988. const char * escapes = NULL;
  7989. IDistributedFile * dFile = ldFile?ldFile->queryDistributedFile():NULL;
  7990. if (dFile) //only makes sense for distributed (non local) files
  7991. {
  7992. IPropertyTree & options = dFile->queryAttributes();
  7993. quotes = options.queryProp("@csvQuote");
  7994. separators = options.queryProp("@csvSeparate");
  7995. terminators = options.queryProp("@csvTerminate");
  7996. escapes = options.queryProp("@csvEscape");
  7997. }
  7998. csvSplitter.init(helper.getMaxColumns(), csvInfo, quotes, separators, terminators, escapes);
  7999. }
  8000. void CHThorCsvReadActivity::calcFixedDiskRecordSize()
  8001. {
  8002. fixedDiskRecordSize = 0;
  8003. }
  8004. const void *CHThorCsvReadActivity::nextRow()
  8005. {
  8006. while (!stopAfter || (processed - initialProcessed) < stopAfter)
  8007. {
  8008. checkOpenNext();
  8009. if (eofseen)
  8010. break;
  8011. size32_t thisLineLength = csvSplitter.splitLine(inputstream, maxRowSize);
  8012. if (thisLineLength)
  8013. {
  8014. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  8015. unsigned thisSize;
  8016. try
  8017. {
  8018. thisSize = helper.transform(rowBuilder, csvSplitter.queryLengths(), (const char * *)csvSplitter.queryData());
  8019. }
  8020. catch(IException * e)
  8021. {
  8022. throw makeWrappedException(e);
  8023. }
  8024. inputstream->skip(thisLineLength);
  8025. localOffset += thisLineLength;
  8026. if (thisSize)
  8027. {
  8028. OwnedConstRoxieRow ret = rowBuilder.finalizeRowClear(thisSize);
  8029. if ((processed - initialProcessed) >= limit)
  8030. {
  8031. if ( agent.queryCodeContext()->queryDebugContext())
  8032. agent.queryCodeContext()->queryDebugContext()->checkBreakpoint(DebugStateLimit, NULL, static_cast<IActivityBase *>(this));
  8033. helper.onLimitExceeded();
  8034. return NULL;
  8035. }
  8036. processed++;
  8037. return ret.getClear();
  8038. }
  8039. }
  8040. }
  8041. close();
  8042. return NULL;
  8043. }
  8044. bool CHThorCsvReadActivity::openNext()
  8045. {
  8046. if (CHThorDiskReadBaseActivity::openNext())
  8047. {
  8048. unsigned lines = headerLines;
  8049. while (lines-- && !inputstream->eos())
  8050. {
  8051. size32_t numAvailable;
  8052. const void * next = inputstream->peek(maxDiskSize, numAvailable);
  8053. inputstream->skip(csvSplitter.splitLine(numAvailable, (const byte *)next));
  8054. }
  8055. // only skip header in the first file - since spray doesn't duplicate the header.
  8056. headerLines = 0;
  8057. return true;
  8058. }
  8059. return false;
  8060. }
  8061. void CHThorCsvReadActivity::checkOpenNext()
  8062. {
  8063. agent.reportProgress(NULL);
  8064. if (!opened)
  8065. {
  8066. agent.reportProgress(NULL);
  8067. if (!helper.canMatchAny())
  8068. {
  8069. eofseen = true;
  8070. opened = true;
  8071. }
  8072. else
  8073. open();
  8074. }
  8075. for (;;)
  8076. {
  8077. if (eofseen || !inputstream->eos())
  8078. return;
  8079. eofseen = !openNext();
  8080. }
  8081. }
  8082. //=====================================================================================================
  8083. CHThorXmlReadActivity::CHThorXmlReadActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorXmlReadArg &_arg, ThorActivityKind _kind) : CHThorDiskReadBaseActivity(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  8084. {
  8085. }
  8086. void CHThorXmlReadActivity::ready()
  8087. {
  8088. CHThorDiskReadBaseActivity::ready();
  8089. rowTransformer.set(helper.queryTransformer());
  8090. localOffset = 0;
  8091. limit = helper.getRowLimit();
  8092. if (helper.getFlags() & TDRlimitskips)
  8093. limit = (unsigned __int64) -1;
  8094. stopAfter = helper.getChooseNLimit();
  8095. }
  8096. void CHThorXmlReadActivity::stop()
  8097. {
  8098. xmlParser.clear();
  8099. CHThorDiskReadBaseActivity::stop();
  8100. }
  8101. void CHThorXmlReadActivity::gatherInfo(IFileDescriptor * fd)
  8102. {
  8103. PARENT::gatherInfo(fd);
  8104. }
  8105. void CHThorXmlReadActivity::calcFixedDiskRecordSize()
  8106. {
  8107. fixedDiskRecordSize = 0;
  8108. }
  8109. const void *CHThorXmlReadActivity::nextRow()
  8110. {
  8111. if(!opened) open();
  8112. while (!eofseen && (!stopAfter || (processed - initialProcessed) < stopAfter))
  8113. {
  8114. agent.reportProgress(NULL);
  8115. //call to next() will callback on the IXmlSelect interface
  8116. bool gotNext = false;
  8117. try
  8118. {
  8119. gotNext = xmlParser->next();
  8120. }
  8121. catch(IException * e)
  8122. {
  8123. throw makeWrappedException(e, inputfile->queryFilename());
  8124. }
  8125. if(!gotNext)
  8126. eofseen = !openNext();
  8127. else if (lastMatch)
  8128. {
  8129. RtlDynamicRowBuilder rowBuilder(rowAllocator);
  8130. unsigned sizeGot;
  8131. try
  8132. {
  8133. sizeGot = rowTransformer->transform(rowBuilder, lastMatch, this);
  8134. }
  8135. catch(IException * e)
  8136. {
  8137. throw makeWrappedException(e);
  8138. }
  8139. lastMatch.clear();
  8140. localOffset = 0;
  8141. if (sizeGot)
  8142. {
  8143. OwnedConstRoxieRow ret = rowBuilder.finalizeRowClear(sizeGot);
  8144. if ((processed - initialProcessed) >= limit)
  8145. {
  8146. if ( agent.queryCodeContext()->queryDebugContext())
  8147. agent.queryCodeContext()->queryDebugContext()->checkBreakpoint(DebugStateLimit, NULL, static_cast<IActivityBase *>(this));
  8148. helper.onLimitExceeded();
  8149. return NULL;
  8150. }
  8151. processed++;
  8152. return ret.getClear();
  8153. }
  8154. }
  8155. }
  8156. return NULL;
  8157. }
  8158. bool CHThorXmlReadActivity::openNext()
  8159. {
  8160. if (inputfileio)
  8161. offsetOfPart += inputfileio->size();
  8162. localOffset = 0;
  8163. if (CHThorDiskReadBaseActivity::openNext())
  8164. {
  8165. unsigned readBufferSize = queryReadBufferSize();
  8166. OwnedIFileIOStream inputfileiostream;
  8167. if(readBufferSize)
  8168. inputfileiostream.setown(createBufferedIOStream(inputfileio, readBufferSize));
  8169. else
  8170. inputfileiostream.setown(createIOStream(inputfileio));
  8171. OwnedRoxieString xmlIterator(helper.getXmlIteratorPath());
  8172. if (kind==TAKjsonread)
  8173. xmlParser.setown(createJSONParse(*inputfileiostream, xmlIterator, *this, (0 != (TDRxmlnoroot & helper.getFlags()))?ptr_noRoot:ptr_none, (helper.getFlags() & TDRusexmlcontents) != 0));
  8174. else
  8175. xmlParser.setown(createXMLParse(*inputfileiostream, xmlIterator, *this, (0 != (TDRxmlnoroot & helper.getFlags()))?ptr_noRoot:ptr_none, (helper.getFlags() & TDRusexmlcontents) != 0));
  8176. return true;
  8177. }
  8178. return false;
  8179. }
  8180. void CHThorXmlReadActivity::closepart()
  8181. {
  8182. xmlParser.clear();
  8183. CHThorDiskReadBaseActivity::closepart();
  8184. }
  8185. //---------------------------------------------------------------------------
  8186. CHThorLocalResultReadActivity::CHThorLocalResultReadActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorLocalResultReadArg &_arg, ThorActivityKind _kind, __int64 graphId) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  8187. {
  8188. physicalRecordSize = outputMeta;
  8189. grouped = outputMeta.isGrouped();
  8190. graph = resolveLocalQuery(graphId);
  8191. result = NULL;
  8192. }
  8193. void CHThorLocalResultReadActivity::ready()
  8194. {
  8195. CHThorSimpleActivityBase::ready();
  8196. result = graph->queryResult(helper.querySequence());
  8197. curRow = 0;
  8198. }
  8199. const void *CHThorLocalResultReadActivity::nextRow()
  8200. {
  8201. const void * next = result->queryRow(curRow++);
  8202. if (next)
  8203. {
  8204. processed++;
  8205. LinkRoxieRow(next);
  8206. return next;
  8207. }
  8208. return NULL;
  8209. }
  8210. //=====================================================================================================
  8211. CHThorLocalResultWriteActivity::CHThorLocalResultWriteActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorLocalResultWriteArg &_arg, ThorActivityKind _kind, __int64 graphId)
  8212. : CHThorActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  8213. {
  8214. graph = resolveLocalQuery(graphId);
  8215. }
  8216. void CHThorLocalResultWriteActivity::execute()
  8217. {
  8218. IHThorGraphResult * result = graph->createResult(helper.querySequence(), LINK(rowAllocator));
  8219. for (;;)
  8220. {
  8221. const void *nextrec = input->nextRow();
  8222. if (!nextrec)
  8223. {
  8224. nextrec = input->nextRow();
  8225. if (!nextrec)
  8226. break;
  8227. result->addRowOwn(NULL);
  8228. }
  8229. result->addRowOwn(nextrec);
  8230. }
  8231. }
  8232. //=====================================================================================================
  8233. CHThorDictionaryResultWriteActivity::CHThorDictionaryResultWriteActivity (IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorDictionaryResultWriteArg &_arg, ThorActivityKind _kind, __int64 graphId)
  8234. : CHThorActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  8235. {
  8236. graph = resolveLocalQuery(graphId);
  8237. }
  8238. void CHThorDictionaryResultWriteActivity::execute()
  8239. {
  8240. RtlLinkedDictionaryBuilder builder(rowAllocator, helper.queryHashLookupInfo());
  8241. for (;;)
  8242. {
  8243. const void *row = input->nextRow();
  8244. if (!row)
  8245. {
  8246. row = input->nextRow();
  8247. if (!row)
  8248. break;
  8249. }
  8250. builder.appendOwn(row);
  8251. }
  8252. IHThorGraphResult * result = graph->createResult(helper.querySequence(), LINK(rowAllocator));
  8253. size32_t dictSize = builder.getcount();
  8254. const byte ** dictRows = builder.queryrows();
  8255. for (size32_t row = 0; row < dictSize; row++)
  8256. {
  8257. const byte *thisRow = dictRows[row];
  8258. if (thisRow)
  8259. LinkRoxieRow(thisRow);
  8260. result->addRowOwn(thisRow);
  8261. }
  8262. }
  8263. //=====================================================================================================
  8264. CHThorLocalResultSpillActivity::CHThorLocalResultSpillActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorLocalResultSpillArg &_arg, ThorActivityKind _kind, __int64 graphId)
  8265. : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  8266. {
  8267. result = NULL;
  8268. nullPending = false;
  8269. graph = resolveLocalQuery(graphId);
  8270. assertex(graph);
  8271. }
  8272. void CHThorLocalResultSpillActivity::ready()
  8273. {
  8274. CHThorSimpleActivityBase::ready();
  8275. result = graph->createResult(helper.querySequence(), LINK(rowAllocator));
  8276. nullPending = false;
  8277. }
  8278. const void * CHThorLocalResultSpillActivity::nextRow()
  8279. {
  8280. const void * ret = input->nextRow();
  8281. if (ret)
  8282. {
  8283. if (nullPending)
  8284. {
  8285. result->addRowOwn(NULL);
  8286. nullPending = false;
  8287. }
  8288. LinkRoxieRow(ret);
  8289. result->addRowOwn(ret);
  8290. processed++;
  8291. }
  8292. else
  8293. nullPending = true;
  8294. return ret;
  8295. }
  8296. void CHThorLocalResultSpillActivity::stop()
  8297. {
  8298. for (;;)
  8299. {
  8300. const void * ret = input->nextRow();
  8301. if (!ret)
  8302. {
  8303. if (nullPending)
  8304. break;
  8305. nullPending = true;
  8306. }
  8307. else
  8308. {
  8309. if (nullPending)
  8310. {
  8311. result->addRowOwn(NULL);
  8312. nullPending = false;
  8313. }
  8314. result->addRowOwn(ret);
  8315. }
  8316. }
  8317. CHThorSimpleActivityBase::stop();
  8318. }
  8319. //=====================================================================================================
  8320. CHThorLoopActivity::CHThorLoopActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorLoopArg &_arg, ThorActivityKind _kind)
  8321. : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  8322. {
  8323. flags = helper.getFlags();
  8324. maxIterations = 0;
  8325. }
  8326. CHThorLoopActivity::~CHThorLoopActivity()
  8327. {
  8328. ForEachItemIn(idx, loopPending)
  8329. ReleaseRoxieRow(loopPending.item(idx));
  8330. }
  8331. void CHThorLoopActivity::ready()
  8332. {
  8333. curInput = &input->queryStream();
  8334. eof = false;
  8335. loopCounter = 1;
  8336. CHThorSimpleActivityBase::ready();
  8337. maxIterations = helper.numIterations();
  8338. if ((int)maxIterations < 0) maxIterations = 0;
  8339. finishedLooping = ((kind == TAKloopcount) && (maxIterations == 0));
  8340. if ((flags & IHThorLoopArg::LFnewloopagain) && !helper.loopFirstTime())
  8341. finishedLooping = true;
  8342. extractBuilder.clear();
  8343. helper.createParentExtract(extractBuilder);
  8344. }
  8345. const void * CHThorLoopActivity::nextRow()
  8346. {
  8347. if (eof)
  8348. return NULL;
  8349. unsigned emptyIterations = 0;
  8350. for (;;)
  8351. {
  8352. for (;;)
  8353. {
  8354. const void * ret = curInput->nextRow();
  8355. if (!ret)
  8356. {
  8357. ret = curInput->nextRow(); // more cope with groups somehow....
  8358. if (!ret)
  8359. {
  8360. if (finishedLooping)
  8361. {
  8362. eof = true;
  8363. return NULL;
  8364. }
  8365. break;
  8366. }
  8367. }
  8368. if (finishedLooping ||
  8369. ((flags & IHThorLoopArg::LFfiltered) && !helper.sendToLoop(loopCounter, ret)))
  8370. {
  8371. processed++;
  8372. return ret;
  8373. }
  8374. loopPending.append(ret);
  8375. }
  8376. switch (kind)
  8377. {
  8378. case TAKloopdataset:
  8379. {
  8380. if (!(flags & IHThorLoopArg::LFnewloopagain))
  8381. {
  8382. if (!helper.loopAgain(loopCounter, loopPending.ordinality(), (const void * *)loopPending.getArray()))
  8383. {
  8384. if (loopPending.ordinality() == 0)
  8385. {
  8386. eof = true;
  8387. return NULL;
  8388. }
  8389. arrayInput.init(&loopPending);
  8390. curInput = &arrayInput;
  8391. finishedLooping = true;
  8392. continue; // back to the input loop again
  8393. }
  8394. }
  8395. break;
  8396. }
  8397. case TAKlooprow:
  8398. if (loopPending.empty())
  8399. {
  8400. finishedLooping = true;
  8401. eof = true;
  8402. return NULL;
  8403. }
  8404. break;
  8405. }
  8406. if (loopPending.ordinality())
  8407. emptyIterations = 0;
  8408. else
  8409. {
  8410. //note: any outputs which didn't go around the loop again, would return the record, reinitializing emptyIterations
  8411. emptyIterations++;
  8412. if (emptyIterations > EMPTY_LOOP_LIMIT)
  8413. throw MakeStringException(0, "Executed LOOP with empty input and output %u times", emptyIterations);
  8414. if (emptyIterations % 32 == 0)
  8415. DBGLOG("Executing LOOP with empty input and output %u times", emptyIterations);
  8416. }
  8417. void * counterRow = NULL;
  8418. if (flags & IHThorLoopArg::LFcounter)
  8419. {
  8420. counterRow = queryRowManager()->allocate(sizeof(thor_loop_counter_t), activityId);
  8421. *((thor_loop_counter_t *)counterRow) = loopCounter;
  8422. }
  8423. Owned<IHThorGraphResults> curResults = loopGraph->execute(counterRow, loopPending, extractBuilder.getbytes());
  8424. if (flags & IHThorLoopArg::LFnewloopagain)
  8425. {
  8426. IHThorGraphResult * result = curResults->queryResult(helper.loopAgainResult());
  8427. assertex(result);
  8428. const void * row = result->queryRow(0);
  8429. assertex(row);
  8430. //Result is a row which contains a single boolean field.
  8431. if (!((const bool *)row)[0])
  8432. finishedLooping = true;
  8433. }
  8434. resultInput.init(curResults->queryResult(0));
  8435. curInput = &resultInput;
  8436. loopCounter++;
  8437. if ((kind == TAKloopcount) && (loopCounter > maxIterations))
  8438. finishedLooping = true;
  8439. }
  8440. }
  8441. void CHThorLoopActivity::stop()
  8442. {
  8443. ForEachItemIn(idx, loopPending)
  8444. ReleaseRoxieRow(loopPending.item(idx));
  8445. loopPending.kill();
  8446. CHThorSimpleActivityBase::stop();
  8447. }
  8448. //---------------------------------------------------------------------------
  8449. CHThorGraphLoopResultReadActivity::CHThorGraphLoopResultReadActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorGraphLoopResultReadArg &_arg, ThorActivityKind _kind, __int64 graphId) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(&_arg)
  8450. {
  8451. physicalRecordSize = outputMeta;
  8452. grouped = outputMeta.isGrouped();
  8453. result = NULL;
  8454. graph = resolveLocalQuery(graphId);
  8455. }
  8456. CHThorGraphLoopResultReadActivity::CHThorGraphLoopResultReadActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorArg & _arg, ThorActivityKind _kind, __int64 graphId, unsigned _sequence, bool _grouped) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(NULL)
  8457. {
  8458. physicalRecordSize = outputMeta;
  8459. sequence = _sequence;
  8460. grouped = _grouped;
  8461. result = NULL;
  8462. graph = resolveLocalQuery(graphId);
  8463. }
  8464. void CHThorGraphLoopResultReadActivity::ready()
  8465. {
  8466. CHThorSimpleActivityBase::ready();
  8467. if (helper)
  8468. sequence = helper->querySequence();
  8469. if ((int)sequence >= 0)
  8470. result = graph->queryGraphLoopResult(sequence);
  8471. else
  8472. result = NULL;
  8473. curRow = 0;
  8474. }
  8475. const void *CHThorGraphLoopResultReadActivity::nextRow()
  8476. {
  8477. if (result)
  8478. {
  8479. const void * next = result->queryRow(curRow++);
  8480. if (next)
  8481. {
  8482. processed++;
  8483. LinkRoxieRow(next);
  8484. return (void *)next;
  8485. }
  8486. }
  8487. return NULL;
  8488. }
  8489. //=====================================================================================================
  8490. CHThorGraphLoopResultWriteActivity::CHThorGraphLoopResultWriteActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorGraphLoopResultWriteArg &_arg, ThorActivityKind _kind, __int64 graphId)
  8491. : CHThorActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  8492. {
  8493. graph = resolveLocalQuery(graphId);
  8494. }
  8495. void CHThorGraphLoopResultWriteActivity::execute()
  8496. {
  8497. IHThorGraphResult * result = graph->createGraphLoopResult(LINK(rowAllocator));
  8498. for (;;)
  8499. {
  8500. const void *nextrec = input->nextRow();
  8501. if (!nextrec)
  8502. {
  8503. nextrec = input->nextRow();
  8504. if (!nextrec)
  8505. break;
  8506. result->addRowOwn(NULL);
  8507. }
  8508. result->addRowOwn(nextrec);
  8509. }
  8510. }
  8511. //=====================================================================================================
  8512. class CCounterMeta : implements IOutputMetaData, public CInterface
  8513. {
  8514. public:
  8515. IMPLEMENT_IINTERFACE
  8516. virtual size32_t getRecordSize(const void *rec) { return sizeof(thor_loop_counter_t); }
  8517. virtual size32_t getMinRecordSize() const { return sizeof(thor_loop_counter_t); }
  8518. virtual size32_t getFixedSize() const { return sizeof(thor_loop_counter_t); }
  8519. virtual void toXML(const byte * self, IXmlWriter & out) { }
  8520. virtual unsigned getVersion() const { return OUTPUTMETADATA_VERSION; }
  8521. virtual unsigned getMetaFlags() { return 0; }
  8522. virtual void destruct(byte * self) {}
  8523. virtual IOutputRowSerializer * createDiskSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
  8524. virtual IOutputRowDeserializer * createDiskDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
  8525. virtual ISourceRowPrefetcher * createDiskPrefetcher(ICodeContext * ctx, unsigned activityId) { return NULL; }
  8526. virtual IOutputMetaData * querySerializedDiskMeta() { return this; }
  8527. virtual IOutputRowSerializer * createInternalSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
  8528. virtual IOutputRowDeserializer * createInternalDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
  8529. virtual void walkIndirectMembers(const byte * self, IIndirectMemberVisitor & visitor) {}
  8530. virtual IOutputMetaData * queryChildMeta(unsigned i) { return NULL; }
  8531. virtual const RtlRecord &queryRecordAccessor(bool expand) const { throwUnexpected(); } // could provide a static implementation if needed
  8532. };
  8533. //=====================================================================================================
  8534. CHThorGraphLoopActivity::CHThorGraphLoopActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorGraphLoopArg &_arg, ThorActivityKind _kind)
  8535. : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  8536. {
  8537. flags = helper.getFlags();
  8538. maxIterations = 0;
  8539. counterMeta.setown(new CCounterMeta);
  8540. }
  8541. void CHThorGraphLoopActivity::ready()
  8542. {
  8543. executed = false;
  8544. resultIndex = 0;
  8545. CHThorSimpleActivityBase::ready();
  8546. maxIterations = helper.numIterations();
  8547. if ((int)maxIterations < 0) maxIterations = 0;
  8548. loopResults.setown(agent.createGraphLoopResults());
  8549. extractBuilder.clear();
  8550. helper.createParentExtract(extractBuilder);
  8551. rowAllocator.setown(agent.queryCodeContext()->getRowAllocator(queryOutputMeta(), activityId));
  8552. rowAllocatorCounter.setown(agent.queryCodeContext()->getRowAllocator(counterMeta, activityId));
  8553. }
  8554. const void * CHThorGraphLoopActivity::nextRow()
  8555. {
  8556. if (!executed)
  8557. {
  8558. executed = true;
  8559. IHThorGraphResult * inputResult = loopResults->createResult(0, LINK(rowAllocator));
  8560. for (;;)
  8561. {
  8562. const void * ret = input->nextRow();
  8563. if (!ret)
  8564. {
  8565. ret = input->nextRow();
  8566. if (!ret)
  8567. break;
  8568. inputResult->addRowOwn(NULL);
  8569. }
  8570. inputResult->addRowOwn(ret);
  8571. }
  8572. for (unsigned loopCounter = 1; loopCounter <= maxIterations; loopCounter++)
  8573. {
  8574. void * counterRow = NULL;
  8575. if (flags & IHThorGraphLoopArg::GLFcounter)
  8576. {
  8577. counterRow = rowAllocatorCounter->createRow();
  8578. *((thor_loop_counter_t *)counterRow) = loopCounter;
  8579. counterRow = rowAllocatorCounter->finalizeRow(sizeof(thor_loop_counter_t), counterRow, sizeof(thor_loop_counter_t));
  8580. }
  8581. loopGraph->execute(counterRow, loopResults, extractBuilder.getbytes());
  8582. }
  8583. int iNumResults = loopResults->ordinality();
  8584. finalResult = loopResults->queryResult(iNumResults-1); //Get the last result, which isnt necessarily 'maxIterations'
  8585. }
  8586. const void * next = finalResult->getOwnRow(resultIndex++);
  8587. if (next)
  8588. processed++;
  8589. return next;
  8590. }
  8591. void CHThorGraphLoopActivity::stop()
  8592. {
  8593. rowAllocator.clear();
  8594. finalResult = NULL;
  8595. loopResults.clear();
  8596. CHThorSimpleActivityBase::stop();
  8597. }
  8598. //=====================================================================================================
  8599. CHThorParallelGraphLoopActivity::CHThorParallelGraphLoopActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorGraphLoopArg &_arg, ThorActivityKind _kind)
  8600. : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  8601. {
  8602. flags = helper.getFlags();
  8603. maxIterations = 0;
  8604. }
  8605. void CHThorParallelGraphLoopActivity::ready()
  8606. {
  8607. executed = false;
  8608. resultIndex = 0;
  8609. CHThorSimpleActivityBase::ready();
  8610. maxIterations = helper.numIterations();
  8611. if ((int)maxIterations < 0) maxIterations = 0;
  8612. loopResults.setown(agent.createGraphLoopResults());
  8613. extractBuilder.clear();
  8614. helper.createParentExtract(extractBuilder);
  8615. rowAllocator.setown(agent.queryCodeContext()->getRowAllocator(queryOutputMeta(), activityId));
  8616. }
  8617. const void * CHThorParallelGraphLoopActivity::nextRow()
  8618. {
  8619. if (!executed)
  8620. {
  8621. executed = true;
  8622. IHThorGraphResult * inputResult = loopResults->createResult(0, LINK(rowAllocator));
  8623. for (;;)
  8624. {
  8625. const void * ret = input->nextRow();
  8626. if (!ret)
  8627. {
  8628. ret = input->nextRow();
  8629. if (!ret)
  8630. break;
  8631. inputResult->addRowOwn(NULL);
  8632. }
  8633. inputResult->addRowOwn(ret);
  8634. }
  8635. // The lack of separation between pre-creation and creation means this would require cloning lots of structures.
  8636. // not implemented for the moment.
  8637. // loopGraph->executeParallel(loopResults, extractBuilder.getbytes(), maxIterations);
  8638. finalResult = loopResults->queryResult(maxIterations);
  8639. }
  8640. const void * next = finalResult->getOwnRow(resultIndex++);
  8641. if (next)
  8642. processed++;
  8643. return next;
  8644. }
  8645. void CHThorParallelGraphLoopActivity::stop()
  8646. {
  8647. rowAllocator.clear();
  8648. finalResult = NULL;
  8649. loopResults.clear();
  8650. CHThorSimpleActivityBase::stop();
  8651. }
  8652. //=====================================================================================================
  8653. LibraryCallOutput::LibraryCallOutput(CHThorLibraryCallActivity * _owner, unsigned _output, IOutputMetaData * _meta) : owner(_owner), output(_output), meta(_meta)
  8654. {
  8655. processed = 0;
  8656. }
  8657. const void * LibraryCallOutput::nextRow()
  8658. {
  8659. if (!gotRows)
  8660. {
  8661. result.set(owner->getResultRows(output));
  8662. gotRows = true;
  8663. }
  8664. const void * ret = result->getOwnRow(curRow++);
  8665. if (ret)
  8666. processed++;
  8667. return ret;
  8668. }
  8669. bool LibraryCallOutput::isGrouped()
  8670. {
  8671. return meta->isGrouped();
  8672. }
  8673. IOutputMetaData * LibraryCallOutput::queryOutputMeta() const
  8674. {
  8675. return meta;
  8676. }
  8677. void LibraryCallOutput::ready()
  8678. {
  8679. owner->ready();
  8680. gotRows = false;
  8681. result.clear();
  8682. curRow = 0;
  8683. }
  8684. void LibraryCallOutput::stop()
  8685. {
  8686. owner->stop();
  8687. result.clear();
  8688. }
  8689. void LibraryCallOutput::resetEOF()
  8690. {
  8691. throwUnexpected();
  8692. }
  8693. void LibraryCallOutput::updateProgress(IStatisticGatherer &progress) const
  8694. {
  8695. owner->updateOutputProgress(progress, *this, processed);
  8696. }
  8697. CHThorLibraryCallActivity::CHThorLibraryCallActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorLibraryCallArg &_arg, ThorActivityKind _kind, IPropertyTree * node)
  8698. : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  8699. {
  8700. libraryName.set(node->queryProp("att[@name=\"libname\"]/@value"));
  8701. interfaceHash = node->getPropInt("att[@name=\"_interfaceHash\"]/@value", 0);
  8702. bool embedded = node->getPropBool("att[@name=\"embedded\"]/@value", false) ;
  8703. if (embedded)
  8704. {
  8705. embeddedGraphName.set(node->queryProp("att[@name=\"graph\"]/@value"));
  8706. if (!embeddedGraphName)
  8707. embeddedGraphName.set(libraryName);
  8708. }
  8709. Owned<IPropertyTreeIterator> iter = node->getElements("att[@name=\"_outputUsed\"]");
  8710. ForEach(*iter)
  8711. {
  8712. unsigned whichOutput = iter->query().getPropInt("@value");
  8713. IOutputMetaData * meta = helper.queryOutputMeta(whichOutput);
  8714. outputs.append(*new LibraryCallOutput(this, whichOutput, meta));
  8715. }
  8716. state = StateCreated;
  8717. }
  8718. IHThorGraphResult * CHThorLibraryCallActivity::getResultRows(unsigned whichOutput)
  8719. {
  8720. CriticalBlock procedure(cs);
  8721. if (!results)
  8722. {
  8723. if (libraryName.length() == 0)
  8724. libraryName.setown(helper.getLibraryName());
  8725. helper.createParentExtract(extractBuilder);
  8726. results.setown(agent.executeLibraryGraph(libraryName, interfaceHash, activityId, embeddedGraphName, extractBuilder.getbytes()));
  8727. }
  8728. return results->queryResult(whichOutput);
  8729. }
  8730. IHThorInput * CHThorLibraryCallActivity::queryOutput(unsigned idx)
  8731. {
  8732. assert(outputs.isItem(idx));
  8733. return &outputs.item(idx);
  8734. }
  8735. void CHThorLibraryCallActivity::updateOutputProgress(IStatisticGatherer &progress, const LibraryCallOutput & _output, unsigned __int64 numProcessed) const
  8736. {
  8737. LibraryCallOutput & output = const_cast<LibraryCallOutput &>(_output);
  8738. updateProgressForOther(progress, activityId, subgraphId, outputs.find(output), numProcessed);
  8739. }
  8740. void CHThorLibraryCallActivity::ready()
  8741. {
  8742. CriticalBlock procedure(cs);
  8743. if (state != StateReady)
  8744. {
  8745. results.clear();
  8746. CHThorSimpleActivityBase::ready();
  8747. state = StateReady;
  8748. }
  8749. }
  8750. const void * CHThorLibraryCallActivity::nextRow()
  8751. {
  8752. throwUnexpected();
  8753. }
  8754. void CHThorLibraryCallActivity::stop()
  8755. {
  8756. CriticalBlock procedure(cs);
  8757. if (state != StateDone)
  8758. {
  8759. results.clear();
  8760. CHThorSimpleActivityBase::stop();
  8761. }
  8762. }
  8763. //=====================================================================================================
  8764. class CHThorNWayInputActivity : public CHThorSimpleActivityBase, implements IHThorNWayInput
  8765. {
  8766. IHThorNWayInputArg & helper;
  8767. InputArrayType inputs;
  8768. InputArrayType selectedInputs;
  8769. public:
  8770. CHThorNWayInputActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorNWayInputArg &_arg, ThorActivityKind _kind) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  8771. {
  8772. }
  8773. virtual void ready()
  8774. {
  8775. bool selectionIsAll;
  8776. size32_t selectionLen;
  8777. rtlDataAttr selection;
  8778. helper.getInputSelection(selectionIsAll, selectionLen, selection.refdata());
  8779. selectedInputs.kill();
  8780. if (selectionIsAll)
  8781. {
  8782. ForEachItemIn(i, inputs)
  8783. selectedInputs.append(inputs.item(i));
  8784. }
  8785. else
  8786. {
  8787. const size32_t * selections = (const size32_t *)selection.getdata();
  8788. unsigned max = selectionLen/sizeof(size32_t);
  8789. for (unsigned i = 0; i < max; i++)
  8790. {
  8791. unsigned nextIndex = selections[i];
  8792. //Check there are no duplicates..... Assumes there are a fairly small number of inputs, so n^2 search is ok.
  8793. for (unsigned j=i+1; j < max; j++)
  8794. {
  8795. if (nextIndex == selections[j])
  8796. throw MakeStringException(100, "Selection list for nway input can not contain duplicates");
  8797. }
  8798. if (!inputs.isItem(nextIndex-1))
  8799. throw MakeStringException(100, "Index %d in RANGE selection list is out of range", nextIndex);
  8800. selectedInputs.append(inputs.item(nextIndex-1));
  8801. }
  8802. }
  8803. ForEachItemIn(i2, selectedInputs)
  8804. selectedInputs.item(i2)->ready();
  8805. }
  8806. virtual void setInput(unsigned idx, IHThorInput *_in)
  8807. {
  8808. assertex(idx == inputs.ordinality());
  8809. inputs.append(_in);
  8810. }
  8811. virtual const void * nextRow()
  8812. {
  8813. throwUnexpected();
  8814. }
  8815. virtual void updateProgress(IStatisticGatherer &progress) const
  8816. {
  8817. // CHThorSimpleActivityBase::updateProgress(progress);
  8818. ForEachItemIn(i, inputs)
  8819. inputs.item(i)->updateProgress(progress);
  8820. }
  8821. virtual unsigned numConcreteOutputs() const
  8822. {
  8823. return selectedInputs.ordinality();
  8824. }
  8825. virtual IHThorInput * queryConcreteInput(unsigned idx) const
  8826. {
  8827. if (selectedInputs.isItem(idx))
  8828. return selectedInputs.item(idx);
  8829. return NULL;
  8830. }
  8831. };
  8832. //=====================================================================================================
  8833. class CHThorNWayGraphLoopResultReadActivity : public CHThorSimpleActivityBase, implements IHThorNWayInput
  8834. {
  8835. IHThorNWayGraphLoopResultReadArg & helper;
  8836. CIArrayOf<CHThorActivityBase> inputs;
  8837. __int64 graphId;
  8838. bool grouped;
  8839. public:
  8840. CHThorNWayGraphLoopResultReadActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorNWayGraphLoopResultReadArg &_arg, ThorActivityKind _kind, __int64 _graphId) : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  8841. {
  8842. grouped = helper.isGrouped();
  8843. graphId = _graphId;
  8844. }
  8845. virtual bool isGrouped()
  8846. {
  8847. return grouped;
  8848. }
  8849. virtual void ready()
  8850. {
  8851. bool selectionIsAll;
  8852. size32_t selectionLen;
  8853. rtlDataAttr selection;
  8854. helper.getInputSelection(selectionIsAll, selectionLen, selection.refdata());
  8855. if (selectionIsAll)
  8856. throw MakeStringException(100, "ALL not yet supported for NWay graph inputs");
  8857. unsigned max = selectionLen / sizeof(size32_t);
  8858. const size32_t * selections = (const size32_t *)selection.getdata();
  8859. for (unsigned i = 0; i < max; i++)
  8860. {
  8861. CHThorActivityBase * resultInput = new CHThorGraphLoopResultReadActivity(agent, activityId, subgraphId, helper, kind, graphId, selections[i], grouped);
  8862. inputs.append(*resultInput);
  8863. resultInput->ready();
  8864. }
  8865. }
  8866. virtual void stop()
  8867. {
  8868. inputs.kill();
  8869. }
  8870. virtual void setInput(unsigned idx, IHThorInput *_in)
  8871. {
  8872. throwUnexpected();
  8873. }
  8874. virtual const void * nextRow()
  8875. {
  8876. throwUnexpected();
  8877. }
  8878. virtual unsigned numConcreteOutputs() const
  8879. {
  8880. return inputs.ordinality();
  8881. }
  8882. virtual IHThorInput * queryConcreteInput(unsigned idx) const
  8883. {
  8884. if (inputs.isItem(idx))
  8885. return &inputs.item(idx);
  8886. return NULL;
  8887. }
  8888. };
  8889. //=====================================================================================================
  8890. CHThorNWaySelectActivity::CHThorNWaySelectActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorNWaySelectArg &_arg, ThorActivityKind _kind) : CHThorMultiInputActivity(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  8891. {
  8892. selectedInput = NULL;
  8893. }
  8894. void CHThorNWaySelectActivity::stop()
  8895. {
  8896. selectedInput = NULL;
  8897. CHThorMultiInputActivity::stop();
  8898. }
  8899. void CHThorNWaySelectActivity::ready()
  8900. {
  8901. CHThorMultiInputActivity::ready();
  8902. unsigned whichInput = helper.getInputIndex();
  8903. selectedInput = NULL;
  8904. if (whichInput--)
  8905. {
  8906. ForEachItemIn(i, inputs)
  8907. {
  8908. IHThorInput * cur = inputs.item(i);
  8909. IHThorNWayInput * nWayInput = dynamic_cast<IHThorNWayInput *>(cur);
  8910. if (nWayInput)
  8911. {
  8912. unsigned numRealInputs = nWayInput->numConcreteOutputs();
  8913. if (whichInput < numRealInputs)
  8914. selectedInput = nWayInput->queryConcreteInput(whichInput);
  8915. whichInput -= numRealInputs;
  8916. }
  8917. else
  8918. {
  8919. if (whichInput == 0)
  8920. selectedInput = cur;
  8921. whichInput -= 1;
  8922. }
  8923. if (selectedInput)
  8924. break;
  8925. }
  8926. }
  8927. }
  8928. const void * CHThorNWaySelectActivity::nextRow()
  8929. {
  8930. if (!selectedInput)
  8931. return NULL;
  8932. return selectedInput->nextRow();
  8933. }
  8934. const void * CHThorNWaySelectActivity::nextRowGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
  8935. {
  8936. if (!selectedInput)
  8937. return NULL;
  8938. return selectedInput->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
  8939. }
  8940. IInputSteppingMeta * CHThorNWaySelectActivity::querySteppingMeta()
  8941. {
  8942. if (selectedInput)
  8943. return selectedInput->querySteppingMeta();
  8944. return NULL;
  8945. }
  8946. //=====================================================================================================
  8947. CHThorStreamedIteratorActivity::CHThorStreamedIteratorActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorStreamedIteratorArg &_arg, ThorActivityKind _kind)
  8948. : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
  8949. {
  8950. }
  8951. void CHThorStreamedIteratorActivity::ready()
  8952. {
  8953. CHThorSimpleActivityBase::ready();
  8954. rows.setown(helper.createInput());
  8955. }
  8956. const void *CHThorStreamedIteratorActivity::nextRow()
  8957. {
  8958. assertex(rows);
  8959. const void * next = rows->nextRow();
  8960. if (next)
  8961. processed++;
  8962. return next;
  8963. }
  8964. void CHThorStreamedIteratorActivity::stop()
  8965. {
  8966. if (rows)
  8967. {
  8968. rows->stop();
  8969. rows.clear();
  8970. }
  8971. }
  8972. //=====================================================================================================
  8973. CHThorExternalActivity::CHThorExternalActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorExternalArg &_arg, ThorActivityKind _kind, IPropertyTree * _graphNode)
  8974. : CHThorMultiInputActivity(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg), graphNode(_graphNode)
  8975. {
  8976. }
  8977. void CHThorExternalActivity::ready()
  8978. {
  8979. CHThorMultiInputActivity::ready();
  8980. //must be called after onStart()
  8981. processor.setown(helper.createProcessor());
  8982. processor->onCreate(agent.queryCodeContext(), graphNode);
  8983. ForEachItemIn(idx, inputs)
  8984. {
  8985. Owned<CHThorInputAdaptor> adaptedInput = new CHThorInputAdaptor(inputs.item(idx));
  8986. processor->addInput(idx, adaptedInput);
  8987. }
  8988. processor->start();
  8989. if (outputMeta.getMinRecordSize() > 0)
  8990. rows.setown(processor->createOutput(0));
  8991. }
  8992. const void *CHThorExternalActivity::nextRow()
  8993. {
  8994. assertex(rows);
  8995. const void * next = rows->nextRow();
  8996. if (next)
  8997. processed++;
  8998. return next;
  8999. }
  9000. void CHThorExternalActivity::execute()
  9001. {
  9002. assertex(!rows);
  9003. processor->execute();
  9004. }
  9005. void CHThorExternalActivity::reset()
  9006. {
  9007. rows.clear();
  9008. processor->reset();
  9009. processor.clear();
  9010. }
  9011. void CHThorExternalActivity::stop()
  9012. {
  9013. if (rows)
  9014. {
  9015. rows->stop();
  9016. rows.clear();
  9017. }
  9018. processor->stop();
  9019. }
  9020. //=====================================================================================================
  9021. MAKEFACTORY(DiskWrite);
  9022. MAKEFACTORY(Iterate);
  9023. MAKEFACTORY(Filter);
  9024. MAKEFACTORY(Aggregate);
  9025. MAKEFACTORY(Rollup);
  9026. MAKEFACTORY(Project);
  9027. MAKEFACTORY(PrefetchProject);
  9028. MAKEFACTORY(FilterProject);
  9029. extern HTHOR_API IHThorActivity * createGroupDedupActivity(IAgentContext & _agent, unsigned _activityId, unsigned _subgraphId, IHThorDedupArg & arg, ThorActivityKind kind)
  9030. {
  9031. if(arg.compareAll())
  9032. return new CHThorGroupDedupAllActivity(_agent, _activityId, _subgraphId, arg, kind);
  9033. else if (arg.keepLeft() && !arg.keepBest())
  9034. return new CHThorGroupDedupKeepLeftActivity(_agent, _activityId, _subgraphId, arg, kind);
  9035. else
  9036. return new CHThorGroupDedupKeepRightActivity(_agent, _activityId, _subgraphId, arg, kind);
  9037. }
  9038. MAKEFACTORY(HashDedup);
  9039. MAKEFACTORY(Group);
  9040. MAKEFACTORY(Degroup);
  9041. MAKEFACTORY_ARG(GroupSort, Sort);
  9042. MAKEFACTORY(Join);
  9043. MAKEFACTORY_ARG(SelfJoin, Join);
  9044. MAKEFACTORY_ARG(LookupJoin, HashJoin);
  9045. MAKEFACTORY(AllJoin);
  9046. MAKEFACTORY(WorkUnitWrite);
  9047. MAKEFACTORY(DictionaryWorkUnitWrite);
  9048. MAKEFACTORY(FirstN);
  9049. MAKEFACTORY(InlineTable);
  9050. MAKEFACTORY_ARG(Concat, Funnel);
  9051. MAKEFACTORY(Apply);
  9052. MAKEFACTORY(Sample);
  9053. MAKEFACTORY(Normalize);
  9054. MAKEFACTORY(NormalizeChild);
  9055. MAKEFACTORY(NormalizeLinkedChild);
  9056. MAKEFACTORY(Distribution);
  9057. MAKEFACTORY(RemoteResult);
  9058. MAKEFACTORY(ChooseSets);
  9059. MAKEFACTORY_ARG(ChooseSetsLast, ChooseSetsEx);
  9060. MAKEFACTORY_ARG(ChooseSetsEnth, ChooseSetsEx);
  9061. MAKEFACTORY(WorkunitRead);
  9062. MAKEFACTORY(PipeRead);
  9063. MAKEFACTORY(PipeWrite);
  9064. MAKEFACTORY(CsvWrite);
  9065. MAKEFACTORY(XmlWrite);
  9066. MAKEFACTORY(PipeThrough);
  9067. MAKEFACTORY(If);
  9068. extern HTHOR_API IHThorActivity *createChildIfActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorIfArg &arg, ThorActivityKind kind)
  9069. {
  9070. return new CHThorIfActivity(_agent, _activityId, _subgraphId, arg, kind);
  9071. }
  9072. extern HTHOR_API IHThorActivity *createHashAggregateActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorHashAggregateArg &arg, ThorActivityKind kind, bool _isGroupedAggregate)
  9073. {
  9074. return new CHThorHashAggregateActivity(_agent, _activityId, _subgraphId, arg, kind, _isGroupedAggregate);
  9075. }
  9076. MAKEFACTORY(Null);
  9077. MAKEFACTORY(SideEffect);
  9078. MAKEFACTORY(Action);
  9079. MAKEFACTORY(SelectN);
  9080. MAKEFACTORY(Spill);
  9081. MAKEFACTORY(Limit);
  9082. MAKEFACTORY_ARG(SkipLimit, Limit);
  9083. MAKEFACTORY_ARG(OnFailLimit, Limit);
  9084. MAKEFACTORY(Catch);
  9085. MAKEFACTORY_ARG(SkipCatch, Catch);
  9086. MAKEFACTORY(CountProject);
  9087. MAKEFACTORY(IndexWrite);
  9088. MAKEFACTORY(Parse);
  9089. MAKEFACTORY(Enth);
  9090. MAKEFACTORY(TopN);
  9091. MAKEFACTORY(XmlParse);
  9092. MAKEFACTORY(Merge);
  9093. MAKEFACTORY_ARG(HttpRowCall, HttpCall);
  9094. MAKEFACTORY_ARG(SoapRowCall, SoapCall);
  9095. MAKEFACTORY_ARG(SoapRowAction, SoapAction);
  9096. MAKEFACTORY_ARG(SoapDatasetCall, SoapCall);
  9097. MAKEFACTORY_ARG(SoapDatasetAction, SoapAction);
  9098. MAKEFACTORY(DatasetResult);
  9099. MAKEFACTORY(RowResult);
  9100. MAKEFACTORY(ChildIterator);
  9101. extern HTHOR_API IHThorActivity *createDummyActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorArg &arg, ThorActivityKind kind)
  9102. {
  9103. return new CHThorDummyActivity(_agent, _activityId, _subgraphId, arg, kind);
  9104. }
  9105. MAKEFACTORY_EXTRA(WhenAction,EclGraphElement *)
  9106. MAKEFACTORY_EXTRA(LibraryCall, IPropertyTree *)
  9107. MAKEFACTORY(ChildNormalize)
  9108. MAKEFACTORY(ChildAggregate)
  9109. MAKEFACTORY(ChildGroupAggregate)
  9110. MAKEFACTORY(ChildThroughNormalize)
  9111. MAKEFACTORY(DiskRead)
  9112. MAKEFACTORY(DiskNormalize)
  9113. MAKEFACTORY(DiskAggregate)
  9114. MAKEFACTORY(DiskCount)
  9115. MAKEFACTORY(DiskGroupAggregate)
  9116. MAKEFACTORY(CsvRead)
  9117. MAKEFACTORY(XmlRead)
  9118. MAKEFACTORY_EXTRA(LocalResultRead, __int64)
  9119. MAKEFACTORY_EXTRA(LocalResultWrite, __int64)
  9120. MAKEFACTORY_EXTRA(DictionaryResultWrite, __int64)
  9121. MAKEFACTORY_EXTRA(LocalResultSpill, __int64)
  9122. MAKEFACTORY_EXTRA(GraphLoopResultRead, __int64)
  9123. MAKEFACTORY_EXTRA(GraphLoopResultWrite, __int64)
  9124. MAKEFACTORY_EXTRA(NWayGraphLoopResultRead, __int64)
  9125. MAKEFACTORY(Combine)
  9126. MAKEFACTORY(RollupGroup)
  9127. MAKEFACTORY(Regroup)
  9128. MAKEFACTORY(CombineGroup)
  9129. MAKEFACTORY(Case)
  9130. MAKEFACTORY(LinkedRawIterator)
  9131. MAKEFACTORY(GraphLoop)
  9132. MAKEFACTORY(Loop)
  9133. MAKEFACTORY(Process)
  9134. MAKEFACTORY(Grouped)
  9135. MAKEFACTORY(Sorted)
  9136. MAKEFACTORY(Trace)
  9137. MAKEFACTORY(NWayInput)
  9138. MAKEFACTORY(NWaySelect)
  9139. MAKEFACTORY(NonEmpty)
  9140. MAKEFACTORY(FilterGroup);
  9141. MAKEFACTORY(StreamedIterator);
  9142. MAKEFACTORY_EXTRA(External, IPropertyTree *);
  9143. IHThorException * makeHThorException(ThorActivityKind kind, unsigned activityId, unsigned subgraphId, int code, char const * format, ...)
  9144. {
  9145. va_list args;
  9146. va_start(args, format);
  9147. IHThorException * ret = new CHThorException(code, format, args, MSGAUD_user, kind, activityId, subgraphId);
  9148. va_end(args);
  9149. return ret;
  9150. }
  9151. IHThorException * makeHThorException(ThorActivityKind kind, unsigned activityId, unsigned subgraphId, IException * exc)
  9152. {
  9153. return new CHThorException(exc, kind, activityId, subgraphId);
  9154. }
  9155. IHThorException * makeHThorException(ThorActivityKind kind, unsigned activityId, unsigned subgraphId, IException * exc, char const * extra)
  9156. {
  9157. return new CHThorException(exc, extra, kind, activityId, subgraphId);
  9158. }