workunit.cpp 355 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383638463856386638763886389639063916392639363946395639663976398639964006401640264036404640564066407640864096410641164126413641464156416641764186419642064216422642364246425642664276428642964306431643264336434643564366437643864396440644164426443644464456446644764486449645064516452645364546455645664576458645964606461646264636464646564666467646864696470647164726473647464756476647764786479648064816482648364846485648664876488648964906491649264936494649564966497649864996500650165026503650465056506650765086509651065116512651365146515651665176518651965206521652265236524652565266527652865296530653165326533653465356536653765386539654065416542654365446545654665476548654965506551655265536554655565566557655865596560656165626563656465656566656765686569657065716572657365746575657665776578657965806581658265836584658565866587658865896590659165926593659465956596659765986599660066016602660366046605660666076608660966106611661266136614661566166617661866196620662166226623662466256626662766286629663066316632663366346635663666376638663966406641664266436644664566466647664866496650665166526653665466556656665766586659666066616662666366646665666666676668666966706671667266736674667566766677667866796680668166826683668466856686668766886689669066916692669366946695669666976698669967006701670267036704670567066707670867096710671167126713671467156716671767186719672067216722672367246725672667276728672967306731673267336734673567366737673867396740674167426743674467456746674767486749675067516752675367546755675667576758675967606761676267636764676567666767676867696770677167726773677467756776677767786779678067816782678367846785678667876788678967906791679267936794679567966797679867996800680168026803680468056806680768086809681068116812681368146815681668176818681968206821682268236824682568266827682868296830683168326833683468356836683768386839684068416842684368446845684668476848684968506851685268536854685568566857685868596860686168626863686468656866686768686869687068716872687368746875687668776878687968806881688268836884688568866887688868896890689168926893689468956896689768986899690069016902690369046905690669076908690969106911691269136914691569166917691869196920692169226923692469256926692769286929693069316932693369346935693669376938693969406941694269436944694569466947694869496950695169526953695469556956695769586959696069616962696369646965696669676968696969706971697269736974697569766977697869796980698169826983698469856986698769886989699069916992699369946995699669976998699970007001700270037004700570067007700870097010701170127013701470157016701770187019702070217022702370247025702670277028702970307031703270337034703570367037703870397040704170427043704470457046704770487049705070517052705370547055705670577058705970607061706270637064706570667067706870697070707170727073707470757076707770787079708070817082708370847085708670877088708970907091709270937094709570967097709870997100710171027103710471057106710771087109711071117112711371147115711671177118711971207121712271237124712571267127712871297130713171327133713471357136713771387139714071417142714371447145714671477148714971507151715271537154715571567157715871597160716171627163716471657166716771687169717071717172717371747175717671777178717971807181718271837184718571867187718871897190719171927193719471957196719771987199720072017202720372047205720672077208720972107211721272137214721572167217721872197220722172227223722472257226722772287229723072317232723372347235723672377238723972407241724272437244724572467247724872497250725172527253725472557256725772587259726072617262726372647265726672677268726972707271727272737274727572767277727872797280728172827283728472857286728772887289729072917292729372947295729672977298729973007301730273037304730573067307730873097310731173127313731473157316731773187319732073217322732373247325732673277328732973307331733273337334733573367337733873397340734173427343734473457346734773487349735073517352735373547355735673577358735973607361736273637364736573667367736873697370737173727373737473757376737773787379738073817382738373847385738673877388738973907391739273937394739573967397739873997400740174027403740474057406740774087409741074117412741374147415741674177418741974207421742274237424742574267427742874297430743174327433743474357436743774387439744074417442744374447445744674477448744974507451745274537454745574567457745874597460746174627463746474657466746774687469747074717472747374747475747674777478747974807481748274837484748574867487748874897490749174927493749474957496749774987499750075017502750375047505750675077508750975107511751275137514751575167517751875197520752175227523752475257526752775287529753075317532753375347535753675377538753975407541754275437544754575467547754875497550755175527553755475557556755775587559756075617562756375647565756675677568756975707571757275737574757575767577757875797580758175827583758475857586758775887589759075917592759375947595759675977598759976007601760276037604760576067607760876097610761176127613761476157616761776187619762076217622762376247625762676277628762976307631763276337634763576367637763876397640764176427643764476457646764776487649765076517652765376547655765676577658765976607661766276637664766576667667766876697670767176727673767476757676767776787679768076817682768376847685768676877688768976907691769276937694769576967697769876997700770177027703770477057706770777087709771077117712771377147715771677177718771977207721772277237724772577267727772877297730773177327733773477357736773777387739774077417742774377447745774677477748774977507751775277537754775577567757775877597760776177627763776477657766776777687769777077717772777377747775777677777778777977807781778277837784778577867787778877897790779177927793779477957796779777987799780078017802780378047805780678077808780978107811781278137814781578167817781878197820782178227823782478257826782778287829783078317832783378347835783678377838783978407841784278437844784578467847784878497850785178527853785478557856785778587859786078617862786378647865786678677868786978707871787278737874787578767877787878797880788178827883788478857886788778887889789078917892789378947895789678977898789979007901790279037904790579067907790879097910791179127913791479157916791779187919792079217922792379247925792679277928792979307931793279337934793579367937793879397940794179427943794479457946794779487949795079517952795379547955795679577958795979607961796279637964796579667967796879697970797179727973797479757976797779787979798079817982798379847985798679877988798979907991799279937994799579967997799879998000800180028003800480058006800780088009801080118012801380148015801680178018801980208021802280238024802580268027802880298030803180328033803480358036803780388039804080418042804380448045804680478048804980508051805280538054805580568057805880598060806180628063806480658066806780688069807080718072807380748075807680778078807980808081808280838084808580868087808880898090809180928093809480958096809780988099810081018102810381048105810681078108810981108111811281138114811581168117811881198120812181228123812481258126812781288129813081318132813381348135813681378138813981408141814281438144814581468147814881498150815181528153815481558156815781588159816081618162816381648165816681678168816981708171817281738174817581768177817881798180818181828183818481858186818781888189819081918192819381948195819681978198819982008201820282038204820582068207820882098210821182128213821482158216821782188219822082218222822382248225822682278228822982308231823282338234823582368237823882398240824182428243824482458246824782488249825082518252825382548255825682578258825982608261826282638264826582668267826882698270827182728273827482758276827782788279828082818282828382848285828682878288828982908291829282938294829582968297829882998300830183028303830483058306830783088309831083118312831383148315831683178318831983208321832283238324832583268327832883298330833183328333833483358336833783388339834083418342834383448345834683478348834983508351835283538354835583568357835883598360836183628363836483658366836783688369837083718372837383748375837683778378837983808381838283838384838583868387838883898390839183928393839483958396839783988399840084018402840384048405840684078408840984108411841284138414841584168417841884198420842184228423842484258426842784288429843084318432843384348435843684378438843984408441844284438444844584468447844884498450845184528453845484558456845784588459846084618462846384648465846684678468846984708471847284738474847584768477847884798480848184828483848484858486848784888489849084918492849384948495849684978498849985008501850285038504850585068507850885098510851185128513851485158516851785188519852085218522852385248525852685278528852985308531853285338534853585368537853885398540854185428543854485458546854785488549855085518552855385548555855685578558855985608561856285638564856585668567856885698570857185728573857485758576857785788579858085818582858385848585858685878588858985908591859285938594859585968597859885998600860186028603860486058606860786088609861086118612861386148615861686178618861986208621862286238624862586268627862886298630863186328633863486358636863786388639864086418642864386448645864686478648864986508651865286538654865586568657865886598660866186628663866486658666866786688669867086718672867386748675867686778678867986808681868286838684868586868687868886898690869186928693869486958696869786988699870087018702870387048705870687078708870987108711871287138714871587168717871887198720872187228723872487258726872787288729873087318732873387348735873687378738873987408741874287438744874587468747874887498750875187528753875487558756875787588759876087618762876387648765876687678768876987708771877287738774877587768777877887798780878187828783878487858786878787888789879087918792879387948795879687978798879988008801880288038804880588068807880888098810881188128813881488158816881788188819882088218822882388248825882688278828882988308831883288338834883588368837883888398840884188428843884488458846884788488849885088518852885388548855885688578858885988608861886288638864886588668867886888698870887188728873887488758876887788788879888088818882888388848885888688878888888988908891889288938894889588968897889888998900890189028903890489058906890789088909891089118912891389148915891689178918891989208921892289238924892589268927892889298930893189328933893489358936893789388939894089418942894389448945894689478948894989508951895289538954895589568957895889598960896189628963896489658966896789688969897089718972897389748975897689778978897989808981898289838984898589868987898889898990899189928993899489958996899789988999900090019002900390049005900690079008900990109011901290139014901590169017901890199020902190229023902490259026902790289029903090319032903390349035903690379038903990409041904290439044904590469047904890499050905190529053905490559056905790589059906090619062906390649065906690679068906990709071907290739074907590769077907890799080908190829083908490859086908790889089909090919092909390949095909690979098909991009101910291039104910591069107910891099110911191129113911491159116911791189119912091219122912391249125912691279128912991309131913291339134913591369137913891399140914191429143914491459146914791489149915091519152915391549155915691579158915991609161916291639164916591669167916891699170917191729173917491759176917791789179918091819182918391849185918691879188918991909191919291939194919591969197919891999200920192029203920492059206920792089209921092119212921392149215921692179218921992209221922292239224922592269227922892299230923192329233923492359236923792389239924092419242924392449245924692479248924992509251925292539254925592569257925892599260926192629263926492659266926792689269927092719272927392749275927692779278927992809281928292839284928592869287928892899290929192929293929492959296929792989299930093019302930393049305930693079308930993109311931293139314931593169317931893199320932193229323932493259326932793289329933093319332933393349335933693379338933993409341934293439344934593469347934893499350935193529353935493559356935793589359936093619362936393649365936693679368936993709371937293739374937593769377937893799380938193829383938493859386938793889389939093919392939393949395939693979398939994009401940294039404940594069407940894099410941194129413941494159416941794189419942094219422942394249425942694279428942994309431943294339434943594369437943894399440944194429443944494459446944794489449945094519452945394549455945694579458945994609461946294639464946594669467946894699470947194729473947494759476947794789479948094819482948394849485948694879488948994909491949294939494949594969497949894999500950195029503950495059506950795089509951095119512951395149515951695179518951995209521952295239524952595269527952895299530953195329533953495359536953795389539954095419542954395449545954695479548954995509551955295539554955595569557955895599560956195629563956495659566956795689569957095719572957395749575957695779578957995809581958295839584958595869587958895899590959195929593959495959596959795989599960096019602960396049605960696079608960996109611961296139614961596169617961896199620962196229623962496259626962796289629963096319632963396349635963696379638963996409641964296439644964596469647964896499650965196529653965496559656965796589659966096619662966396649665966696679668966996709671967296739674967596769677967896799680968196829683968496859686968796889689969096919692969396949695969696979698969997009701970297039704970597069707970897099710971197129713971497159716971797189719972097219722972397249725972697279728972997309731973297339734973597369737973897399740974197429743974497459746974797489749975097519752975397549755975697579758975997609761976297639764976597669767976897699770977197729773977497759776977797789779978097819782978397849785978697879788978997909791979297939794979597969797979897999800980198029803980498059806980798089809981098119812981398149815981698179818981998209821982298239824982598269827982898299830983198329833983498359836983798389839984098419842984398449845984698479848984998509851985298539854985598569857985898599860986198629863986498659866986798689869987098719872987398749875987698779878987998809881988298839884988598869887988898899890989198929893989498959896989798989899990099019902990399049905990699079908990999109911991299139914991599169917991899199920992199229923992499259926992799289929993099319932993399349935993699379938993999409941994299439944994599469947994899499950995199529953995499559956995799589959996099619962996399649965996699679968996999709971997299739974997599769977997899799980998199829983998499859986998799889989999099919992999399949995999699979998999910000100011000210003100041000510006100071000810009100101001110012100131001410015100161001710018100191002010021100221002310024100251002610027100281002910030100311003210033100341003510036100371003810039100401004110042100431004410045100461004710048100491005010051100521005310054100551005610057100581005910060100611006210063100641006510066100671006810069100701007110072100731007410075100761007710078100791008010081100821008310084100851008610087100881008910090100911009210093100941009510096100971009810099101001010110102101031010410105101061010710108101091011010111101121011310114101151011610117101181011910120101211012210123101241012510126101271012810129101301013110132101331013410135101361013710138101391014010141101421014310144101451014610147101481014910150101511015210153101541015510156101571015810159101601016110162101631016410165101661016710168101691017010171101721017310174101751017610177101781017910180101811018210183101841018510186101871018810189101901019110192101931019410195101961019710198101991020010201102021020310204102051020610207102081020910210102111021210213102141021510216102171021810219102201022110222102231022410225102261022710228102291023010231102321023310234102351023610237102381023910240102411024210243102441024510246102471024810249102501025110252102531025410255102561025710258102591026010261102621026310264102651026610267102681026910270102711027210273102741027510276102771027810279102801028110282102831028410285102861028710288102891029010291102921029310294102951029610297102981029910300103011030210303103041030510306103071030810309103101031110312103131031410315103161031710318103191032010321103221032310324103251032610327103281032910330103311033210333103341033510336103371033810339103401034110342103431034410345103461034710348103491035010351103521035310354103551035610357103581035910360103611036210363103641036510366103671036810369103701037110372103731037410375103761037710378103791038010381103821038310384103851038610387103881038910390103911039210393103941039510396103971039810399104001040110402104031040410405104061040710408104091041010411104121041310414104151041610417104181041910420104211042210423104241042510426104271042810429104301043110432104331043410435104361043710438104391044010441104421044310444104451044610447104481044910450104511045210453104541045510456104571045810459104601046110462104631046410465104661046710468104691047010471104721047310474104751047610477104781047910480104811048210483104841048510486104871048810489104901049110492104931049410495104961049710498104991050010501105021050310504105051050610507105081050910510105111051210513105141051510516105171051810519105201052110522105231052410525105261052710528105291053010531105321053310534105351053610537105381053910540105411054210543105441054510546105471054810549105501055110552105531055410555105561055710558105591056010561105621056310564105651056610567105681056910570105711057210573105741057510576105771057810579105801058110582105831058410585105861058710588105891059010591105921059310594105951059610597105981059910600106011060210603106041060510606106071060810609106101061110612106131061410615106161061710618106191062010621106221062310624106251062610627106281062910630106311063210633106341063510636106371063810639106401064110642106431064410645106461064710648106491065010651106521065310654106551065610657106581065910660106611066210663106641066510666106671066810669106701067110672106731067410675106761067710678106791068010681106821068310684106851068610687106881068910690106911069210693106941069510696106971069810699107001070110702107031070410705107061070710708107091071010711107121071310714107151071610717107181071910720107211072210723107241072510726107271072810729107301073110732107331073410735107361073710738107391074010741107421074310744107451074610747107481074910750107511075210753107541075510756107571075810759107601076110762107631076410765107661076710768107691077010771107721077310774107751077610777107781077910780107811078210783107841078510786107871078810789107901079110792107931079410795107961079710798107991080010801108021080310804108051080610807108081080910810108111081210813108141081510816108171081810819108201082110822108231082410825108261082710828108291083010831108321083310834108351083610837108381083910840108411084210843108441084510846108471084810849108501085110852108531085410855108561085710858108591086010861
  1. /*##############################################################################
  2. HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. ############################################################################## */
  13. #include "jlib.hpp"
  14. #include "workunit.hpp"
  15. #include "jprop.hpp"
  16. #include "jmisc.hpp"
  17. #include "jexcept.hpp"
  18. #include "jiter.ipp"
  19. #include "jptree.hpp"
  20. #include "jtime.ipp"
  21. #include "jencrypt.hpp"
  22. #include "junicode.hpp"
  23. #include "jlzw.hpp"
  24. #include "jregexp.hpp"
  25. #include "eclrtl.hpp"
  26. #include "deftype.hpp"
  27. #include <time.h>
  28. #include "mpbase.hpp"
  29. #include "daclient.hpp"
  30. #include "dadfs.hpp"
  31. #include "dafdesc.hpp"
  32. #include "dasds.hpp"
  33. #include "danqs.hpp"
  34. #include "dautils.hpp"
  35. #include "dllserver.hpp"
  36. #include "thorplugin.hpp"
  37. #include "thorhelper.hpp"
  38. #include "workflow.hpp"
  39. #include "nbcd.hpp"
  40. #include "seclib.hpp"
  41. #include "wuerror.hpp"
  42. #include "wujobq.hpp"
  43. #include "environment.hpp"
  44. #include "workunit.ipp"
  45. static int workUnitTraceLevel = 1;
  46. static StringBuffer &getXPath(StringBuffer &wuRoot, const char *wuid)
  47. {
  48. // MORE - can fold in the date
  49. return wuRoot.append("/WorkUnits/").append(wuid);
  50. }
  51. //To be called by eclserver, but esp etc. won't know, so we need to store it.
  52. static StringBuffer & appendLibrarySuffix(StringBuffer & suffix)
  53. {
  54. #ifdef _WIN32
  55. suffix.append("W");
  56. #else
  57. suffix.append("L");
  58. #endif
  59. #ifdef __64BIT__
  60. suffix.append("64");
  61. #else
  62. suffix.append("32");
  63. #endif
  64. return suffix;
  65. }
  66. typedef MapStringTo<bool> UniqueScopes;
  67. static void wuAccessError(const char *username, const char *action, const char *wuscope, const char *wuid, bool excpt, bool log)
  68. {
  69. StringBuffer err;
  70. err.append("Workunit Access Denied - action: ").append(action).append(" user:").append(username ? username : "<Unknown>");
  71. if (wuid)
  72. err.append(" workunit:").append(wuid);
  73. if (wuscope)
  74. err.append(" scope:").append(wuscope);
  75. //MORE - we would need more information passed in from outside if we want to make the audit message format the same as from higher level ESP calls
  76. SYSLOG(AUDIT_TYPE_ACCESS_FAILURE, err.str());
  77. if (log)
  78. LOG(MCuserError, "%s", err.str());
  79. if (excpt)
  80. throw MakeStringException(WUERR_AccessError, "%s", err.str());
  81. }
  82. static bool checkWuScopeSecAccess(const char *wuscope, ISecManager *secmgr, ISecUser *secuser, int required, const char *action, bool excpt, bool log)
  83. {
  84. if (!secmgr || !secuser)
  85. return true;
  86. bool ret = secmgr->authorizeEx(RT_WORKUNIT_SCOPE, *secuser, wuscope)>=required;
  87. if (!ret && (log || excpt))
  88. wuAccessError(secuser->getName(), action, wuscope, NULL, excpt, log);
  89. return ret;
  90. }
  91. static bool checkWuScopeListSecAccess(const char *wuscope, ISecResourceList *scopes, int required, const char *action, bool excpt, bool log)
  92. {
  93. if (!scopes)
  94. return true;
  95. bool ret=true;
  96. if (wuscope)
  97. {
  98. Owned<ISecResource> res=scopes->getResource(wuscope);
  99. if (!res || res->getAccessFlags()<required)
  100. ret=false;
  101. }
  102. else
  103. {
  104. for (int seq=0; ret && seq<scopes->count(); seq++)
  105. {
  106. ISecResource *res=scopes->queryResource(seq);
  107. if (res && res->getAccessFlags()<required)
  108. return false;
  109. }
  110. }
  111. if (!ret && (log || excpt))
  112. wuAccessError(NULL, action, wuscope, NULL, excpt, log);
  113. return ret;
  114. }
  115. static bool checkWuSecAccess(IConstWorkUnit &cw, ISecManager *secmgr, ISecUser *secuser, int required, const char *action, bool excpt, bool log)
  116. {
  117. if (!secmgr || !secuser)
  118. return true;
  119. bool ret=secmgr->authorizeEx(RT_WORKUNIT_SCOPE, *secuser, cw.queryWuScope())>=required;
  120. if (!ret && (log || excpt))
  121. {
  122. wuAccessError(secuser->getName(), action, cw.queryWuScope(), cw.queryWuid(), excpt, log);
  123. }
  124. return ret;
  125. }
  126. static bool checkWuSecAccess(const char *wuid, ISecManager *secmgr, ISecUser *secuser, int required, const char *action, bool excpt, bool log)
  127. {
  128. if (!secmgr || !secuser)
  129. return true;
  130. Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
  131. Owned<IConstWorkUnit> cw = factory->openWorkUnit(wuid);
  132. bool ret=secmgr->authorizeEx(RT_WORKUNIT_SCOPE, *secuser, cw->queryWuScope())>=required;
  133. if (!ret && (log || excpt))
  134. {
  135. wuAccessError(secuser->getName(), action, cw->queryWuScope(), cw->queryWuid(), excpt, log);
  136. }
  137. return ret;
  138. }
  139. void doDescheduleWorkkunit(char const * wuid)
  140. {
  141. StringBuffer xpath;
  142. xpath.append("*/*/*/");
  143. ncnameEscape(wuid, xpath);
  144. Owned<IRemoteConnection> conn = querySDS().connect("/Schedule", myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
  145. if(!conn) return;
  146. Owned<IPropertyTree> root = conn->getRoot();
  147. bool more;
  148. do more = root->removeProp(xpath.str()); while(more);
  149. }
  150. //======================================================
  151. /*
  152. * Graph progress support
  153. */
  154. CWuGraphStats::CWuGraphStats(IPropertyTree *_progress, StatisticCreatorType _creatorType, const char * _creator, const char * _rootScope, unsigned _id)
  155. : progress(_progress), creatorType(_creatorType), creator(_creator), id(_id)
  156. {
  157. StatisticScopeType scopeType = SSTgraph;
  158. StatsScopeId rootScopeId;
  159. verifyex(rootScopeId.setScopeText(_rootScope));
  160. collector.setown(createStatisticsGatherer(_creatorType, _creator, rootScopeId));
  161. }
  162. void CWuGraphStats::beforeDispose()
  163. {
  164. Owned<IStatisticCollection> stats = collector->getResult();
  165. MemoryBuffer compressed;
  166. {
  167. MemoryBuffer serialized;
  168. serializeStatisticCollection(serialized, stats);
  169. compressToBuffer(compressed, serialized.length(), serialized.toByteArray());
  170. }
  171. unsigned minActivity = 0;
  172. unsigned maxActivity = 0;
  173. stats->getMinMaxActivity(minActivity, maxActivity);
  174. StringBuffer tag;
  175. tag.append("sg").append(id);
  176. //Replace the particular subgraph statistics added by this creator
  177. StringBuffer qualified(tag);
  178. qualified.append("[@creator='").append(creator).append("']");
  179. progress->removeProp(qualified);
  180. IPropertyTree * subgraph = progress->addPropTree(tag);
  181. subgraph->setProp("@c", queryCreatorTypeName(creatorType));
  182. subgraph->setProp("@creator", creator);
  183. subgraph->setPropInt("@minActivity", minActivity);
  184. subgraph->setPropInt("@maxActivity", maxActivity);
  185. subgraph->setPropBin("Stats", compressed.length(), compressed.toByteArray());
  186. if (!progress->getPropBool("@stats", false))
  187. progress->setPropBool("@stats", true);
  188. }
  189. IStatisticGatherer & CWuGraphStats::queryStatsBuilder()
  190. {
  191. return *collector;
  192. }
  193. class CConstGraphProgress : public CInterface, implements IConstWUGraphProgress
  194. {
  195. public:
  196. IMPLEMENT_IINTERFACE;
  197. CConstGraphProgress(const char *_wuid, const char *_graphName, IPropertyTree *_progress) : wuid(_wuid), graphName(_graphName), progress(_progress)
  198. {
  199. if (!progress)
  200. progress.setown(createPTree());
  201. formatVersion = progress->getPropInt("@format", PROGRESS_FORMAT_V);
  202. }
  203. virtual IPropertyTree * getProgressTree()
  204. {
  205. if (progress->getPropBool("@stats"))
  206. return createProcessTreeFromStats(); // Should we cache that?
  207. return LINK(progress);
  208. }
  209. virtual unsigned queryFormatVersion()
  210. {
  211. return formatVersion;
  212. }
  213. protected:
  214. CConstGraphProgress(const char *_wuid, const char *_graphName) : wuid(_wuid), graphName(_graphName)
  215. {
  216. formatVersion = PROGRESS_FORMAT_V;
  217. }
  218. static void expandStats(IPropertyTree * target, IStatisticCollection & collection)
  219. {
  220. StringBuffer formattedValue;
  221. unsigned numStats = collection.getNumStatistics();
  222. for (unsigned i=0; i < numStats; i++)
  223. {
  224. StatisticKind kind;
  225. unsigned __int64 value;
  226. collection.getStatistic(kind, value, i);
  227. formatStatistic(formattedValue.clear(), value, kind);
  228. target->setProp(queryTreeTag(kind), formattedValue);
  229. }
  230. }
  231. void expandProcessTreeFromStats(IPropertyTree * rootTarget, IPropertyTree * target, IStatisticCollection * collection)
  232. {
  233. expandStats(target, *collection);
  234. StringBuffer scopeName;
  235. Owned<IStatisticCollectionIterator> activityIter = &collection->getScopes(NULL);
  236. ForEach(*activityIter)
  237. {
  238. IStatisticCollection & cur = activityIter->query();
  239. cur.getScope(scopeName.clear());
  240. const char * id = scopeName.str();
  241. const char * tag;
  242. IPropertyTree * curTarget = target;
  243. switch (cur.queryScopeType())
  244. {
  245. case SSTedge:
  246. tag = "edge";
  247. id += strlen(EdgeScopePrefix);
  248. break;
  249. case SSTactivity:
  250. tag = "node";
  251. id += strlen(ActivityScopePrefix);
  252. break;
  253. case SSTsubgraph:
  254. //All subgraphs are added a root elements in the progress tree
  255. curTarget = rootTarget;
  256. tag = "node";
  257. id += strlen(SubGraphScopePrefix);
  258. break;
  259. case SSTfunction:
  260. //MORE:Should function scopes be included in the graph scope somehow, and if so how?
  261. continue;
  262. default:
  263. throwUnexpected();
  264. }
  265. IPropertyTree * next = curTarget->addPropTree(tag);
  266. next->setProp("@id", id);
  267. expandProcessTreeFromStats(rootTarget, next, &cur);
  268. }
  269. }
  270. IPropertyTree * createProcessTreeFromStats()
  271. {
  272. MemoryBuffer compressed;
  273. MemoryBuffer serialized;
  274. Owned<IPropertyTree> progressTree = createPTree();
  275. Owned<IPropertyTreeIterator> iter = progress->getElements("sg*");
  276. ForEach(*iter)
  277. {
  278. IPropertyTree & curSubGraph = iter->query();
  279. curSubGraph.getPropBin("Stats", compressed.clear());
  280. //Protect against updates that delete the stats while we are iterating
  281. if (compressed.length())
  282. {
  283. decompressToBuffer(serialized.clear(), compressed);
  284. Owned<IStatisticCollection> collection = createStatisticCollection(serialized);
  285. expandProcessTreeFromStats(progressTree, progressTree, collection);
  286. }
  287. }
  288. return progressTree.getClear();
  289. }
  290. protected:
  291. Linked<IPropertyTree> progress;
  292. StringAttr wuid, graphName;
  293. unsigned formatVersion;
  294. };
  295. extern WORKUNIT_API IConstWUGraphProgress *createConstGraphProgress(const char *_wuid, const char *_graphName, IPropertyTree *_progress)
  296. {
  297. return new CConstGraphProgress(_wuid, _graphName, _progress);
  298. }
  299. //--------------------------------------------------------------------------------------------------------------------
  300. class ExtractedStatistic : public CInterfaceOf<IConstWUStatistic>
  301. {
  302. public:
  303. virtual IStringVal & getDescription(IStringVal & str, bool createDefault) const
  304. {
  305. if (!description && createDefault)
  306. {
  307. switch (kind)
  308. {
  309. case StTimeElapsed:
  310. {
  311. if (scopeType != SSTsubgraph)
  312. break;
  313. //Create a default description for a root subgraph
  314. const char * colon = strchr(scope, ':');
  315. if (!colon)
  316. break;
  317. const char * subgraph = colon+1;
  318. //Check for nested subgraph
  319. if (strchr(subgraph, ':'))
  320. break;
  321. assertex(strncmp(subgraph, SubGraphScopePrefix, strlen(SubGraphScopePrefix)) == 0);
  322. StringAttr graphname;
  323. graphname.set(scope, colon - scope);
  324. unsigned subId = atoi(subgraph + strlen(SubGraphScopePrefix));
  325. StringBuffer desc;
  326. formatGraphTimerLabel(desc, graphname, 0, subId);
  327. str.set(desc);
  328. return str;
  329. }
  330. }
  331. }
  332. str.set(description);
  333. return str;
  334. }
  335. virtual IStringVal & getCreator(IStringVal & str) const
  336. {
  337. str.set(creator);
  338. return str;
  339. }
  340. virtual IStringVal & getScope(IStringVal & str) const
  341. {
  342. str.set(scope);
  343. return str;
  344. }
  345. virtual IStringVal & getFormattedValue(IStringVal & str) const
  346. {
  347. StringBuffer formatted;
  348. formatStatistic(formatted, value, measure);
  349. str.set(formatted);
  350. return str;
  351. }
  352. virtual StatisticMeasure getMeasure() const
  353. {
  354. return measure;
  355. }
  356. virtual StatisticKind getKind() const
  357. {
  358. return kind;
  359. }
  360. virtual StatisticCreatorType getCreatorType() const
  361. {
  362. return creatorType;
  363. }
  364. virtual StatisticScopeType getScopeType() const
  365. {
  366. return scopeType;
  367. }
  368. virtual unsigned __int64 getValue() const
  369. {
  370. return value;
  371. }
  372. virtual unsigned __int64 getCount() const
  373. {
  374. return count;
  375. }
  376. virtual unsigned __int64 getMax() const
  377. {
  378. return max;
  379. }
  380. virtual unsigned __int64 getTimestamp() const
  381. {
  382. return timeStamp;
  383. }
  384. virtual bool matches(const IStatisticsFilter * filter) const
  385. {
  386. return filter->matches(creatorType, creator, scopeType, scope, measure, kind, value);
  387. }
  388. public:
  389. StringAttr creator;
  390. StringAttr description;
  391. StringBuffer scope;
  392. StatisticMeasure measure;
  393. StatisticKind kind;
  394. StatisticCreatorType creatorType;
  395. StatisticScopeType scopeType;
  396. unsigned __int64 value;
  397. unsigned __int64 count;
  398. unsigned __int64 max;
  399. unsigned __int64 timeStamp;
  400. };
  401. class CConstGraphProgressStatisticsIterator : public CInterfaceOf<IConstWUStatisticIterator>
  402. {
  403. public:
  404. CConstGraphProgressStatisticsIterator(const char * wuid, const IStatisticsFilter * _filter) : filter(_filter)
  405. {
  406. if (filter)
  407. scopes.appendList(filter->queryScope(), ":");
  408. const char * searchGraph = "*";
  409. if (scopes.ordinality())
  410. searchGraph = scopes.item(0);
  411. rootPath.append("/GraphProgress/").append(wuid).append('/');
  412. bool singleGraph = false;
  413. if (!containsWildcard(searchGraph))
  414. {
  415. rootPath.append(searchGraph).append("/");
  416. singleGraph = true;
  417. }
  418. //Don't lock the statistics while we iterate - any partial updates must not cause problems
  419. if (daliClientActive())
  420. conn.setown(querySDS().connect(rootPath.str(), myProcessSession(), RTM_NONE, SDS_LOCK_TIMEOUT));
  421. if (conn && !singleGraph)
  422. graphIter.setown(conn->queryRoot()->getElements("*"));
  423. curStat.setown(new ExtractedStatistic);
  424. //These are currently constant for all graph statistics instances
  425. curStat->count = 1;
  426. curStat->max = 0;
  427. valid = false;
  428. }
  429. virtual IConstWUStatistic & query()
  430. {
  431. return *curStat;
  432. }
  433. virtual bool first()
  434. {
  435. valid = false;
  436. if (!conn)
  437. return false;
  438. if (graphIter && !graphIter->first())
  439. return false;
  440. ensureUniqueStatistic();
  441. if (!firstSubGraph())
  442. {
  443. if (!nextGraph())
  444. return false;
  445. }
  446. valid = true;
  447. return true;
  448. }
  449. virtual bool next()
  450. {
  451. ensureUniqueStatistic();
  452. if (!nextStatistic())
  453. {
  454. if (!nextSubGraph())
  455. {
  456. if (!nextGraph())
  457. {
  458. valid = false;
  459. return false;
  460. }
  461. }
  462. }
  463. return true;
  464. }
  465. virtual bool isValid()
  466. {
  467. return valid;
  468. }
  469. protected:
  470. bool firstSubGraph()
  471. {
  472. IPropertyTree & graphNode = graphIter ? graphIter->query() : *conn->queryRoot();
  473. const char * xpath = "sg*";
  474. StringBuffer childXpath;
  475. if (scopes.isItem(1))
  476. {
  477. const char * scope1 = scopes.item(1);
  478. if (strnicmp(scope1, "sg", 2) == 0)
  479. {
  480. childXpath.append(scope1);
  481. xpath = childXpath.str();
  482. }
  483. }
  484. subgraphIter.setown(graphNode.getElements(xpath));
  485. if (!subgraphIter)
  486. subgraphIter.setown(graphNode.getElements("sg0"));
  487. if (!subgraphIter->first())
  488. return false;
  489. if (firstStat())
  490. return true;
  491. return nextSubGraph();
  492. }
  493. bool nextSubGraph()
  494. {
  495. for (;;)
  496. {
  497. if (!subgraphIter->next())
  498. return false;
  499. if (firstStat())
  500. return true;
  501. }
  502. }
  503. bool nextGraph()
  504. {
  505. if (!graphIter)
  506. return false;
  507. for (;;)
  508. {
  509. if (!graphIter->next())
  510. return false;
  511. if (firstSubGraph())
  512. return true;
  513. }
  514. }
  515. bool firstStat()
  516. {
  517. IPropertyTree & curSubGraph = subgraphIter->query();
  518. if (!checkSubGraph())
  519. return false;
  520. curSubGraph.getPropBin("Stats", compressed.clear());
  521. //Don't crash on old format progress...
  522. if (compressed.length() == 0)
  523. return false;
  524. decompressToBuffer(serialized.clear(), compressed);
  525. Owned<IStatisticCollection> collection = createStatisticCollection(serialized);
  526. curStat->timeStamp = collection->queryWhenCreated();
  527. return beginCollection(*collection);
  528. }
  529. bool beginCollection(IStatisticCollection & collection)
  530. {
  531. collections.append(OLINK(collection));
  532. numStats = collection.getNumStatistics();
  533. curStatIndex = 0;
  534. if (checkScope())
  535. {
  536. if (curStatIndex < numStats)
  537. {
  538. if (checkStatistic())
  539. return true;
  540. return nextStatistic();
  541. }
  542. }
  543. return nextChildScope();
  544. }
  545. bool nextStatistic()
  546. {
  547. //Finish iterating the statistics at this level.
  548. while (++curStatIndex < numStats)
  549. {
  550. if (checkStatistic())
  551. return true;
  552. }
  553. return nextChildScope();
  554. }
  555. bool nextChildScope()
  556. {
  557. for (;;)
  558. {
  559. if (collections.ordinality() == 0)
  560. return false;
  561. IStatisticCollection * curCollection = &collections.tos();
  562. if (childIterators.ordinality() < collections.ordinality())
  563. {
  564. if (!filter || filter->recurseChildScopes(curStat->scopeType, curStat->scope))
  565. {
  566. //Start iterating the children for the current collection
  567. childIterators.append(curCollection->getScopes(NULL));
  568. if (!childIterators.tos().first())
  569. {
  570. finishCollection();
  571. continue;
  572. }
  573. }
  574. else
  575. {
  576. //Don't walk the child scopes
  577. collections.pop();
  578. continue;
  579. }
  580. }
  581. else if (!childIterators.tos().next())
  582. {
  583. finishCollection();
  584. continue;
  585. }
  586. if (beginCollection(childIterators.tos().query()))
  587. return true;
  588. }
  589. }
  590. void finishCollection()
  591. {
  592. collections.pop();
  593. childIterators.pop();
  594. }
  595. bool checkSubGraph()
  596. {
  597. if (!filter)
  598. return true;
  599. IPropertyTree & curSubGraph = subgraphIter->query();
  600. curStat->creatorType = queryCreatorType(curSubGraph.queryProp("@c"));
  601. curStat->creator.set(curSubGraph.queryProp("@creator"));
  602. return filter->matches(curStat->creatorType, curStat->creator, SSTall, NULL, SMeasureAll, StKindAll, AnyStatisticValue);
  603. }
  604. bool checkScope()
  605. {
  606. if (!filter)
  607. return true;
  608. IStatisticCollection * collection = &collections.tos();
  609. curStat->scopeType = collection->queryScopeType();
  610. collection->getFullScope(curStat->scope.clear());
  611. return filter->matches(SCTall, NULL, curStat->scopeType, curStat->scope, SMeasureAll, StKindAll, AnyStatisticValue);
  612. }
  613. bool checkStatistic()
  614. {
  615. IStatisticCollection & collection = collections.tos();
  616. collection.getStatistic(curStat->kind, curStat->value, curStatIndex);
  617. curStat->measure = queryMeasure(curStat->kind);
  618. if (!filter)
  619. return true;
  620. if (!filter->matches(SCTall, NULL, SSTall, NULL, curStat->measure, curStat->kind, curStat->value))
  621. return false;
  622. return true;
  623. }
  624. void ensureUniqueStatistic()
  625. {
  626. //If something else has linked this statistic, clone a unique one.
  627. if (curStat->IsShared())
  628. curStat.setown(new ExtractedStatistic(*curStat));
  629. }
  630. private:
  631. Owned<IRemoteConnection> conn;
  632. Owned<ExtractedStatistic> curStat;
  633. const IStatisticsFilter * filter;
  634. StringArray scopes;
  635. StringBuffer rootPath;
  636. Owned<IPropertyTreeIterator> graphIter;
  637. Owned<IPropertyTreeIterator> subgraphIter;
  638. IArrayOf<IStatisticCollection> collections;
  639. IArrayOf<IStatisticCollectionIterator> childIterators;
  640. MemoryBuffer compressed;
  641. MemoryBuffer serialized;
  642. unsigned numStats;
  643. unsigned curStatIndex;
  644. bool valid;
  645. };
  646. //--------------------------------------------------------------------------------------------------------------------
  647. mapEnums states[] = {
  648. { WUStateUnknown, "unknown" },
  649. { WUStateCompiled, "compiled" },
  650. { WUStateRunning, "running" },
  651. { WUStateCompleted, "completed" },
  652. { WUStateFailed, "failed" },
  653. { WUStateArchived, "archived" },
  654. { WUStateAborting, "aborting" },
  655. { WUStateAborted, "aborted" },
  656. { WUStateBlocked, "blocked" },
  657. { WUStateSubmitted, "submitted" },
  658. { WUStateScheduled, "scheduled" },
  659. { WUStateCompiling, "compiling" },
  660. { WUStateWait, "wait" },
  661. { WUStateUploadingFiles, "uploading_files" },
  662. { WUStateDebugPaused, "debugging" },
  663. { WUStateDebugRunning, "debug_running" },
  664. { WUStatePaused, "paused" },
  665. { WUStateSize, NULL }
  666. };
  667. mapEnums actions[] = {
  668. { WUActionUnknown, "unknown" },
  669. { WUActionCompile, "compile" },
  670. { WUActionCheck, "check" },
  671. { WUActionRun, "run" },
  672. { WUActionExecuteExisting, "execute" },
  673. { WUActionPause, "pause" },
  674. { WUActionPauseNow, "pausenow" },
  675. { WUActionResume, "resume" },
  676. { WUActionSize, NULL },
  677. };
  678. mapEnums priorityClasses[] = {
  679. { PriorityClassUnknown, "unknown" },
  680. { PriorityClassLow, "low" },
  681. { PriorityClassNormal, "normal" },
  682. { PriorityClassHigh, "high" },
  683. { PriorityClassSize, NULL },
  684. };
  685. const char * getWorkunitStateStr(WUState state)
  686. {
  687. dbgassertex(state < WUStateSize);
  688. return states[state].str; // MORE - should be using getEnumText, or need to take steps to ensure values remain contiguous and in order.
  689. }
  690. void setEnum(IPropertyTree *p, const char *propname, int value, const mapEnums *map)
  691. {
  692. const char *defval = map->str;
  693. while (map->str)
  694. {
  695. if (value==map->val)
  696. {
  697. p->setProp(propname, map->str);
  698. return;
  699. }
  700. map++;
  701. }
  702. assertex(!"Unexpected value in setEnum");
  703. p->setProp(propname, defval);
  704. }
  705. static int getEnum(const char *v, const mapEnums *map)
  706. {
  707. if (v && *v)
  708. {
  709. while (map->str)
  710. {
  711. if (stricmp(v, map->str)==0)
  712. return map->val;
  713. map++;
  714. }
  715. assertex(!"Unexpected value in getEnum");
  716. }
  717. return 0;
  718. }
  719. static int getEnum(const IPropertyTree *p, const char *propname, const mapEnums *map)
  720. {
  721. return getEnum(p->queryProp(propname),map);
  722. }
  723. const char * getWorkunitActionStr(WUAction action)
  724. {
  725. return getEnumText(action, actions);
  726. }
  727. WUAction getWorkunitAction(const char *actionStr)
  728. {
  729. return (WUAction) getEnum(actionStr, actions);
  730. }
  731. //==========================================================================================
  732. class CLightweightWorkunitInfo : public CInterfaceOf<IConstWorkUnitInfo>
  733. {
  734. public:
  735. CLightweightWorkunitInfo(IPropertyTree &p)
  736. {
  737. wuid.set(p.queryName());
  738. user.set(p.queryProp("@submitID"));
  739. jobName.set(p.queryProp("@jobName"));
  740. clusterName.set(p.queryProp("@clusterName"));
  741. timeScheduled.set(p.queryProp("@timeScheduled"));
  742. state = (WUState) getEnum(&p, "@state", states);
  743. action = (WUAction) getEnum(&p, "Action", actions);
  744. priority = (WUPriorityClass) getEnum(&p, "@priorityClass", priorityClasses);
  745. priorityLevel = calcPriorityValue(&p);
  746. wuscope.set(p.queryProp("@scope"));
  747. appvalues.loadBranch(&p,"Application");
  748. totalThorTime = (unsigned)nanoToMilli(extractTimeCollatable(p.queryProp("@totalThorTime"), false));
  749. _isProtected = p.getPropBool("@protected", false);
  750. }
  751. virtual const char *queryWuid() const { return wuid.str(); }
  752. virtual const char *queryUser() const { return user.str(); }
  753. virtual const char *queryJobName() const { return jobName.str(); }
  754. virtual const char *queryClusterName() const { return clusterName.str(); }
  755. virtual const char *queryWuScope() const { return wuscope.str(); }
  756. virtual WUState getState() const { return state; }
  757. virtual const char *queryStateDesc() const { return getEnumText(state, states); }
  758. virtual WUAction getAction() const { return action; }
  759. virtual const char *queryActionDesc() const { return getEnumText(action, actions); }
  760. virtual WUPriorityClass getPriority() const { return priority; }
  761. virtual const char *queryPriorityDesc() const { return getEnumText(priority, priorityClasses); }
  762. virtual int getPriorityLevel() const { return priorityLevel; }
  763. virtual bool isProtected() const { return _isProtected; }
  764. virtual IJlibDateTime & getTimeScheduled(IJlibDateTime & val) const
  765. {
  766. if (timeScheduled.length())
  767. val.setGmtString(timeScheduled.str());
  768. return val;
  769. }
  770. virtual unsigned getTotalThorTime() const { return totalThorTime; };
  771. virtual IConstWUAppValueIterator & getApplicationValues() const { return *new CArrayIteratorOf<IConstWUAppValue,IConstWUAppValueIterator> (appvalues, 0, (IConstWorkUnitInfo *) this); };
  772. protected:
  773. StringAttr wuid, user, jobName, clusterName, timeScheduled, wuscope;
  774. mutable CachedTags<CLocalWUAppValue,IConstWUAppValue> appvalues;
  775. unsigned totalThorTime;
  776. WUState state;
  777. WUAction action;
  778. WUPriorityClass priority;
  779. int priorityLevel;
  780. bool _isProtected;
  781. };
  782. extern IConstWorkUnitInfo *createConstWorkUnitInfo(IPropertyTree &p)
  783. {
  784. return new CLightweightWorkunitInfo(p);
  785. }
  786. class CDaliWuGraphStats : public CWuGraphStats
  787. {
  788. public:
  789. CDaliWuGraphStats(IRemoteConnection *_conn, StatisticCreatorType _creatorType, const char * _creator, const char * _rootScope, unsigned _id)
  790. : CWuGraphStats(LINK(_conn->queryRoot()), _creatorType, _creator, _rootScope, _id), conn(_conn)
  791. {
  792. }
  793. protected:
  794. Owned<IRemoteConnection> conn;
  795. };
  796. CWorkUnitWatcher::CWorkUnitWatcher(IWorkUnitSubscriber *_subscriber, WUSubscribeOptions flags, const char *wuid) : subscriber(_subscriber)
  797. {
  798. abortId = 0;
  799. stateId = 0;
  800. actionId = 0;
  801. assertex((flags & ~SubscribeOptionAbort) == 0);
  802. if (flags & SubscribeOptionAbort)
  803. {
  804. VStringBuffer xpath("/WorkUnitAborts/%s", wuid);
  805. abortId = querySDS().subscribe(xpath.str(), *this);
  806. }
  807. }
  808. CWorkUnitWatcher::~CWorkUnitWatcher()
  809. {
  810. assertex(abortId==0 && stateId==0 && actionId==0);
  811. }
  812. void CWorkUnitWatcher::unsubscribe()
  813. {
  814. CriticalBlock b(crit);
  815. if (abortId)
  816. querySDS().unsubscribe(abortId);
  817. if (stateId)
  818. querySDS().unsubscribe(stateId);
  819. if (actionId)
  820. querySDS().unsubscribe(actionId);
  821. abortId = 0;
  822. stateId = 0;
  823. actionId = 0;
  824. }
  825. void CWorkUnitWatcher::notify(SubscriptionId id, const char *xpath, SDSNotifyFlags flags, unsigned valueLen, const void *valueData)
  826. {
  827. CriticalBlock b(crit);
  828. if (id==stateId)
  829. subscriber->notify(SubscribeOptionState);
  830. else if (id==actionId)
  831. subscriber->notify(SubscribeOptionAction);
  832. else if (id==abortId)
  833. subscriber->notify(SubscribeOptionAbort);
  834. }
  835. class CDaliWorkUnitWatcher : public CWorkUnitWatcher
  836. {
  837. public:
  838. CDaliWorkUnitWatcher(IWorkUnitSubscriber *_subscriber, WUSubscribeOptions flags, const char *wuid)
  839. : CWorkUnitWatcher(_subscriber, (WUSubscribeOptions) (flags & SubscribeOptionAbort), wuid)
  840. {
  841. if (flags & SubscribeOptionState)
  842. {
  843. VStringBuffer xpath("/WorkUnits/%s/State", wuid);
  844. stateId = querySDS().subscribe(xpath.str(), *this);
  845. }
  846. if (flags & SubscribeOptionAction)
  847. {
  848. VStringBuffer xpath("/WorkUnits/%s/Action", wuid);
  849. actionId = querySDS().subscribe(xpath.str(), *this);
  850. }
  851. }
  852. };
  853. void CPersistedWorkUnit::subscribe(WUSubscribeOptions options)
  854. {
  855. CriticalBlock block(crit);
  856. assertex(options==SubscribeOptionAbort);
  857. if (!abortWatcher)
  858. {
  859. abortWatcher.setown(new CWorkUnitWatcher(this, SubscribeOptionAbort, p->queryName()));
  860. abortDirty = true;
  861. }
  862. }
  863. void CPersistedWorkUnit::unsubscribe()
  864. {
  865. CriticalBlock block(crit);
  866. if (abortWatcher)
  867. {
  868. abortWatcher->unsubscribe();
  869. abortWatcher.clear();
  870. }
  871. }
  872. bool CPersistedWorkUnit::aborting() const
  873. {
  874. CriticalBlock block(crit);
  875. if (abortDirty)
  876. {
  877. StringBuffer apath;
  878. apath.append("/WorkUnitAborts/").append(p->queryName());
  879. Owned<IRemoteConnection> acon = querySDS().connect(apath.str(), myProcessSession(), 0, SDS_LOCK_TIMEOUT);
  880. if (acon)
  881. abortState = acon->queryRoot()->getPropInt(NULL) != 0;
  882. else
  883. abortState = false;
  884. abortDirty = false;
  885. }
  886. return abortState;
  887. }
  888. class CDaliWorkUnit : public CPersistedWorkUnit
  889. {
  890. public:
  891. CDaliWorkUnit(IRemoteConnection *_conn, ISecManager *secmgr, ISecUser *secuser)
  892. : connection(_conn), CPersistedWorkUnit(secmgr, secuser)
  893. {
  894. loadPTree(connection->getRoot());
  895. }
  896. ~CDaliWorkUnit()
  897. {
  898. // NOTE - order is important - we need to construct connection before p and (especially) destroy after p
  899. // We use the beforeDispose() in base class to help ensure this
  900. p.clear();
  901. }
  902. IConstWUGraphProgress *getGraphProgress(const char *graphName) const
  903. {
  904. Owned<IRemoteConnection> conn = getProgressConnection();
  905. if (conn)
  906. {
  907. IPTree *progress = conn->queryRoot()->queryPropTree(graphName);
  908. if (progress)
  909. return new CConstGraphProgress(p->queryName(), graphName, progress);
  910. }
  911. return NULL;
  912. }
  913. virtual WUGraphState queryGraphState(const char *graphName) const
  914. {
  915. Owned<IRemoteConnection> conn = getProgressConnection();
  916. if (conn)
  917. {
  918. IPTree *progress = conn->queryRoot()->queryPropTree(graphName);
  919. if (progress)
  920. return (WUGraphState) progress->getPropInt("@_state", (unsigned) WUGraphUnknown);
  921. }
  922. return WUGraphUnknown;
  923. }
  924. virtual WUGraphState queryNodeState(const char *graphName, WUGraphIDType nodeId) const
  925. {
  926. Owned<IRemoteConnection> conn = getProgressConnection();
  927. if (conn)
  928. {
  929. IPTree *progress = conn->queryRoot()->queryPropTree(graphName);
  930. if (progress)
  931. {
  932. StringBuffer path;
  933. // NOTE - the node state info still uses the old graph layout, even when the stats are using the new...
  934. path.append("node[@id=\"").append(nodeId).append("\"]/@_state");
  935. return (WUGraphState) progress->getPropInt(path, (unsigned) WUGraphUnknown);
  936. }
  937. }
  938. return WUGraphUnknown;
  939. }
  940. virtual void clearGraphProgress() const
  941. {
  942. CriticalBlock block(crit);
  943. progressConnection.clear(); // Make sure nothing is locking for read or we won't be able to lock for write
  944. StringBuffer path("/GraphProgress/");
  945. path.append(p->queryName());
  946. Owned<IRemoteConnection> delconn = querySDS().connect(path.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
  947. if (delconn)
  948. delconn->close(true);
  949. }
  950. virtual bool getRunningGraph(IStringVal &graphName, WUGraphIDType &subId) const
  951. {
  952. Owned<IRemoteConnection> conn = getProgressConnection();
  953. if (!conn)
  954. return false;
  955. const char *name = conn->queryRoot()->queryProp("Running/@graph");
  956. if (name)
  957. {
  958. graphName.set(name);
  959. subId = conn->queryRoot()->getPropInt64("Running/@subId");
  960. return true;
  961. }
  962. else
  963. return false;
  964. }
  965. virtual void forceReload()
  966. {
  967. synchronized sync(locked); // protect locked workunits (uncommitted writes) from reload
  968. CriticalBlock block(crit);
  969. clearCached(true);
  970. connection->reload();
  971. progressConnection.clear();
  972. abortDirty = true;
  973. p.setown(connection->getRoot());
  974. }
  975. virtual void cleanupAndDelete(bool deldll, bool deleteOwned, const StringArray *deleteExclusions)
  976. {
  977. CPersistedWorkUnit::cleanupAndDelete(deldll, deleteOwned, deleteExclusions);
  978. clearGraphProgress();
  979. connection->close(true);
  980. connection.clear();
  981. }
  982. virtual void commit()
  983. {
  984. CPersistedWorkUnit::commit();
  985. if (connection)
  986. connection->commit();
  987. }
  988. virtual void _lockRemote()
  989. {
  990. StringBuffer wuRoot;
  991. getXPath(wuRoot, p->queryName());
  992. if (connection)
  993. connection->changeMode(RTM_LOCK_WRITE,SDS_LOCK_TIMEOUT);
  994. else
  995. connection.setown(querySDS().connect(wuRoot.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT));
  996. if (!connection)
  997. throw MakeStringException(WUERR_LockFailed, "Failed to get connection for xpath %s", wuRoot.str());
  998. clearCached(true);
  999. p.setown(connection->getRoot());
  1000. }
  1001. virtual void _unlockRemote()
  1002. {
  1003. if (connection)
  1004. {
  1005. try
  1006. {
  1007. try
  1008. {
  1009. connection->commit();
  1010. }
  1011. catch (IException *e)
  1012. {
  1013. EXCLOG(e, "Error during workunit commit");
  1014. connection->rollback();
  1015. connection->changeMode(0, SDS_LOCK_TIMEOUT);
  1016. throw;
  1017. }
  1018. connection->changeMode(0, SDS_LOCK_TIMEOUT);
  1019. }
  1020. catch (IException *E)
  1021. {
  1022. StringBuffer s;
  1023. PrintLog("Failed to release write lock on workunit: %s", E->errorMessage(s).str());
  1024. throw;
  1025. }
  1026. }
  1027. }
  1028. virtual void setGraphState(const char *graphName, WUGraphState state) const
  1029. {
  1030. Owned<IRemoteConnection> conn = getWritableProgressConnection(graphName);
  1031. conn->queryRoot()->setPropInt("@_state", state);
  1032. }
  1033. virtual void setNodeState(const char *graphName, WUGraphIDType nodeId, WUGraphState state) const
  1034. {
  1035. CriticalBlock block(crit);
  1036. progressConnection.clear(); // Make sure nothing is locking for read or we won't be able to lock for write
  1037. VStringBuffer path("/GraphProgress/%s", queryWuid());
  1038. Owned<IRemoteConnection> conn = querySDS().connect(path, myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
  1039. IPTree *progress = ensurePTree(conn->queryRoot(), graphName);
  1040. // NOTE - the node state info still uses the old graph layout, even when the stats are using the new...
  1041. path.clear().append("node[@id=\"").append(nodeId).append("\"]");
  1042. IPropertyTree *node = progress->queryPropTree(path.str());
  1043. if (!node)
  1044. {
  1045. node = progress->addPropTree("node");
  1046. node->setPropInt64("@id", nodeId);
  1047. }
  1048. node->setPropInt("@_state", (unsigned)state);
  1049. switch (state)
  1050. {
  1051. case WUGraphRunning:
  1052. {
  1053. IPropertyTree *running = conn->queryRoot()->setPropTree("Running");
  1054. running->setProp("@graph", graphName);
  1055. running->setPropInt64("@subId", nodeId);
  1056. break;
  1057. }
  1058. case WUGraphComplete:
  1059. {
  1060. conn->queryRoot()->removeProp("Running"); // only one thing running at any given time and one thing with lockWrite access
  1061. break;
  1062. }
  1063. }
  1064. }
  1065. virtual IWUGraphStats *updateStats(const char *graphName, StatisticCreatorType creatorType, const char * creator, unsigned subgraph) const
  1066. {
  1067. return new CDaliWuGraphStats(getWritableProgressConnection(graphName), creatorType, creator, graphName, subgraph);
  1068. }
  1069. protected:
  1070. IRemoteConnection *getProgressConnection() const
  1071. {
  1072. CriticalBlock block(crit);
  1073. if (!progressConnection)
  1074. {
  1075. VStringBuffer path("/GraphProgress/%s", queryWuid());
  1076. progressConnection.setown(querySDS().connect(path, myProcessSession(), 0, SDS_LOCK_TIMEOUT)); // Note - we don't lock. The writes are atomic.
  1077. }
  1078. return progressConnection.getLink();
  1079. }
  1080. IRemoteConnection *getWritableProgressConnection(const char *graphName) const
  1081. {
  1082. CriticalBlock block(crit);
  1083. progressConnection.clear(); // Make sure subsequent reads from this workunit get the changes I am making
  1084. VStringBuffer path("/GraphProgress/%s/%s", queryWuid(), graphName);
  1085. return querySDS().connect(path, myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
  1086. }
  1087. IPropertyTree *getGraphProgressTree() const
  1088. {
  1089. Owned<IRemoteConnection> conn = getProgressConnection();
  1090. if (conn)
  1091. {
  1092. Owned<IPropertyTree> tmp = createPTree("GraphProgress");
  1093. mergePTree(tmp,conn->queryRoot());
  1094. return tmp.getClear();
  1095. }
  1096. return NULL;
  1097. }
  1098. Owned<IRemoteConnection> connection;
  1099. mutable Owned<IRemoteConnection> progressConnection;
  1100. };
  1101. class CLockedWorkUnit : implements ILocalWorkUnit, implements IExtendedWUInterface, public CInterface
  1102. {
  1103. public:
  1104. Owned<CLocalWorkUnit> c;
  1105. IMPLEMENT_IINTERFACE;
  1106. CLockedWorkUnit(CLocalWorkUnit *_c) : c(_c) {}
  1107. ~CLockedWorkUnit()
  1108. {
  1109. if (workUnitTraceLevel > 1)
  1110. PrintLog("Releasing locked workunit %s", queryWuid());
  1111. if (c)
  1112. c->unlockRemote();
  1113. }
  1114. virtual IConstWorkUnit * unlock()
  1115. {
  1116. c->unlockRemote();
  1117. return c.getClear();
  1118. }
  1119. virtual bool aborting() const
  1120. { return c->aborting(); }
  1121. virtual void forceReload()
  1122. { UNIMPLEMENTED; }
  1123. virtual WUAction getAction() const
  1124. { return c->getAction(); }
  1125. virtual const char *queryActionDesc() const
  1126. { return c->queryActionDesc(); }
  1127. virtual IStringVal & getApplicationValue(const char * application, const char * propname, IStringVal & str) const
  1128. { return c->getApplicationValue(application, propname, str); }
  1129. virtual int getApplicationValueInt(const char * application, const char * propname, int defVal) const
  1130. { return c->getApplicationValueInt(application, propname, defVal); }
  1131. virtual IConstWUAppValueIterator & getApplicationValues() const
  1132. { return c->getApplicationValues(); }
  1133. virtual bool hasWorkflow() const
  1134. { return c->hasWorkflow(); }
  1135. virtual unsigned queryEventScheduledCount() const
  1136. { return c->queryEventScheduledCount(); }
  1137. virtual IPropertyTree * queryWorkflowTree() const
  1138. { return c->queryWorkflowTree(); }
  1139. virtual IConstWorkflowItemIterator * getWorkflowItems() const
  1140. { return c->getWorkflowItems(); }
  1141. virtual IWorkflowItemArray * getWorkflowClone() const
  1142. { return c->getWorkflowClone(); }
  1143. virtual bool requiresLocalFileUpload() const
  1144. { return c->requiresLocalFileUpload(); }
  1145. virtual IConstLocalFileUploadIterator * getLocalFileUploads() const
  1146. { return c->getLocalFileUploads(); }
  1147. virtual bool getIsQueryService() const
  1148. { return c->getIsQueryService(); }
  1149. virtual bool getCloneable() const
  1150. { return c->getCloneable(); }
  1151. virtual IUserDescriptor * queryUserDescriptor() const
  1152. { return c->queryUserDescriptor(); }
  1153. virtual const char *queryClusterName() const
  1154. { return c->queryClusterName(); }
  1155. virtual unsigned getCodeVersion() const
  1156. { return c->getCodeVersion(); }
  1157. virtual unsigned getWuidVersion() const
  1158. { return c->getWuidVersion(); }
  1159. virtual void getBuildVersion(IStringVal & buildVersion, IStringVal & eclVersion) const
  1160. { c->getBuildVersion(buildVersion, eclVersion); }
  1161. virtual bool hasDebugValue(const char * propname) const
  1162. { return c->hasDebugValue(propname); }
  1163. virtual IStringVal & getDebugValue(const char * propname, IStringVal & str) const
  1164. { return c->getDebugValue(propname, str); }
  1165. virtual int getDebugValueInt(const char * propname, int defVal) const
  1166. { return c->getDebugValueInt(propname, defVal); }
  1167. virtual __int64 getDebugValueInt64(const char * propname, __int64 defVal) const
  1168. { return c->getDebugValueInt64(propname, defVal); }
  1169. virtual bool getDebugValueBool(const char * propname, bool defVal) const
  1170. { return c->getDebugValueBool(propname, defVal); }
  1171. virtual IStringIterator & getDebugValues() const
  1172. { return c->getDebugValues(NULL); }
  1173. virtual IStringIterator & getDebugValues(const char *prop) const
  1174. { return c->getDebugValues(prop); }
  1175. virtual unsigned getExceptionCount() const
  1176. { return c->getExceptionCount(); }
  1177. virtual IConstWUExceptionIterator & getExceptions() const
  1178. { return c->getExceptions(); }
  1179. virtual unsigned getGraphCount() const
  1180. { return c->getGraphCount(); }
  1181. virtual unsigned getSourceFileCount() const
  1182. { return c->getSourceFileCount(); }
  1183. virtual unsigned getResultCount() const
  1184. { return c->getResultCount(); }
  1185. virtual unsigned getVariableCount() const
  1186. { return c->getVariableCount(); }
  1187. virtual unsigned getApplicationValueCount() const
  1188. { return c->getApplicationValueCount(); }
  1189. virtual IConstWUGraphIterator & getGraphs(WUGraphType type) const
  1190. { return c->getGraphs(type); }
  1191. virtual IConstWUGraphMetaIterator & getGraphsMeta(WUGraphType type) const
  1192. { return c->getGraphsMeta(type); }
  1193. virtual IConstWUGraph * getGraph(const char *name) const
  1194. { return c->getGraph(name); }
  1195. virtual IConstWUGraphProgress * getGraphProgress(const char * name) const
  1196. { return c->getGraphProgress(name); }
  1197. virtual const char *queryJobName() const
  1198. { return c->queryJobName(); }
  1199. virtual IConstWUPlugin * getPluginByName(const char * name) const
  1200. { return c->getPluginByName(name); }
  1201. virtual IConstWUPluginIterator & getPlugins() const
  1202. { return c->getPlugins(); }
  1203. virtual IConstWULibrary* getLibraryByName(const char *name) const
  1204. { return c->getLibraryByName(name); }
  1205. virtual IConstWULibraryIterator & getLibraries() const
  1206. { return c->getLibraries(); }
  1207. virtual WUPriorityClass getPriority() const
  1208. { return c->getPriority(); }
  1209. virtual const char *queryPriorityDesc() const
  1210. { return c->queryPriorityDesc(); }
  1211. virtual int getPriorityLevel() const
  1212. { return c->getPriorityLevel(); }
  1213. virtual int getPriorityValue() const
  1214. { return c->getPriorityValue(); }
  1215. virtual IConstWUQuery * getQuery() const
  1216. { return c->getQuery(); }
  1217. virtual IConstWUWebServicesInfo * getWebServicesInfo() const
  1218. { return c->getWebServicesInfo(); }
  1219. virtual bool getRescheduleFlag() const
  1220. { return c->getRescheduleFlag(); }
  1221. virtual IConstWUResult * getResultByName(const char * name) const
  1222. { return c->getResultByName(name); }
  1223. virtual IConstWUResult * getResultBySequence(unsigned seq) const
  1224. { return c->getResultBySequence(seq); }
  1225. virtual unsigned getResultLimit() const
  1226. { return c->getResultLimit(); }
  1227. virtual IConstWUResultIterator & getResults() const
  1228. { return c->getResults(); }
  1229. virtual IStringVal & getScope(IStringVal & str) const
  1230. { return c->getScope(str); }
  1231. virtual IStringVal & getSecurityToken(IStringVal & str) const
  1232. { return c->getSecurityToken(str); }
  1233. virtual WUState getState() const
  1234. { return c->getState(); }
  1235. virtual IStringVal & getStateEx(IStringVal & str) const
  1236. { return c->getStateEx(str); }
  1237. virtual __int64 getAgentSession() const
  1238. { return c->getAgentSession(); }
  1239. virtual unsigned getAgentPID() const
  1240. { return c->getAgentPID(); }
  1241. virtual const char *queryStateDesc() const
  1242. { return c->queryStateDesc(); }
  1243. virtual bool getRunningGraph(IStringVal & graphName, WUGraphIDType & subId) const
  1244. { return c->getRunningGraph(graphName, subId); }
  1245. virtual IConstWUStatisticIterator & getStatistics(const IStatisticsFilter * filter) const
  1246. { return c->getStatistics(filter); }
  1247. virtual IConstWUStatistic * getStatistic(const char * creator, const char * scope, StatisticKind kind) const
  1248. { return c->getStatistic(creator, scope, kind); }
  1249. virtual IStringVal & getSnapshot(IStringVal & str) const
  1250. { return c->getSnapshot(str); }
  1251. virtual const char *queryUser() const
  1252. { return c->queryUser(); }
  1253. virtual ErrorSeverity getWarningSeverity(unsigned code, ErrorSeverity defaultSeverity) const
  1254. { return c->getWarningSeverity(code, defaultSeverity); }
  1255. virtual const char *queryWuScope() const
  1256. { return c->queryWuScope(); }
  1257. virtual const char *queryWuid() const
  1258. { return c->queryWuid(); }
  1259. virtual IConstWUResult * getGlobalByName(const char * name) const
  1260. { return c->getGlobalByName(name); }
  1261. virtual IConstWUResult * getTemporaryByName(const char * name) const
  1262. { return c->getTemporaryByName(name); }
  1263. virtual IConstWUResultIterator & getTemporaries() const
  1264. { return c->getTemporaries(); }
  1265. virtual IConstWUResult * getVariableByName(const char * name) const
  1266. { return c->getVariableByName(name); }
  1267. virtual IConstWUResultIterator & getVariables() const
  1268. { return c->getVariables(); }
  1269. virtual bool isProtected() const
  1270. { return c->isProtected(); }
  1271. virtual bool isPausing() const
  1272. { return c->isPausing(); }
  1273. virtual IWorkUnit & lock()
  1274. { ((CInterface *)this)->Link(); return (IWorkUnit &) *this; }
  1275. virtual bool reload()
  1276. { UNIMPLEMENTED; }
  1277. virtual void subscribe(WUSubscribeOptions options)
  1278. { c->subscribe(options); }
  1279. virtual void requestAbort()
  1280. { c->requestAbort(); }
  1281. virtual unsigned calculateHash(unsigned prevHash)
  1282. { return queryExtendedWU(c)->calculateHash(prevHash); }
  1283. virtual void copyWorkUnit(IConstWorkUnit *cached, bool all)
  1284. { queryExtendedWU(c)->copyWorkUnit(cached, all); }
  1285. virtual IPropertyTree *queryPTree() const
  1286. { return queryExtendedWU(c)->queryPTree(); }
  1287. virtual IPropertyTree *getUnpackedTree(bool includeProgress) const
  1288. { return queryExtendedWU(c)->getUnpackedTree(includeProgress); }
  1289. virtual bool archiveWorkUnit(const char *base,bool del,bool deldll,bool deleteOwned,bool exportAssociatedFiles)
  1290. { return queryExtendedWU(c)->archiveWorkUnit(base,del,deldll,deleteOwned,exportAssociatedFiles); }
  1291. virtual unsigned queryFileUsage(const char *filename) const
  1292. { return c->queryFileUsage(filename); }
  1293. virtual IConstWUFileUsageIterator * getFieldUsage() const
  1294. { return c->getFieldUsage(); }
  1295. virtual bool getFieldUsageArray(StringArray & filenames, StringArray & columnnames, const char * clusterName) const
  1296. { return c->getFieldUsageArray(filenames, columnnames, clusterName); }
  1297. virtual IJlibDateTime & getTimeScheduled(IJlibDateTime &val) const
  1298. { return c->getTimeScheduled(val); }
  1299. virtual unsigned getDebugAgentListenerPort() const
  1300. { return c->getDebugAgentListenerPort(); }
  1301. virtual IStringVal & getDebugAgentListenerIP(IStringVal &ip) const
  1302. { return c->getDebugAgentListenerIP(ip); }
  1303. virtual IStringVal & getXmlParams(IStringVal & params, bool hidePasswords) const
  1304. { return c->getXmlParams(params, hidePasswords); }
  1305. virtual const IPropertyTree *getXmlParams() const
  1306. { return c->getXmlParams(); }
  1307. virtual unsigned __int64 getHash() const
  1308. { return c->getHash(); }
  1309. virtual IStringIterator *getLogs(const char *type, const char *instance) const
  1310. { return c->getLogs(type, instance); }
  1311. virtual IStringIterator *getProcesses(const char *type) const
  1312. { return c->getProcesses(type); }
  1313. virtual IPropertyTreeIterator* getProcesses(const char *type, const char *instance) const
  1314. { return c->getProcesses(type, instance); }
  1315. virtual unsigned getTotalThorTime() const
  1316. { return c->getTotalThorTime(); }
  1317. virtual WUGraphState queryGraphState(const char *graphName) const
  1318. { return c->queryGraphState(graphName); }
  1319. virtual WUGraphState queryNodeState(const char *graphName, WUGraphIDType nodeId) const
  1320. { return c->queryNodeState(graphName, nodeId); }
  1321. virtual void setGraphState(const char *graphName, WUGraphState state) const
  1322. { c->setGraphState(graphName, state); }
  1323. virtual void setNodeState(const char *graphName, WUGraphIDType nodeId, WUGraphState state) const
  1324. { c->setNodeState(graphName, nodeId, state); }
  1325. virtual IWUGraphStats *updateStats(const char *graphName, StatisticCreatorType creatorType, const char * creator, unsigned subgraph) const
  1326. { return c->updateStats(graphName, creatorType, creator, subgraph); }
  1327. virtual void clearGraphProgress() const
  1328. { c->clearGraphProgress(); }
  1329. virtual IStringVal & getAbortBy(IStringVal & str) const
  1330. { return c->getAbortBy(str); }
  1331. virtual unsigned __int64 getAbortTimeStamp() const
  1332. { return c->getAbortTimeStamp(); }
  1333. virtual void clearExceptions()
  1334. { c->clearExceptions(); }
  1335. virtual void commit()
  1336. { c->commit(); }
  1337. virtual IWUException * createException()
  1338. { return c->createException(); }
  1339. virtual void addProcess(const char *type, const char *instance, unsigned pid, const char *log)
  1340. { c->addProcess(type, instance, pid, log); }
  1341. virtual void protect(bool protectMode)
  1342. { c->protect(protectMode); }
  1343. virtual void setAction(WUAction action)
  1344. { c->setAction(action); }
  1345. virtual void setApplicationValue(const char * application, const char * propname, const char * value, bool overwrite)
  1346. { c->setApplicationValue(application, propname, value, overwrite); }
  1347. virtual void setApplicationValueInt(const char * application, const char * propname, int value, bool overwrite)
  1348. { c->setApplicationValueInt(application, propname, value, overwrite); }
  1349. virtual void incEventScheduledCount()
  1350. { c->incEventScheduledCount(); }
  1351. virtual void setIsQueryService(bool value)
  1352. { c->setIsQueryService(value); }
  1353. virtual void setCloneable(bool value)
  1354. { c->setCloneable(value); }
  1355. virtual void setIsClone(bool value)
  1356. { c->setIsClone(value); }
  1357. virtual void setClusterName(const char * value)
  1358. { c->setClusterName(value); }
  1359. virtual void setCodeVersion(unsigned version, const char * buildVersion, const char * eclVersion)
  1360. { c->setCodeVersion(version, buildVersion, eclVersion); }
  1361. virtual void setDebugValue(const char * propname, const char * value, bool overwrite)
  1362. { c->setDebugValue(propname, value, overwrite); }
  1363. virtual void setDebugValueInt(const char * propname, int value, bool overwrite)
  1364. { c->setDebugValueInt(propname, value, overwrite); }
  1365. virtual void setJobName(const char * value)
  1366. { c->setJobName(value); }
  1367. virtual void setPriority(WUPriorityClass cls)
  1368. { c->setPriority(cls); }
  1369. virtual void setPriorityLevel(int level)
  1370. { c->setPriorityLevel(level); }
  1371. virtual void setRescheduleFlag(bool value)
  1372. { c->setRescheduleFlag(value); }
  1373. virtual void setResultLimit(unsigned value)
  1374. { c->setResultLimit(value); }
  1375. virtual void setSecurityToken(const char *value)
  1376. { c->setSecurityToken(value); }
  1377. virtual void setState(WUState state)
  1378. { c->setState(state); }
  1379. virtual void setStateEx(const char * text)
  1380. { c->setStateEx(text); }
  1381. virtual void setAgentSession(__int64 sessionId)
  1382. { c->setAgentSession(sessionId); }
  1383. virtual void setStatistic(StatisticCreatorType creatorType, const char * creator, StatisticScopeType scopeType, const char * scope, StatisticKind kind, const char * optDescription, unsigned __int64 value, unsigned __int64 count, unsigned __int64 maxValue, StatsMergeAction mergeAction)
  1384. { c->setStatistic(creatorType, creator, scopeType, scope, kind, optDescription, value, count, maxValue, mergeAction); }
  1385. virtual void setTracingValue(const char * propname, const char * value)
  1386. { c->setTracingValue(propname, value); }
  1387. virtual void setTracingValueInt(const char * propname, int value)
  1388. { c->setTracingValueInt(propname, value); }
  1389. virtual void setTracingValueInt64(const char * propname, __int64 value)
  1390. { c->setTracingValueInt64(propname, value); }
  1391. virtual void setUser(const char * value)
  1392. { c->setUser(value); }
  1393. virtual void setWuScope(const char * value)
  1394. { c->setWuScope(value); }
  1395. virtual IWorkflowItem* addWorkflowItem(unsigned wfid, WFType type, WFMode mode, unsigned success, unsigned failure, unsigned recovery, unsigned retriesAllowed, unsigned contingencyFor)
  1396. { return c->addWorkflowItem(wfid, type, mode, success, failure, recovery, retriesAllowed, contingencyFor); }
  1397. virtual void syncRuntimeWorkflow(IWorkflowItemArray * array)
  1398. { c->syncRuntimeWorkflow(array); }
  1399. virtual IWorkflowItemIterator * updateWorkflowItems()
  1400. { return c->updateWorkflowItems(); }
  1401. virtual void resetWorkflow()
  1402. { c->resetWorkflow(); }
  1403. virtual void schedule()
  1404. { c->schedule(); }
  1405. virtual void deschedule()
  1406. { c->deschedule(); }
  1407. virtual unsigned addLocalFileUpload(LocalFileUploadType type, char const * source, char const * destination, char const * eventTag)
  1408. { return c->addLocalFileUpload(type, source, destination, eventTag); }
  1409. virtual IWUResult * updateGlobalByName(const char * name)
  1410. { return c->updateGlobalByName(name); }
  1411. virtual void createGraph(const char * name, const char *label, WUGraphType type, IPropertyTree *xgmml)
  1412. { c->createGraph(name, label, type, xgmml); }
  1413. virtual IWUQuery * updateQuery()
  1414. { return c->updateQuery(); }
  1415. virtual IWUWebServicesInfo * updateWebServicesInfo(bool create)
  1416. { return c->updateWebServicesInfo(create); }
  1417. virtual IWUPlugin * updatePluginByName(const char * name)
  1418. { return c->updatePluginByName(name); }
  1419. virtual IWULibrary * updateLibraryByName(const char * name)
  1420. { return c->updateLibraryByName(name); }
  1421. virtual IWUResult * updateResultByName(const char * name)
  1422. { return c->updateResultByName(name); }
  1423. virtual IWUResult * updateResultBySequence(unsigned seq)
  1424. { return c->updateResultBySequence(seq); }
  1425. virtual IWUResult * updateTemporaryByName(const char * name)
  1426. { return c->updateTemporaryByName(name); }
  1427. virtual IWUResult * updateVariableByName(const char * name)
  1428. { return c->updateVariableByName(name); }
  1429. virtual void addFile(const char *fileName, StringArray *clusters, unsigned usageCount, WUFileKind fileKind, const char *graphOwner)
  1430. { c->addFile(fileName, clusters, usageCount, fileKind, graphOwner); }
  1431. virtual void noteFileRead(IDistributedFile *file)
  1432. { c->noteFileRead(file); }
  1433. virtual void noteFieldUsage(IPropertyTree * usage)
  1434. { c->noteFieldUsage(usage); }
  1435. virtual void releaseFile(const char *fileName)
  1436. { c->releaseFile(fileName); }
  1437. virtual void resetBeforeGeneration()
  1438. { c->resetBeforeGeneration(); }
  1439. virtual void deleteTempFiles(const char *graph, bool deleteOwned, bool deleteJobOwned)
  1440. { c->deleteTempFiles(graph, deleteOwned, deleteJobOwned); }
  1441. virtual void deleteTemporaries()
  1442. { c->deleteTemporaries(); }
  1443. virtual void addDiskUsageStats(__int64 avgNodeUsage, unsigned minNode, __int64 minNodeUsage, unsigned maxNode, __int64 maxNodeUsage, __int64 graphId)
  1444. { c->addDiskUsageStats(avgNodeUsage, minNode, minNodeUsage, maxNode, maxNodeUsage, graphId); }
  1445. virtual IPropertyTree * getDiskUsageStats()
  1446. { return c->getDiskUsageStats(); }
  1447. virtual IPropertyTreeIterator & getFileIterator() const
  1448. { return c->getFileIterator(); }
  1449. virtual IPropertyTreeIterator & getFilesReadIterator() const
  1450. { return c->getFilesReadIterator(); }
  1451. virtual void setSnapshot(const char * value)
  1452. { c->setSnapshot(value); }
  1453. virtual void setWarningSeverity(unsigned code, ErrorSeverity severity)
  1454. { c->setWarningSeverity(code, severity); }
  1455. virtual void setTimeScheduled(const IJlibDateTime &val)
  1456. { c->setTimeScheduled(val); }
  1457. virtual void setDebugAgentListenerPort(unsigned port)
  1458. { c->setDebugAgentListenerPort(port); }
  1459. virtual void setDebugAgentListenerIP(const char * ip)
  1460. { c->setDebugAgentListenerIP(ip); }
  1461. virtual void setXmlParams(const char *params)
  1462. { c->setXmlParams(params); }
  1463. virtual void setXmlParams(IPropertyTree *tree)
  1464. { c->setXmlParams(tree); }
  1465. virtual void setHash(unsigned __int64 hash)
  1466. { c->setHash(hash); }
  1467. // ILocalWorkUnit - used for debugging etc
  1468. virtual void serialize(MemoryBuffer &tgt)
  1469. { c->serialize(tgt); }
  1470. virtual void deserialize(MemoryBuffer &src)
  1471. { c->deserialize(src); }
  1472. virtual bool switchThorQueue(const char *cluster, IQueueSwitcher *qs)
  1473. { return c->switchThorQueue(cluster,qs); }
  1474. virtual void setAllowedClusters(const char *value)
  1475. { c->setAllowedClusters(value); }
  1476. virtual IStringVal& getAllowedClusters(IStringVal &str) const
  1477. { return c->getAllowedClusters(str); }
  1478. virtual void remoteCheckAccess(IUserDescriptor *user, bool writeaccess) const
  1479. { c->remoteCheckAccess(user,writeaccess); }
  1480. virtual void setAllowAutoQueueSwitch(bool val)
  1481. { c->setAllowAutoQueueSwitch(val); }
  1482. virtual bool getAllowAutoQueueSwitch() const
  1483. { return c->getAllowAutoQueueSwitch(); }
  1484. virtual void setLibraryInformation(const char * name, unsigned interfaceHash, unsigned definitionHash)
  1485. { c->setLibraryInformation(name, interfaceHash, definitionHash); }
  1486. virtual void setResultInt(const char * name, unsigned sequence, __int64 val)
  1487. { c->setResultInt(name, sequence, val); }
  1488. virtual void setResultUInt(const char * name, unsigned sequence, unsigned __int64 val)
  1489. { c->setResultUInt(name, sequence, val); }
  1490. virtual void setResultReal(const char *name, unsigned sequence, double val)
  1491. { c->setResultReal(name, sequence, val); }
  1492. virtual void setResultVarString(const char * stepname, unsigned sequence, const char *val)
  1493. { c->setResultVarString(stepname, sequence, val); }
  1494. virtual void setResultVarUnicode(const char * stepname, unsigned sequence, UChar const *val)
  1495. { c->setResultVarUnicode(stepname, sequence, val); }
  1496. virtual void setResultString(const char * stepname, unsigned sequence, int len, const char *val)
  1497. { c->setResultString(stepname, sequence, len, val); }
  1498. virtual void setResultData(const char * stepname, unsigned sequence, int len, const void *val)
  1499. { c->setResultData(stepname, sequence, len, val); }
  1500. virtual void setResultRaw(const char * name, unsigned sequence, int len, const void *val)
  1501. { c->setResultRaw(name, sequence, len, val); }
  1502. virtual void setResultSet(const char * name, unsigned sequence, bool isAll, size32_t len, const void *val, ISetToXmlTransformer *xform)
  1503. { c->setResultSet(name, sequence, isAll, len, val, xform); }
  1504. virtual void setResultUnicode(const char * name, unsigned sequence, int len, UChar const * val)
  1505. { c->setResultUnicode(name, sequence, len, val); }
  1506. virtual void setResultBool(const char *name, unsigned sequence, bool val)
  1507. { c->setResultBool(name, sequence, val); }
  1508. virtual void setResultDecimal(const char *name, unsigned sequence, int len, int precision, bool isSigned, const void *val)
  1509. { c->setResultDecimal(name, sequence, len, precision, isSigned, val); }
  1510. virtual void setResultDataset(const char * name, unsigned sequence, size32_t len, const void *val, unsigned numRows, bool extend)
  1511. { c->setResultDataset(name, sequence, len, val, numRows, extend); }
  1512. };
  1513. class CLocalWUAssociated : implements IConstWUAssociatedFile, public CInterface
  1514. {
  1515. Owned<IPropertyTree> p;
  1516. public:
  1517. IMPLEMENT_IINTERFACE;
  1518. CLocalWUAssociated(IPropertyTree *p);
  1519. virtual WUFileType getType() const;
  1520. virtual IStringVal & getDescription(IStringVal & ret) const;
  1521. virtual IStringVal & getIp(IStringVal & ret) const;
  1522. virtual IStringVal & getName(IStringVal & ret) const;
  1523. virtual IStringVal & getNameTail(IStringVal & ret) const;
  1524. virtual unsigned getCrc() const;
  1525. virtual unsigned getMinActivityId() const;
  1526. virtual unsigned getMaxActivityId() const;
  1527. };
  1528. class CLocalWUQuery : implements IWUQuery, public CInterface
  1529. {
  1530. Owned<IPropertyTree> p;
  1531. mutable IArrayOf<IConstWUAssociatedFile> associated;
  1532. mutable CriticalSection crit;
  1533. mutable bool associatedCached;
  1534. private:
  1535. void addSpecialCaseAssociated(WUFileType type, const char * propname, unsigned crc) const;
  1536. void loadAssociated() const;
  1537. public:
  1538. IMPLEMENT_IINTERFACE;
  1539. CLocalWUQuery(IPropertyTree *p);
  1540. virtual WUQueryType getQueryType() const;
  1541. virtual IStringVal& getQueryText(IStringVal &str) const;
  1542. virtual IStringVal& getQueryShortText(IStringVal &str) const;
  1543. virtual IStringVal& getQueryName(IStringVal &str) const;
  1544. virtual IStringVal & getQueryMainDefinition(IStringVal & str) const;
  1545. virtual IStringVal& getQueryDllName(IStringVal &str) const;
  1546. virtual unsigned getQueryDllCrc() const;
  1547. virtual IStringVal& getQueryCppName(IStringVal &str) const;
  1548. virtual IStringVal& getQueryResTxtName(IStringVal &str) const;
  1549. virtual IConstWUAssociatedFile * getAssociatedFile(WUFileType type, unsigned index) const;
  1550. virtual IConstWUAssociatedFileIterator& getAssociatedFiles() const;
  1551. virtual bool isArchive() const;
  1552. virtual bool hasArchive() const
  1553. {
  1554. return p->getPropBool("@hasArchive");
  1555. }
  1556. virtual void setQueryType(WUQueryType qt);
  1557. virtual void setQueryText(const char *pstr);
  1558. virtual void setQueryName(const char *);
  1559. virtual void setQueryMainDefinition(const char * str);
  1560. virtual void addAssociatedFile(WUFileType type, const char * name, const char * ip, const char * desc, unsigned crc, unsigned minActivity, unsigned maxActivity);
  1561. virtual void removeAssociatedFiles();
  1562. virtual void removeAssociatedFile(WUFileType type, const char * name, const char * desc);
  1563. };
  1564. class CLocalWUWebServicesInfo : implements IWUWebServicesInfo, public CInterface
  1565. {
  1566. Owned<IPropertyTree> p;
  1567. mutable CriticalSection crit;
  1568. private:
  1569. public:
  1570. IMPLEMENT_IINTERFACE;
  1571. CLocalWUWebServicesInfo(IPropertyTree *p);
  1572. virtual IStringVal& getModuleName(IStringVal &str) const;
  1573. virtual IStringVal& getAttributeName(IStringVal &str) const;
  1574. virtual IStringVal& getDefaultName(IStringVal &str) const;
  1575. virtual IStringVal& getInfo(const char *name, IStringVal &str) const;
  1576. virtual IStringVal& getText(const char *name, IStringVal &str) const;
  1577. virtual unsigned getWebServicesCRC() const;
  1578. virtual void setModuleName(const char *);
  1579. virtual void setAttributeName(const char *);
  1580. virtual void setDefaultName(const char *);
  1581. virtual void setInfo(const char *name, const char *info);
  1582. virtual void setText(const char *name, const char *info);
  1583. virtual void setWebServicesCRC(unsigned);
  1584. };
  1585. class CLocalWUResult : implements IWUResult, public CInterface
  1586. {
  1587. friend class CLocalWorkUnit;
  1588. mutable CriticalSection crit;
  1589. Owned<IPropertyTree> p;
  1590. Owned<IProperties> xmlns;
  1591. public:
  1592. IMPLEMENT_IINTERFACE;
  1593. CLocalWUResult(IPropertyTree *props);
  1594. ~CLocalWUResult() { try { p.clear(); } catch (IException *E) {E->Release();}}
  1595. virtual WUResultStatus getResultStatus() const;
  1596. virtual IStringVal& getResultName(IStringVal &str) const;
  1597. virtual int getResultSequence() const;
  1598. virtual bool isResultScalar() const;
  1599. virtual IStringVal& getResultXml(IStringVal &str, bool hidePasswords) const;
  1600. virtual unsigned getResultFetchSize() const;
  1601. virtual __int64 getResultTotalRowCount() const;
  1602. virtual __int64 getResultRowCount() const;
  1603. virtual void getResultDataset(IStringVal & ecl, IStringVal & defs) const;
  1604. virtual IStringVal& getResultLogicalName(IStringVal &ecl) const;
  1605. virtual IStringVal& getResultKeyField(IStringVal& ecl) const;
  1606. virtual unsigned getResultRequestedRows() const;
  1607. virtual __int64 getResultInt() const;
  1608. virtual bool getResultBool() const;
  1609. virtual double getResultReal() const;
  1610. virtual IStringVal& getResultString(IStringVal & str, bool hidePassword) const;
  1611. virtual IDataVal& getResultRaw(IDataVal & data, IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const;
  1612. virtual IDataVal& getResultUnicode(IDataVal & data) const;
  1613. virtual void getResultDecimal(void * val, unsigned length, unsigned precision, bool isSigned) const;
  1614. virtual IStringVal& getResultEclSchema(IStringVal & str) const;
  1615. virtual __int64 getResultRawSize(IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const;
  1616. virtual IDataVal& getResultRaw(IDataVal & data, __int64 from, __int64 length, IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const;
  1617. virtual IStringVal& getResultRecordSizeEntry(IStringVal & str) const;
  1618. virtual IStringVal& getResultTransformerEntry(IStringVal & str) const;
  1619. virtual __int64 getResultRowLimit() const;
  1620. virtual IStringVal& getResultFilename(IStringVal & str) const;
  1621. virtual WUResultFormat getResultFormat() const;
  1622. virtual unsigned getResultHash() const;
  1623. virtual bool getResultIsAll() const;
  1624. virtual IProperties *queryResultXmlns();
  1625. virtual IStringVal& getResultFieldOpt(const char *name, IStringVal &str) const;
  1626. virtual void getSchema(IArrayOf<ITypeInfo> &types, StringAttrArray &names, IStringVal * ecl=NULL) const;
  1627. virtual void getResultWriteLocation(IStringVal & _graph, unsigned & _activityId) const;
  1628. // interface IWUResult
  1629. virtual void setResultStatus(WUResultStatus status);
  1630. virtual void setResultName(const char *name);
  1631. virtual void setResultSequence(unsigned seq);
  1632. virtual void setResultSchemaRaw(unsigned len, const void *schema);
  1633. virtual void setResultScalar(bool isScalar);
  1634. virtual void setResultRaw(unsigned len, const void *xml, WUResultFormat format);
  1635. virtual void setResultFetchSize(unsigned rows); // 0 means file-loaded
  1636. virtual void setResultTotalRowCount(__int64 rows); // -1 means unknown
  1637. virtual void setResultRowCount(__int64 rows);
  1638. virtual void setResultDataset(const char *ecl, const char *defs);
  1639. virtual void setResultLogicalName(const char *logicalName);
  1640. virtual void setResultKeyField(const char * name);
  1641. virtual void setResultRequestedRows(unsigned req);
  1642. virtual void setResultRecordSizeEntry(const char * val);
  1643. virtual void setResultTransformerEntry(const char * val);
  1644. virtual void setResultInt(__int64 val);
  1645. virtual void setResultReal(double val);
  1646. virtual void setResultBool(bool val);
  1647. virtual void setResultString(const char * val, unsigned length);
  1648. virtual void setResultUnicode(const void * val, unsigned length);
  1649. virtual void setResultData(const void * val, unsigned length);
  1650. virtual void setResultDecimal(const void * val, unsigned length);
  1651. virtual void addResultRaw(unsigned len, const void * data, WUResultFormat format);
  1652. virtual void setResultRowLimit(__int64 value);
  1653. virtual void setResultFilename(const char * name);
  1654. virtual void setResultUInt(unsigned __int64 val);
  1655. virtual void setResultIsAll(bool value);
  1656. virtual void setResultFormat(WUResultFormat format);
  1657. virtual void setResultXML(const char *val);
  1658. virtual void setResultRow(unsigned len, const void * data);
  1659. virtual void setResultXmlns(const char *prefix, const char *uri);
  1660. virtual void setResultFieldOpt(const char *name, const char *value);
  1661. virtual void setResultWriteLocation(const char * _graph, unsigned _activityId);
  1662. virtual IPropertyTree *queryPTree() { return p; }
  1663. };
  1664. class CLocalWUPlugin : implements IWUPlugin, public CInterface
  1665. {
  1666. Owned<IPropertyTree> p;
  1667. public:
  1668. IMPLEMENT_IINTERFACE;
  1669. CLocalWUPlugin(IPropertyTree *p);
  1670. virtual IStringVal& getPluginName(IStringVal &str) const;
  1671. virtual IStringVal& getPluginVersion(IStringVal &str) const;
  1672. virtual void setPluginName(const char *str);
  1673. virtual void setPluginVersion(const char *str);
  1674. };
  1675. class CLocalWULibrary : implements IWULibrary, public CInterface
  1676. {
  1677. Owned<IPropertyTree> p;
  1678. public:
  1679. IMPLEMENT_IINTERFACE;
  1680. CLocalWULibrary(IPropertyTree *p);
  1681. virtual IStringVal & getName(IStringVal & str) const;
  1682. virtual void setName(const char * str);
  1683. };
  1684. class CLocalWUException : implements IWUException, public CInterface
  1685. {
  1686. Owned<IPropertyTree> p;
  1687. public:
  1688. IMPLEMENT_IINTERFACE;
  1689. CLocalWUException(IPropertyTree *p);
  1690. virtual IStringVal& getExceptionSource(IStringVal &str) const;
  1691. virtual IStringVal& getExceptionMessage(IStringVal &str) const;
  1692. virtual unsigned getExceptionCode() const;
  1693. virtual ErrorSeverity getSeverity() const;
  1694. virtual IStringVal & getTimeStamp(IStringVal & dt) const;
  1695. virtual IStringVal & getExceptionFileName(IStringVal & str) const;
  1696. virtual unsigned getExceptionLineNo() const;
  1697. virtual unsigned getExceptionColumn() const;
  1698. virtual unsigned getActivityId() const;
  1699. virtual unsigned getSequence() const;
  1700. virtual void setExceptionSource(const char *str);
  1701. virtual void setExceptionMessage(const char *str);
  1702. virtual void setExceptionCode(unsigned code);
  1703. virtual void setSeverity(ErrorSeverity level);
  1704. virtual void setTimeStamp(const char * dt);
  1705. virtual void setExceptionFileName(const char *str);
  1706. virtual void setExceptionLineNo(unsigned r);
  1707. virtual void setExceptionColumn(unsigned c);
  1708. virtual void setActivityId(unsigned _id);
  1709. };
  1710. //==========================================================================================
  1711. extern WORKUNIT_API bool isSpecialResultSequence(unsigned sequence)
  1712. {
  1713. switch ((int) sequence)
  1714. {
  1715. case ResultSequenceInternal:
  1716. case ResultSequenceOnce:
  1717. case ResultSequencePersist:
  1718. case ResultSequenceStored:
  1719. return true;
  1720. default:
  1721. assertex(sequence <= INT_MAX);
  1722. return false;
  1723. }
  1724. }
  1725. class CConstWUArrayIterator : implements IConstWorkUnitIterator, public CInterface
  1726. {
  1727. unsigned curTreeNum;
  1728. IArrayOf<IPropertyTree> trees;
  1729. Owned<IConstWorkUnitInfo> cur;
  1730. void setCurrent()
  1731. {
  1732. cur.setown(new CLightweightWorkunitInfo(trees.item(curTreeNum)));
  1733. }
  1734. public:
  1735. IMPLEMENT_IINTERFACE;
  1736. CConstWUArrayIterator(IArrayOf<IPropertyTree> &_trees)
  1737. {
  1738. ForEachItemIn(t, _trees)
  1739. trees.append(*LINK(&_trees.item(t)));
  1740. curTreeNum = 0;
  1741. }
  1742. bool first()
  1743. {
  1744. curTreeNum = 0;
  1745. return next();
  1746. }
  1747. bool isValid()
  1748. {
  1749. return (NULL != cur.get());
  1750. }
  1751. bool next()
  1752. {
  1753. if (curTreeNum >= trees.ordinality())
  1754. {
  1755. cur.clear();
  1756. return false;
  1757. }
  1758. setCurrent();
  1759. ++curTreeNum;
  1760. return true;
  1761. }
  1762. IConstWorkUnitInfo & query() { return *cur; }
  1763. };
  1764. class CLocalWUFieldUsage : public CInterface, implements IConstWUFieldUsage
  1765. {
  1766. Owned<IPropertyTree> p;
  1767. public:
  1768. IMPLEMENT_IINTERFACE;
  1769. CLocalWUFieldUsage(IPropertyTree& _p) { p.setown(&_p); }
  1770. virtual const char * queryName() const { return p->queryProp("@name"); }
  1771. };
  1772. class CConstWUFieldUsageIterator : public CInterface, implements IConstWUFieldUsageIterator
  1773. {
  1774. public:
  1775. IMPLEMENT_IINTERFACE;
  1776. CConstWUFieldUsageIterator(IPropertyTreeIterator * tree) { iter.setown(tree); }
  1777. bool first() override { return iter->first(); }
  1778. bool isValid() override { return iter->isValid(); }
  1779. bool next() override { return iter->next(); }
  1780. IConstWUFieldUsage * get() const override { return new CLocalWUFieldUsage(iter->get()); }
  1781. private:
  1782. Owned<IPropertyTreeIterator> iter;
  1783. };
  1784. class CLocalWUFileUsage : public CInterface, implements IConstWUFileUsage
  1785. {
  1786. Owned<IPropertyTree> p;
  1787. public:
  1788. IMPLEMENT_IINTERFACE;
  1789. CLocalWUFileUsage(IPropertyTree& _p) { p.setown(&_p); }
  1790. virtual const char * queryName() const { return p->queryProp("@name"); }
  1791. virtual const char * queryType() const { return p->queryProp("@type"); }
  1792. virtual unsigned getNumFields() const { return p->getPropInt("@numFields"); }
  1793. virtual unsigned getNumFieldsUsed() const { return p->getPropInt("@numFieldsUsed"); }
  1794. virtual IConstWUFieldUsageIterator * getFields() const { return new CConstWUFieldUsageIterator(p->getElements("fields/field")); }
  1795. };
  1796. class CConstWUFileUsageIterator : public CInterface, implements IConstWUFileUsageIterator
  1797. {
  1798. public:
  1799. IMPLEMENT_IINTERFACE;
  1800. CConstWUFileUsageIterator(IPropertyTreeIterator * tree) { iter.setown(tree); }
  1801. bool first() override { return iter->first(); }
  1802. bool isValid() override { return iter->isValid(); }
  1803. bool next() override { return iter->next(); }
  1804. IConstWUFileUsage * get() const override { return new CLocalWUFileUsage(iter->get()); }
  1805. private:
  1806. Owned<IPropertyTreeIterator> iter;
  1807. };
  1808. //==========================================================================================
  1809. class CStringArrayIterator : implements IStringIterator, public CInterface
  1810. {
  1811. unsigned idx;
  1812. StringArray strings;
  1813. public:
  1814. IMPLEMENT_IINTERFACE;
  1815. CStringArrayIterator() { idx = 0; };
  1816. void append(const char *str) { strings.append(str); }
  1817. virtual bool first() { idx = 0; return strings.isItem(idx); }
  1818. virtual bool next() { idx ++; return strings.isItem(idx); }
  1819. virtual bool isValid() { return strings.isItem(idx); }
  1820. virtual IStringVal & str(IStringVal &s) { s.set(strings.item(idx)); return s; }
  1821. };
  1822. class CCachedJobNameIterator : implements IStringIterator, public CInterface
  1823. {
  1824. Owned<IPropertyTreeIterator> it;
  1825. public:
  1826. IMPLEMENT_IINTERFACE;
  1827. CCachedJobNameIterator(IPropertyTreeIterator *p) : it(p) {};
  1828. virtual bool first() { return it->first(); }
  1829. virtual bool next() { return it->next(); }
  1830. virtual bool isValid() { return it->isValid(); }
  1831. virtual IStringVal & str(IStringVal &s) { s.set(it->query().queryName()+1); return s; }
  1832. };
  1833. class CEmptyStringIterator : implements IStringIterator, public CInterface
  1834. {
  1835. public:
  1836. IMPLEMENT_IINTERFACE;
  1837. virtual bool first() { return false; }
  1838. virtual bool next() { return false; }
  1839. virtual bool isValid() { return false; }
  1840. virtual IStringVal & str(IStringVal &s) { s.clear(); return s; }
  1841. };
  1842. mapEnums workunitSortFields[] =
  1843. {
  1844. { WUSFuser, "@submitID" },
  1845. { WUSFcluster, "@clusterName" },
  1846. { WUSFjob, "@jobName" },
  1847. { WUSFstate, "@state" },
  1848. { WUSFpriority, "@priorityClass" },
  1849. { WUSFprotected, "@protected" },
  1850. { WUSFwuid, "@" },
  1851. { WUSFecl, "Query/Text" },
  1852. { WUSFfileread, "FilesRead/File/@name" },
  1853. { WUSFtotalthortime, "@totalThorTime|"
  1854. "Statistics/Statistic[@c='summary'][@creator='thor'][@kind='TimeElapsed']/@value|"
  1855. "Statistics/Statistic[@c='summary'][@creator='hthor'][@kind='TimeElapsed']/@value|"
  1856. "Statistics/Statistic[@c='summary'][@creator='roxie'][@kind='TimeElapsed']/@value|"
  1857. "Statistics/Statistic[@desc='Total thor time']/@value|"
  1858. "Timings/Timing[@name='Total thor time']/@duration" //Use Statistics first. If not found, use Timings
  1859. },
  1860. { WUSFwuidhigh, "@" },
  1861. { WUSFwildwuid, "@" },
  1862. { WUSFappvalue, "Application" },
  1863. { WUSFfilewritten, "Files/File/@name" },
  1864. { WUSFterm, NULL }
  1865. };
  1866. extern const char *queryFilterXPath(WUSortField field)
  1867. {
  1868. return getEnumText(field, workunitSortFields);
  1869. }
  1870. mapEnums querySortFields[] =
  1871. {
  1872. { WUQSFId, "@id" },
  1873. { WUQSFwuid, "@wuid" },
  1874. { WUQSFname, "@name" },
  1875. { WUQSFdll, "@dll" },
  1876. { WUQSFmemoryLimit, "@memoryLimit" },
  1877. { WUQSFmemoryLimitHi, "@memoryLimit" },
  1878. { WUQSFtimeLimit, "@timeLimit" },
  1879. { WUQSFtimeLimitHi, "@timeLimit" },
  1880. { WUQSFwarnTimeLimit, "@warnTimeLimit" },
  1881. { WUQSFwarnTimeLimitHi, "@warnTimeLimit" },
  1882. { WUQSFpriority, "@priority" },
  1883. { WUQSFpriorityHi, "@priority" },
  1884. { WUQSFQuerySet, "@querySetId" },
  1885. { WUQSFActivited, "@activated" },
  1886. { WUQSFSuspendedByUser, "@suspended" },
  1887. { WUQSFLibrary, "Library"},
  1888. { WUQSFPublishedBy, "@publishedBy" },
  1889. { WUQSFterm, NULL }
  1890. };
  1891. class asyncRemoveDllWorkItem: public CInterface, implements IWorkQueueItem // class only used in asyncRemoveDll
  1892. {
  1893. StringAttr name;
  1894. public:
  1895. IMPLEMENT_IINTERFACE;
  1896. asyncRemoveDllWorkItem(const char * _name) : name(_name)
  1897. {
  1898. }
  1899. void execute()
  1900. {
  1901. PROGLOG("WU removeDll %s", name.get());
  1902. queryDllServer().removeDll(name, true, true); // <name>, removeDlls=true, removeDirectory=true
  1903. }
  1904. };
  1905. class asyncRemoveRemoteFileWorkItem: public CInterface, implements IWorkQueueItem // class only used in asyncRemoveFile
  1906. {
  1907. RemoteFilename name;
  1908. public:
  1909. IMPLEMENT_IINTERFACE;
  1910. asyncRemoveRemoteFileWorkItem(const char * _ip, const char * _name)
  1911. {
  1912. SocketEndpoint ep(_ip);
  1913. name.setPath(ep, _name);
  1914. }
  1915. void execute()
  1916. {
  1917. Owned<IFile> file = createIFile(name);
  1918. PROGLOG("WU removeDll %s",file->queryFilename());
  1919. file->remove();
  1920. }
  1921. };
  1922. //==========================================================================================
  1923. class CConstQuerySetQueryIterator : implements IConstQuerySetQueryIterator, public CInterface
  1924. {
  1925. unsigned index;
  1926. IArrayOf<IPropertyTree> trees;
  1927. public:
  1928. IMPLEMENT_IINTERFACE;
  1929. CConstQuerySetQueryIterator(IArrayOf<IPropertyTree> &_trees)
  1930. {
  1931. ForEachItemIn(t, _trees)
  1932. trees.append(*LINK(&_trees.item(t)));
  1933. index = 0;
  1934. }
  1935. ~CConstQuerySetQueryIterator()
  1936. {
  1937. trees.kill();
  1938. }
  1939. bool first()
  1940. {
  1941. index = 0;
  1942. return (trees.ordinality()!=0);
  1943. }
  1944. bool next()
  1945. {
  1946. index++;
  1947. return (index<trees.ordinality());
  1948. }
  1949. bool isValid()
  1950. {
  1951. return (index<trees.ordinality());
  1952. }
  1953. IPropertyTree &query()
  1954. {
  1955. return trees.item(index);
  1956. }
  1957. };
  1958. class CSecurityCache
  1959. {
  1960. };
  1961. class CConstWUIterator : implements IConstWorkUnitIterator, public CInterface
  1962. {
  1963. public:
  1964. IMPLEMENT_IINTERFACE;
  1965. CConstWUIterator(IPropertyTreeIterator *_ptreeIter)
  1966. : ptreeIter(_ptreeIter)
  1967. {
  1968. }
  1969. bool first()
  1970. {
  1971. if (!ptreeIter->first())
  1972. {
  1973. cur.clear();
  1974. return false;
  1975. }
  1976. cur.setown(new CLightweightWorkunitInfo(ptreeIter->query()));
  1977. return true;
  1978. }
  1979. bool isValid()
  1980. {
  1981. return (NULL != cur.get());
  1982. }
  1983. bool next()
  1984. {
  1985. if (!ptreeIter->next())
  1986. {
  1987. cur.clear();
  1988. return false;
  1989. }
  1990. cur.setown(new CLightweightWorkunitInfo(ptreeIter->query()));
  1991. return true;
  1992. }
  1993. IConstWorkUnitInfo & query() { return *cur; }
  1994. private:
  1995. Owned<IConstWorkUnitInfo> cur;
  1996. Owned<IPropertyTreeIterator> ptreeIter;
  1997. };
  1998. class CSecureConstWUIterator : public CInterfaceOf<IConstWorkUnitIterator>
  1999. {
  2000. public:
  2001. CSecureConstWUIterator(IConstWorkUnitIterator *_parent, ISecManager *_secmgr=NULL, ISecUser *_secuser=NULL)
  2002. : parent(_parent), secmgr(_secmgr), secuser(_secuser)
  2003. {
  2004. assertex(_secuser && _secmgr);
  2005. }
  2006. bool first()
  2007. {
  2008. if (!parent->first())
  2009. return false;
  2010. return getNext();
  2011. }
  2012. bool next()
  2013. {
  2014. if (!parent->next())
  2015. return false;
  2016. return getNext();
  2017. }
  2018. virtual bool isValid()
  2019. {
  2020. return parent->isValid();
  2021. }
  2022. virtual IConstWorkUnitInfo &query()
  2023. {
  2024. return parent->query();
  2025. }
  2026. private:
  2027. Owned<IConstWorkUnitIterator> parent;
  2028. MapStringTo<int> scopePermissions;
  2029. Linked<ISecManager> secmgr;
  2030. Linked<ISecUser> secuser;
  2031. bool getNext() // scan for a workunit with permissions
  2032. {
  2033. do
  2034. {
  2035. const char *scopeName = parent->query().queryWuScope();
  2036. if (!scopeName || !*scopeName || checkScope(scopeName))
  2037. return true;
  2038. } while (parent->next());
  2039. return false;
  2040. }
  2041. bool checkScope(const char *scopeName)
  2042. {
  2043. int *perms = scopePermissions.getValue(scopeName);
  2044. SecAccessFlags perm;
  2045. if (!perms)
  2046. {
  2047. perm = secuser.get() ? secmgr->authorizeWorkunitScope(*secuser, scopeName) : SecAccess_Unavailable;
  2048. scopePermissions.setValue(scopeName, perm);
  2049. }
  2050. else
  2051. perm = (SecAccessFlags)*perms;
  2052. return perm >= SecAccess_Read;
  2053. }
  2054. };
  2055. CWorkUnitFactory::CWorkUnitFactory()
  2056. {
  2057. }
  2058. CWorkUnitFactory::~CWorkUnitFactory()
  2059. {
  2060. }
  2061. IWorkUnit* CWorkUnitFactory::createNamedWorkUnit(const char *wuid, const char *app, const char *scope, ISecManager *secmgr, ISecUser *secuser)
  2062. {
  2063. checkWuScopeSecAccess(scope, secmgr, secuser, SecAccess_Write, "Create", true, true);
  2064. Owned<CLocalWorkUnit> cw = _createWorkUnit(wuid, secmgr, secuser);
  2065. if (scope)
  2066. cw->setWuScope(scope); // Note - this may check access rights and throw exception. Is that correct? We might prefer to only check access once, and this will check on the lock too...
  2067. IWorkUnit* ret = &cw->lockRemote(false); // Note - this may throw exception if user does not have rights.
  2068. ret->setDebugValue("CREATED_BY", app, true);
  2069. ret->setDebugValue("CREATED_FOR", scope, true);
  2070. return ret;
  2071. }
  2072. IWorkUnit* CWorkUnitFactory::createWorkUnit(const char *app, const char *scope, ISecManager *secmgr, ISecUser *secuser)
  2073. {
  2074. StringBuffer wuid("W");
  2075. char result[32];
  2076. time_t ltime;
  2077. time( &ltime );
  2078. tm *today = localtime( &ltime ); // MORE - this is not threadsafe. But I probably don't care that much!
  2079. strftime(result, sizeof(result), "%Y%m%d-%H%M%S", today);
  2080. wuid.append(result);
  2081. if (workUnitTraceLevel > 1)
  2082. PrintLog("createWorkUnit created %s", wuid.str());
  2083. IWorkUnit* ret = createNamedWorkUnit(wuid.str(), app, scope, secmgr, secuser);
  2084. if (workUnitTraceLevel > 1)
  2085. PrintLog("createWorkUnit created %s", ret->queryWuid());
  2086. addTimeStamp(ret, SSTglobal, NULL, StWhenCreated);
  2087. return ret;
  2088. }
  2089. bool CWorkUnitFactory::deleteWorkUnit(const char * wuid, ISecManager *secmgr, ISecUser *secuser)
  2090. {
  2091. if (workUnitTraceLevel > 1)
  2092. PrintLog("deleteWorkUnit %s", wuid);
  2093. StringBuffer wuRoot;
  2094. getXPath(wuRoot, wuid);
  2095. Owned<CLocalWorkUnit> cw = _updateWorkUnit(wuid, secmgr, secuser);
  2096. if (!checkWuSecAccess(*cw.get(), secmgr, secuser, SecAccess_Full, "delete", true, true))
  2097. return false;
  2098. try
  2099. {
  2100. cw->cleanupAndDelete(true, true);
  2101. }
  2102. catch (IException *E)
  2103. {
  2104. StringBuffer s;
  2105. LOG(MCexception(E, MSGCLS_warning), E, s.append("Exception during deleteWorkUnit: ").append(wuid).str());
  2106. E->Release();
  2107. return false;
  2108. }
  2109. removeWorkUnitFromAllQueues(wuid); //known active workunits wouldn't make it this far
  2110. return true;
  2111. }
  2112. IConstWorkUnit* CWorkUnitFactory::openWorkUnit(const char *wuid, ISecManager *secmgr, ISecUser *secuser)
  2113. {
  2114. StringBuffer wuidStr(wuid);
  2115. wuidStr.trim();
  2116. if (wuidStr.length() && ('w' == wuidStr.charAt(0)))
  2117. wuidStr.setCharAt(0, 'W');
  2118. if (!wuidStr.length() || ('W' != wuidStr.charAt(0)))
  2119. {
  2120. if (workUnitTraceLevel > 1)
  2121. PrintLog("openWorkUnit %s invalid WUID", nullText(wuidStr.str()));
  2122. return NULL;
  2123. }
  2124. if (workUnitTraceLevel > 1)
  2125. PrintLog("openWorkUnit %s", wuidStr.str());
  2126. Owned<IConstWorkUnit> wu = _openWorkUnit(wuid, secmgr, secuser);
  2127. if (wu)
  2128. {
  2129. if (!checkWuSecAccess(*wu, secmgr, secuser, SecAccess_Read, "opening", true, true))
  2130. return NULL; // Actually throws exception on failure, so won't reach here
  2131. return wu.getClear();
  2132. }
  2133. else
  2134. {
  2135. if (workUnitTraceLevel > 0)
  2136. PrintLog("openWorkUnit %s not found", wuidStr.str());
  2137. return NULL;
  2138. }
  2139. }
  2140. IWorkUnit* CWorkUnitFactory::updateWorkUnit(const char *wuid, ISecManager *secmgr, ISecUser *secuser)
  2141. {
  2142. if (workUnitTraceLevel > 1)
  2143. PrintLog("updateWorkUnit %s", wuid);
  2144. Owned<CLocalWorkUnit> wu = _updateWorkUnit(wuid, secmgr, secuser);
  2145. if (wu)
  2146. {
  2147. if (!checkWuSecAccess(*wu.get(), secmgr, secuser, SecAccess_Write, "updating", true, true))
  2148. return NULL;
  2149. return &wu->lockRemote(false);
  2150. }
  2151. else
  2152. {
  2153. if (workUnitTraceLevel > 0)
  2154. PrintLog("updateWorkUnit %s not found", wuid);
  2155. return NULL;
  2156. }
  2157. }
  2158. IPropertyTree * pruneBranch(IPropertyTree * from, char const * xpath)
  2159. {
  2160. Owned<IPropertyTree> ret;
  2161. IPropertyTree * branch = from->queryPropTree(xpath);
  2162. if(branch)
  2163. {
  2164. ret.setown(createPTreeFromIPT(branch));
  2165. from->removeTree(branch);
  2166. }
  2167. return ret.getClear();
  2168. }
  2169. bool CWorkUnitFactory::restoreWorkUnit(const char *base, const char *wuid, bool restoreAssociated)
  2170. {
  2171. StringBuffer path(base);
  2172. addPathSepChar(path).append(wuid).append(".xml");
  2173. Owned<IPTree> pt = createPTreeFromXMLFile(path);
  2174. if (!pt)
  2175. return false;
  2176. CDateTime dt;
  2177. dt.setNow();
  2178. StringBuffer dts;
  2179. dt.getString(dts);
  2180. pt->setProp("@restoredDate", dts.str());
  2181. Owned<IPropertyTree> generatedDlls = pruneBranch(pt, "GeneratedDlls[1]");
  2182. Owned<IPropertyTree> associatedFiles;
  2183. IPropertyTree *srcAssociated = pt->queryPropTree("Query/Associated");
  2184. if (srcAssociated)
  2185. associatedFiles.setown(createPTreeFromIPT(srcAssociated));
  2186. // The updating of the repo is implementation specific...
  2187. if (!_restoreWorkUnit(pt.getClear(), wuid))
  2188. return false;
  2189. // now kludge back GeneratedDlls
  2190. if (generatedDlls)
  2191. {
  2192. Owned<IPropertyTreeIterator> dlls = generatedDlls->getElements("GeneratedDll");
  2193. for(dlls->first(); dlls->isValid(); dlls->next())
  2194. {
  2195. IPropertyTree & dll = dlls->query();
  2196. char const * name = dll.queryProp("@name");
  2197. char const * kind = dll.queryProp("@kind");
  2198. char const * location = dll.queryProp("@location");
  2199. Owned<IDllEntry> got = queryDllServer().getEntry(name);
  2200. if (!got)
  2201. {
  2202. RemoteFilename dstRfn;
  2203. dstRfn.setRemotePath(location);
  2204. StringBuffer srcPath(base);
  2205. addPathSepChar(srcPath);
  2206. dstRfn.getTail(srcPath);
  2207. OwnedIFile srcFile = createIFile(srcPath);
  2208. OwnedIFile dstFile = createIFile(dstRfn);
  2209. copyFile(dstFile, srcFile);
  2210. queryDllServer().registerDll(name, kind, location);
  2211. }
  2212. }
  2213. }
  2214. if (associatedFiles)
  2215. {
  2216. Owned<IPropertyTreeIterator> associated = associatedFiles->getElements("*");
  2217. ForEach(*associated)
  2218. {
  2219. IPropertyTree &file = associated->query();
  2220. const char *filename = file.queryProp("@filename");
  2221. SocketEndpoint ep(file.queryProp("@ip"));
  2222. RemoteFilename rfn;
  2223. rfn.setPath(ep, filename);
  2224. OwnedIFile dstFile = createIFile(rfn);
  2225. StringBuffer srcPath(base), name;
  2226. addPathSepChar(srcPath);
  2227. rfn.getTail(name);
  2228. srcPath.append(name);
  2229. if (generatedDlls)
  2230. {
  2231. VStringBuffer gDllPath("GeneratedDll[@name=\"%s\"]", name.str());
  2232. if (generatedDlls->hasProp(gDllPath))
  2233. continue; // generated dlls handled separately - see above
  2234. }
  2235. OwnedIFile srcFile = createIFile(srcPath);
  2236. if (srcFile->exists())
  2237. {
  2238. try
  2239. {
  2240. copyFile(dstFile, srcFile);
  2241. }
  2242. catch (IException *e)
  2243. {
  2244. VStringBuffer msg("Failed to restore associated file '%s' to destination '%s'", srcFile->queryFilename(), dstFile->queryFilename());
  2245. EXCLOG(e, msg.str());
  2246. e->Release();
  2247. }
  2248. }
  2249. }
  2250. }
  2251. return true;
  2252. }
  2253. int CWorkUnitFactory::setTracingLevel(int newLevel)
  2254. {
  2255. if (newLevel)
  2256. PrintLog("Setting workunit trace level to %d", newLevel);
  2257. int level = workUnitTraceLevel;
  2258. workUnitTraceLevel = newLevel;
  2259. return level;
  2260. }
  2261. void CWorkUnitFactory::descheduleAllWorkUnits(ISecManager *secmgr, ISecUser *secuser)
  2262. {
  2263. Owned<IRemoteConnection> conn = querySDS().connect("/Schedule", myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
  2264. if(!conn) return;
  2265. Owned<IPropertyTree> root(conn->queryRoot()->getBranch("."));
  2266. KeptAtomTable entries;
  2267. Owned<IPropertyTreeIterator> iter(root->getElements("*/*/*/*"));
  2268. StringBuffer wuid;
  2269. for(iter->first(); iter->isValid(); iter->next())
  2270. {
  2271. char const * entry = iter->query().queryName();
  2272. if(!entries.find(entry))
  2273. {
  2274. entries.addAtom(entry);
  2275. ncnameUnescape(entry, wuid.clear());
  2276. Owned<IWorkUnit> wu = updateWorkUnit(wuid, secmgr, secuser);
  2277. if(wu && (wu->getState() == WUStateWait))
  2278. wu->setState(WUStateCompleted);
  2279. }
  2280. }
  2281. bool more;
  2282. do more = root->removeProp("*"); while(more);
  2283. }
  2284. IConstQuerySetQueryIterator* CWorkUnitFactory::getQuerySetQueriesSorted( WUQuerySortField *sortorder, // list of fields to sort by (terminated by WUSFterm)
  2285. WUQuerySortField *filters, // NULL or list of fields to filter on (terminated by WUSFterm)
  2286. const void *filterbuf, // (appended) string values for filters
  2287. unsigned startoffset,
  2288. unsigned maxnum,
  2289. __int64 *cachehint,
  2290. unsigned *total,
  2291. const MapStringTo<bool> *_subset)
  2292. {
  2293. struct PostFilters
  2294. {
  2295. WUQueryFilterBoolean activatedFilter;
  2296. WUQueryFilterBoolean suspendedByUserFilter;
  2297. PostFilters()
  2298. {
  2299. activatedFilter = WUQFSAll;
  2300. suspendedByUserFilter = WUQFSAll;
  2301. };
  2302. } postFilters;
  2303. class CQuerySetQueriesPager : public CSimpleInterface, implements IElementsPager
  2304. {
  2305. StringAttr querySet;
  2306. StringAttr xPath;
  2307. StringAttr sortOrder;
  2308. PostFilters postFilters;
  2309. StringArray unknownAttributes;
  2310. const MapStringTo<bool> *subset;
  2311. void populateQueryTree(const IPropertyTree* querySetTree, IPropertyTree* queryTree)
  2312. {
  2313. const char* querySetId = querySetTree->queryProp("@id");
  2314. VStringBuffer path("Query%s", xPath.get());
  2315. Owned<IPropertyTreeIterator> iter = querySetTree->getElements(path.str());
  2316. ForEach(*iter)
  2317. {
  2318. IPropertyTree &query = iter->query();
  2319. bool activated = false;
  2320. const char* queryId = query.queryProp("@id");
  2321. if (queryId && *queryId)
  2322. {
  2323. if (subset)
  2324. {
  2325. VStringBuffer match("%s/%s", querySetId, queryId);
  2326. if (!subset->getValue(match))
  2327. continue;
  2328. }
  2329. VStringBuffer aliasXPath("Alias[@id='%s']", queryId);
  2330. IPropertyTree *alias = querySetTree->queryPropTree(aliasXPath.str());
  2331. if (alias)
  2332. activated = true;
  2333. }
  2334. if (activated && (postFilters.activatedFilter == WUQFSNo))
  2335. continue;
  2336. if (!activated && (postFilters.activatedFilter == WUQFSYes))
  2337. continue;
  2338. if ((postFilters.suspendedByUserFilter == WUQFSNo) && query.hasProp(getEnumText(WUQSFSuspendedByUser,querySortFields)))
  2339. continue;
  2340. if ((postFilters.suspendedByUserFilter == WUQFSYes) && !query.hasProp(getEnumText(WUQSFSuspendedByUser,querySortFields)))
  2341. continue;
  2342. IPropertyTree *queryWithSetId = queryTree->addPropTree("Query", createPTreeFromIPT(&query));
  2343. queryWithSetId->setProp("@querySetId", querySetId);
  2344. queryWithSetId->setPropBool("@activated", activated);
  2345. }
  2346. }
  2347. IRemoteConnection* populateQueryTree(IPropertyTree* queryTree)
  2348. {
  2349. StringBuffer querySetXPath("QuerySets");
  2350. if (!querySet.isEmpty())
  2351. querySetXPath.appendf("/QuerySet[@id=\"%s\"]", querySet.get());
  2352. Owned<IRemoteConnection> conn = querySDS().connect(querySetXPath.str(), myProcessSession(), 0, SDS_LOCK_TIMEOUT);
  2353. if (!conn)
  2354. return NULL;
  2355. if (querySet.isEmpty())
  2356. {
  2357. Owned<IPropertyTreeIterator> querySetIter = conn->queryRoot()->getElements("*");
  2358. ForEach(*querySetIter)
  2359. populateQueryTree(&querySetIter->query(), queryTree);
  2360. }
  2361. else
  2362. populateQueryTree(conn->queryRoot(), queryTree);
  2363. return conn.getClear();
  2364. }
  2365. public:
  2366. IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
  2367. CQuerySetQueriesPager(const char* _querySet, const char* _xPath, const char *_sortOrder, PostFilters& _postFilters, StringArray& _unknownAttributes, const MapStringTo<bool> *_subset)
  2368. : querySet(_querySet), xPath(_xPath), sortOrder(_sortOrder), subset(_subset)
  2369. {
  2370. postFilters.activatedFilter = _postFilters.activatedFilter;
  2371. postFilters.suspendedByUserFilter = _postFilters.suspendedByUserFilter;
  2372. ForEachItemIn(x, _unknownAttributes)
  2373. unknownAttributes.append(_unknownAttributes.item(x));
  2374. }
  2375. virtual IRemoteConnection* getElements(IArrayOf<IPropertyTree> &elements)
  2376. {
  2377. Owned<IPropertyTree> elementTree = createPTree("Queries");
  2378. Owned<IRemoteConnection> conn = populateQueryTree(elementTree);
  2379. if (!conn)
  2380. return NULL;
  2381. Owned<IPropertyTreeIterator> iter = elementTree->getElements("*");
  2382. if (!iter)
  2383. return NULL;
  2384. sortElements(iter, sortOrder.get(), NULL, NULL, unknownAttributes, elements);
  2385. return conn.getClear();
  2386. }
  2387. virtual bool allMatchingElementsReceived() { return true; } //For now, dali always returns all of matched Queries.
  2388. };
  2389. StringAttr querySet;
  2390. StringBuffer xPath;
  2391. StringBuffer so;
  2392. StringArray unknownAttributes;
  2393. if (filters)
  2394. {
  2395. const char *fv = (const char *)filterbuf;
  2396. for (unsigned i=0;filters[i]!=WUQSFterm;i++) {
  2397. int fmt = filters[i];
  2398. int subfmt = (fmt&0xff);
  2399. if (subfmt==WUQSFQuerySet)
  2400. querySet.set(fv);
  2401. else if ((subfmt==WUQSFmemoryLimit) || (subfmt==WUQSFtimeLimit) || (subfmt==WUQSFwarnTimeLimit) || (subfmt==WUQSFpriority))
  2402. xPath.append('[').append(getEnumText(subfmt,querySortFields)).append(">=").append(fv).append("]");
  2403. else if ((subfmt==WUQSFmemoryLimitHi) || (subfmt==WUQSFtimeLimitHi) || (subfmt==WUQSFwarnTimeLimitHi) || (subfmt==WUQSFpriorityHi))
  2404. xPath.append('[').append(getEnumText(subfmt,querySortFields)).append("<=").append(fv).append("]");
  2405. else if (subfmt==WUQSFActivited)
  2406. postFilters.activatedFilter = (WUQueryFilterBoolean) atoi(fv);
  2407. else if (subfmt==WUQSFSuspendedByUser)
  2408. postFilters.suspendedByUserFilter = (WUQueryFilterBoolean) atoi(fv);
  2409. else if (!fv || !*fv)
  2410. unknownAttributes.append(getEnumText(subfmt,querySortFields));
  2411. else {
  2412. xPath.append('[').append(getEnumText(subfmt,querySortFields)).append('=');
  2413. if (fmt&WUQSFnocase)
  2414. xPath.append('?');
  2415. if (fmt&WUQSFnumeric)
  2416. xPath.append('#');
  2417. if (fmt&WUQSFwild)
  2418. xPath.append('~');
  2419. xPath.append('"').append(fv).append("\"]");
  2420. }
  2421. fv = fv + strlen(fv)+1;
  2422. }
  2423. }
  2424. if (sortorder) {
  2425. for (unsigned i=0;sortorder[i]!=WUQSFterm;i++) {
  2426. if (so.length())
  2427. so.append(',');
  2428. int fmt = sortorder[i];
  2429. if (fmt&WUQSFreverse)
  2430. so.append('-');
  2431. if (fmt&WUQSFnocase)
  2432. so.append('?');
  2433. if (fmt&WUQSFnumeric)
  2434. so.append('#');
  2435. so.append(getEnumText(fmt&0xff,querySortFields));
  2436. }
  2437. }
  2438. IArrayOf<IPropertyTree> results;
  2439. Owned<IElementsPager> elementsPager = new CQuerySetQueriesPager(querySet.get(), xPath.str(), so.length()?so.str():NULL, postFilters, unknownAttributes, _subset);
  2440. Owned<IRemoteConnection> conn=getElementsPaged(elementsPager,startoffset,maxnum,NULL,"",cachehint,results,total,NULL);
  2441. return new CConstQuerySetQueryIterator(results);
  2442. }
  2443. bool CWorkUnitFactory::isAborting(const char *wuid) const
  2444. {
  2445. VStringBuffer apath("/WorkUnitAborts/%s", wuid);
  2446. try
  2447. {
  2448. Owned<IRemoteConnection> acon = querySDS().connect(apath.str(), myProcessSession(), 0, SDS_LOCK_TIMEOUT);
  2449. if (acon)
  2450. return acon->queryRoot()->getPropInt(NULL) != 0;
  2451. }
  2452. catch (IException *E)
  2453. {
  2454. EXCLOG(E);
  2455. E->Release();
  2456. }
  2457. return false;
  2458. }
  2459. void CWorkUnitFactory::clearAborting(const char *wuid)
  2460. {
  2461. VStringBuffer apath("/WorkUnitAborts/%s", wuid);
  2462. try
  2463. {
  2464. Owned<IRemoteConnection> acon = querySDS().connect(apath.str(), myProcessSession(), RTM_LOCK_WRITE|RTM_LOCK_SUB, SDS_LOCK_TIMEOUT);
  2465. if (acon)
  2466. acon->close(true);
  2467. }
  2468. catch (IException *E)
  2469. {
  2470. EXCLOG(E);
  2471. E->Release();
  2472. }
  2473. }
  2474. void CWorkUnitFactory::reportAbnormalTermination(const char *wuid, WUState &state, SessionId agent)
  2475. {
  2476. WARNLOG("reportAbnormalTermination: session stopped unexpectedly: %" I64F "d state: %d", (__int64) agent, (int) state);
  2477. bool isEcl = false;
  2478. switch (state)
  2479. {
  2480. case WUStateAborting:
  2481. state = WUStateAborted;
  2482. break;
  2483. case WUStateCompiling:
  2484. isEcl = true;
  2485. // drop into
  2486. default:
  2487. state = WUStateFailed;
  2488. }
  2489. Owned<IWorkUnit> wu = updateWorkUnit(wuid, NULL, NULL);
  2490. wu->setState(state);
  2491. Owned<IWUException> e = wu->createException();
  2492. e->setExceptionCode(isEcl ? 1001 : 1000);
  2493. e->setExceptionMessage(isEcl ? "EclServer terminated unexpectedly" : "Workunit terminated unexpectedly");
  2494. }
  2495. static CriticalSection deleteDllLock;
  2496. static IWorkQueueThread *deleteDllWorkQ = nullptr;
  2497. MODULE_INIT(INIT_PRIORITY_STANDARD)
  2498. {
  2499. return true;
  2500. }
  2501. MODULE_EXIT()
  2502. {
  2503. CriticalBlock b(deleteDllLock);
  2504. if (deleteDllWorkQ)
  2505. ::Release(deleteDllWorkQ);
  2506. deleteDllWorkQ = nullptr;
  2507. }
  2508. static void asyncRemoveDll(const char * name)
  2509. {
  2510. CriticalBlock b(deleteDllLock);
  2511. if (!deleteDllWorkQ)
  2512. deleteDllWorkQ = createWorkQueueThread();
  2513. deleteDllWorkQ->post(new asyncRemoveDllWorkItem(name));
  2514. }
  2515. static void asyncRemoveFile(const char * ip, const char * name)
  2516. {
  2517. CriticalBlock b(deleteDllLock);
  2518. if (!deleteDllWorkQ)
  2519. deleteDllWorkQ = createWorkQueueThread();
  2520. deleteDllWorkQ->post(new asyncRemoveRemoteFileWorkItem(ip, name));
  2521. }
  2522. class CDaliWorkUnitFactory : public CWorkUnitFactory, implements IDaliClientShutdown
  2523. {
  2524. public:
  2525. IMPLEMENT_IINTERFACE_USING(CWorkUnitFactory);
  2526. CDaliWorkUnitFactory()
  2527. {
  2528. // Assumes dali client configuration has already been done
  2529. sdsManager = &querySDS();
  2530. session = myProcessSession();
  2531. addShutdownHook(*this);
  2532. }
  2533. ~CDaliWorkUnitFactory()
  2534. {
  2535. removeShutdownHook(*this);
  2536. }
  2537. virtual bool initializeStore()
  2538. {
  2539. throwUnexpected(); // Used when loading a plugin factory - not applicable here
  2540. }
  2541. virtual IWorkUnitWatcher *getWatcher(IWorkUnitSubscriber *subscriber, WUSubscribeOptions options, const char *wuid) const
  2542. {
  2543. return new CDaliWorkUnitWatcher(subscriber, options, wuid);
  2544. }
  2545. virtual unsigned validateRepository(bool fixErrors)
  2546. {
  2547. return 0;
  2548. }
  2549. virtual void deleteRepository(bool recreate)
  2550. {
  2551. Owned<IRemoteConnection> conn = sdsManager->connect("/WorkUnits", session, RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
  2552. if (conn)
  2553. conn->close(true);
  2554. conn.setown(sdsManager->connect("/GraphProgress", session, RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT));
  2555. if (conn)
  2556. conn->close(true);
  2557. }
  2558. virtual void createRepository()
  2559. {
  2560. // Nothing to do
  2561. }
  2562. virtual const char *queryStoreType() const
  2563. {
  2564. return "Dali";
  2565. }
  2566. virtual CLocalWorkUnit *_createWorkUnit(const char *wuid, ISecManager *secmgr, ISecUser *secuser)
  2567. {
  2568. StringBuffer wuRoot;
  2569. getXPath(wuRoot, wuid);
  2570. IRemoteConnection *conn;
  2571. conn = sdsManager->connect(wuRoot.str(), session, RTM_LOCK_WRITE|RTM_CREATE_UNIQUE, SDS_LOCK_TIMEOUT);
  2572. conn->queryRoot()->setProp("@xmlns:xsi", "http://www.w3.org/1999/XMLSchema-instance");
  2573. conn->queryRoot()->setPropInt("@wuidVersion", WUID_VERSION);
  2574. return new CDaliWorkUnit(conn, secmgr, secuser);
  2575. }
  2576. virtual CLocalWorkUnit* _openWorkUnit(const char *wuid, ISecManager *secmgr, ISecUser *secuser)
  2577. {
  2578. StringBuffer wuRoot;
  2579. getXPath(wuRoot, wuid);
  2580. IRemoteConnection* conn = sdsManager->connect(wuRoot.str(), session, 0, SDS_LOCK_TIMEOUT);
  2581. if (conn)
  2582. return new CDaliWorkUnit(conn, secmgr, secuser);
  2583. else
  2584. return NULL;
  2585. }
  2586. virtual bool _restoreWorkUnit(IPTree *_pt, const char *wuid)
  2587. {
  2588. Owned<IPTree> pt(_pt);
  2589. Owned<IPropertyTree> gprogress = pruneBranch(pt, "GraphProgress[1]");
  2590. StringBuffer wuRoot;
  2591. getXPath(wuRoot, wuid);
  2592. Owned<IRemoteConnection> conn = sdsManager->connect(wuRoot.str(), myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
  2593. if (!conn)
  2594. {
  2595. ERRLOG("restoreWorkUnit could not create to %s", wuRoot.str());
  2596. return false;
  2597. }
  2598. IPropertyTree *root = conn->queryRoot();
  2599. if (root->hasChildren())
  2600. {
  2601. ERRLOG("restoreWorkUnit WUID %s already exists", wuid);
  2602. return false;
  2603. }
  2604. root->setPropTree(NULL, pt.getClear());
  2605. conn.clear();
  2606. // now kludge back GraphProgress
  2607. if (gprogress)
  2608. {
  2609. VStringBuffer xpath("/GraphProgress/%s", wuid);
  2610. conn.setown(querySDS().connect(xpath, myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT));
  2611. if (conn)
  2612. {
  2613. IPropertyTree *groot = conn->queryRoot();
  2614. if (groot->hasChildren())
  2615. WARNLOG("restoreWorkUnit WUID %s graphprogress already exists, replacing",wuid);
  2616. groot->setPropTree(NULL, gprogress.getClear());
  2617. }
  2618. }
  2619. return true;
  2620. }
  2621. virtual CLocalWorkUnit* _updateWorkUnit(const char *wuid, ISecManager *secmgr, ISecUser *secuser)
  2622. {
  2623. StringBuffer wuRoot;
  2624. getXPath(wuRoot, wuid);
  2625. IRemoteConnection* conn = sdsManager->connect(wuRoot.str(), session, RTM_LOCK_WRITE|RTM_LOCK_SUB, SDS_LOCK_TIMEOUT);
  2626. if (conn)
  2627. return new CDaliWorkUnit(conn, secmgr, secuser);
  2628. else
  2629. return NULL;
  2630. }
  2631. virtual IWorkUnit* getGlobalWorkUnit(ISecManager *secmgr, ISecUser *secuser)
  2632. {
  2633. // MORE - should it check security?
  2634. StringBuffer wuRoot;
  2635. getXPath(wuRoot, GLOBAL_WORKUNIT);
  2636. IRemoteConnection* conn = sdsManager->connect(wuRoot.str(), session, RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
  2637. conn->queryRoot()->setProp("@xmlns:xsi", "http://www.w3.org/1999/XMLSchema-instance");
  2638. Owned<CLocalWorkUnit> cw = new CDaliWorkUnit(conn, (ISecManager *) NULL, NULL);
  2639. return &cw->lockRemote(false);
  2640. }
  2641. virtual IConstWorkUnitIterator* getWorkUnitsByOwner(const char * owner, ISecManager *secmgr, ISecUser *secuser)
  2642. {
  2643. StringBuffer path("*");
  2644. if (owner && *owner)
  2645. path.append("[@submitID=?~\"").append(owner).append("\"]");
  2646. return _getWorkUnitsByXPath(path.str(), secmgr, secuser);
  2647. }
  2648. IConstWorkUnitIterator* getScheduledWorkUnits(ISecManager *secmgr, ISecUser *secuser)
  2649. {
  2650. StringBuffer path("*");
  2651. path.append("[@state=\"").append(getEnumText(WUStateScheduled, states)).append("\"]");
  2652. return _getWorkUnitsByXPath(path.str(), secmgr, secuser);
  2653. }
  2654. virtual void clientShutdown();
  2655. virtual unsigned numWorkUnits()
  2656. {
  2657. Owned<IRemoteConnection> conn = sdsManager->connect("/WorkUnits", session, 0, SDS_LOCK_TIMEOUT);
  2658. if (!conn)
  2659. return 0;
  2660. IPropertyTree *root = conn->queryRoot();
  2661. return root->numChildren();
  2662. }
  2663. /**
  2664. * Add a filter to an xpath query, with the appropriate filter flags
  2665. */
  2666. static void appendFilterToQueryString(StringBuffer& query, int flags, const char* name, const char* value)
  2667. {
  2668. query.append('[').append(name).append('=');
  2669. if (flags & WUSFnocase)
  2670. query.append('?');
  2671. if (flags & WUSFwild)
  2672. query.append('~');
  2673. query.append('"').append(value).append("\"]");
  2674. };
  2675. IConstWorkUnitIterator* getWorkUnitsSorted( WUSortField sortorder, // field to sort by (and flags for desc sort etc)
  2676. WUSortField *filters, // NULL or list of fields to filter on (terminated by WUSFterm)
  2677. const void *filterbuf, // (appended) string values for filters
  2678. unsigned startoffset,
  2679. unsigned maxnum,
  2680. __int64 *cachehint,
  2681. unsigned *total,
  2682. ISecManager *secmgr,
  2683. ISecUser *secuser)
  2684. {
  2685. class CQueryOrFilter : public CInterface
  2686. {
  2687. unsigned flags;
  2688. StringAttr name;
  2689. StringArray values;
  2690. public:
  2691. IMPLEMENT_IINTERFACE;
  2692. CQueryOrFilter(unsigned _flags, const char *_name, const char *_value)
  2693. : flags(_flags), name(_name)
  2694. {
  2695. values.appendListUniq(_value, "|");
  2696. };
  2697. const char* queryName() { return name.get(); };
  2698. unsigned querySearchFlags() { return flags; };
  2699. const StringArray& queryValues() const { return values; };
  2700. };
  2701. class CMultiPTreeIterator : public CInterfaceOf<IPropertyTreeIterator>
  2702. {
  2703. public:
  2704. virtual bool first() override
  2705. {
  2706. curSource = 0;
  2707. while (sources.isItem(curSource))
  2708. {
  2709. if (sources.item(curSource).first())
  2710. return true;
  2711. curSource++;
  2712. }
  2713. return false;
  2714. }
  2715. virtual bool next() override
  2716. {
  2717. if (sources.isItem(curSource))
  2718. {
  2719. if (sources.item(curSource).next())
  2720. return true;
  2721. curSource++;
  2722. while (sources.isItem(curSource))
  2723. {
  2724. if (sources.item(curSource).first())
  2725. return true;
  2726. curSource++;
  2727. }
  2728. }
  2729. return false;
  2730. }
  2731. virtual bool isValid() override
  2732. {
  2733. return sources.isItem(curSource);
  2734. }
  2735. virtual IPropertyTree & query() override
  2736. {
  2737. return sources.item(curSource).query();
  2738. }
  2739. void addSource(IPropertyTreeIterator &source)
  2740. {
  2741. sources.append(source);
  2742. }
  2743. private:
  2744. IArrayOf<IPropertyTreeIterator> sources;
  2745. unsigned curSource = 0;
  2746. };
  2747. class CWorkUnitsPager : public CSimpleInterface, implements IElementsPager
  2748. {
  2749. StringAttr xPath;
  2750. StringAttr sortOrder;
  2751. StringAttr nameFilterLo;
  2752. StringAttr nameFilterHi;
  2753. StringArray unknownAttributes;
  2754. Owned<CQueryOrFilter> orFilter;
  2755. public:
  2756. IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
  2757. CWorkUnitsPager(const char* _xPath, CQueryOrFilter* _orFilter, const char *_sortOrder, const char* _nameFilterLo, const char* _nameFilterHi, StringArray& _unknownAttributes)
  2758. : xPath(_xPath), orFilter(_orFilter), sortOrder(_sortOrder), nameFilterLo(_nameFilterLo), nameFilterHi(_nameFilterHi)
  2759. {
  2760. ForEachItemIn(x, _unknownAttributes)
  2761. unknownAttributes.append(_unknownAttributes.item(x));
  2762. }
  2763. virtual IRemoteConnection* getElements(IArrayOf<IPropertyTree> &elements)
  2764. {
  2765. Owned<IRemoteConnection> conn = querySDS().connect("WorkUnits", myProcessSession(), 0, SDS_LOCK_TIMEOUT);
  2766. if (!conn)
  2767. return NULL;
  2768. Owned<IPropertyTreeIterator> iter;
  2769. if (!orFilter)
  2770. {
  2771. iter.setown(conn->getElements(xPath.get()));
  2772. }
  2773. else
  2774. {
  2775. Owned <CMultiPTreeIterator> multi = new CMultiPTreeIterator;
  2776. bool added = false;
  2777. const char* fieldName = orFilter->queryName();
  2778. unsigned flags = orFilter->querySearchFlags();
  2779. const StringArray& values = orFilter->queryValues();
  2780. ForEachItemIn(i, values)
  2781. {
  2782. StringBuffer path = xPath.get();
  2783. const char* value = values.item(i);
  2784. if (!isEmptyString(value))
  2785. {
  2786. appendFilterToQueryString(path, flags, fieldName, value);
  2787. IPropertyTreeIterator *itr = conn->getElements(path.str());
  2788. if (itr)
  2789. {
  2790. multi->addSource(*itr);
  2791. added = true;
  2792. }
  2793. }
  2794. }
  2795. if (added)
  2796. iter.setown(multi.getClear());
  2797. }
  2798. if (!iter)
  2799. return NULL;
  2800. sortElements(iter, sortOrder.get(), nameFilterLo.get(), nameFilterHi.get(), unknownAttributes, elements);
  2801. return conn.getClear();
  2802. }
  2803. virtual bool allMatchingElementsReceived() { return true; }//For now, dali always returns all of matched WUs.
  2804. };
  2805. class CScopeChecker : public CSimpleInterface, implements ISortedElementsTreeFilter
  2806. {
  2807. UniqueScopes done;
  2808. ISecManager *secmgr;
  2809. ISecUser *secuser;
  2810. CriticalSection crit;
  2811. public:
  2812. IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
  2813. CScopeChecker(ISecManager *_secmgr,ISecUser *_secuser)
  2814. {
  2815. secmgr = _secmgr;
  2816. secuser = _secuser;
  2817. }
  2818. bool isOK(IPropertyTree &tree)
  2819. {
  2820. const char *scopename = tree.queryProp("@scope");
  2821. if (!scopename||!*scopename)
  2822. return true;
  2823. {
  2824. CriticalBlock block(crit);
  2825. const bool *b = done.getValue(scopename);
  2826. if (b)
  2827. return *b;
  2828. }
  2829. bool ret = checkWuScopeSecAccess(scopename,secmgr,secuser,SecAccess_Read,"iterating",false,false);
  2830. {
  2831. // conceivably could have already been checked and added, but ok.
  2832. CriticalBlock block(crit);
  2833. done.setValue(scopename,ret);
  2834. }
  2835. return ret;
  2836. }
  2837. };
  2838. Owned<CQueryOrFilter> orFilter;
  2839. Owned<ISortedElementsTreeFilter> sc = new CScopeChecker(secmgr,secuser);
  2840. StringBuffer query;
  2841. StringBuffer so;
  2842. StringAttr namefilter("*");
  2843. StringAttr namefilterlo;
  2844. StringAttr namefilterhi;
  2845. StringArray unknownAttributes;
  2846. if (filters)
  2847. {
  2848. const char *fv = (const char *) filterbuf;
  2849. for (unsigned i=0;filters[i]!=WUSFterm;i++)
  2850. {
  2851. assertex(fv);
  2852. int fmt = filters[i];
  2853. int subfmt = (fmt&0xff);
  2854. if (subfmt==WUSFwuid)
  2855. namefilterlo.set(fv);
  2856. else if (subfmt==WUSFwuidhigh)
  2857. namefilterhi.set(fv);
  2858. else if (subfmt==WUSFwildwuid)
  2859. namefilter.set(fv);
  2860. else if (subfmt==WUSFappvalue)
  2861. {
  2862. const char *app = fv;
  2863. fv = fv + strlen(fv)+1;
  2864. query.append("[Application/").append(app);
  2865. if (*fv)
  2866. query.append("=?~\"").append(fv).append('\"');
  2867. query.append("]");
  2868. }
  2869. else if (!*fv)
  2870. {
  2871. unknownAttributes.append(getEnumText(subfmt,workunitSortFields));
  2872. if (subfmt==WUSFtotalthortime)
  2873. sortorder = (WUSortField) (sortorder & ~WUSFnumeric);
  2874. }
  2875. else
  2876. {
  2877. const char* fieldName = getEnumText(subfmt,workunitSortFields);
  2878. if (!strchr(fv, '|'))
  2879. appendFilterToQueryString(query, fmt, fieldName, fv);
  2880. else if (orFilter)
  2881. throw MakeStringException(WUERR_InvalidUserInput, "Multiple OR filters not allowed");
  2882. else
  2883. {
  2884. if (!strieq(fieldName, getEnumText(WUSFstate,workunitSortFields)) &&
  2885. !strieq(fieldName, getEnumText(WUSFuser,workunitSortFields)) &&
  2886. !strieq(fieldName, getEnumText(WUSFcluster,workunitSortFields)))
  2887. throw MakeStringException(WUERR_InvalidUserInput, "OR filters not allowed for %s", fieldName);
  2888. orFilter.setown(new CQueryOrFilter(fmt, fieldName, fv));
  2889. }
  2890. }
  2891. fv = fv + strlen(fv)+1;
  2892. }
  2893. }
  2894. if ((sortorder&0xff)==WUSFtotalthortime)
  2895. sortorder = (WUSortField) (sortorder & ~WUSFnumeric);
  2896. query.insert(0, namefilter.get());
  2897. if (sortorder)
  2898. {
  2899. if (so.length())
  2900. so.append(',');
  2901. if (sortorder & WUSFreverse)
  2902. so.append('-');
  2903. if (sortorder & WUSFnocase)
  2904. so.append('?');
  2905. if (sortorder & WUSFnumeric)
  2906. so.append('#');
  2907. so.append(getEnumText(sortorder&0xff,workunitSortFields));
  2908. }
  2909. IArrayOf<IPropertyTree> results;
  2910. Owned<IElementsPager> elementsPager = new CWorkUnitsPager(query.str(), orFilter.getClear(), so.length()?so.str():NULL, namefilterlo.get(), namefilterhi.get(), unknownAttributes);
  2911. Owned<IRemoteConnection> conn=getElementsPaged(elementsPager,startoffset,maxnum,secmgr?sc:NULL,"",cachehint,results,total,NULL);
  2912. return new CConstWUArrayIterator(results);
  2913. }
  2914. virtual WUState waitForWorkUnit(const char * wuid, unsigned timeout, bool compiled, bool returnOnWaitState)
  2915. {
  2916. WUState ret = WUStateUnknown;
  2917. StringBuffer wuRoot;
  2918. getXPath(wuRoot, wuid);
  2919. Owned<IRemoteConnection> conn = sdsManager->connect(wuRoot.str(), session, 0, SDS_LOCK_TIMEOUT);
  2920. if (timeout == 0) //no need to subscribe
  2921. {
  2922. ret = (WUState) getEnum(conn->queryRoot(), "@state", states);
  2923. switch (ret)
  2924. {
  2925. case WUStateCompiled:
  2926. case WUStateUploadingFiles:
  2927. if (!compiled)
  2928. break;
  2929. //fall through
  2930. case WUStateCompleted:
  2931. case WUStateFailed:
  2932. case WUStateAborted:
  2933. return ret;
  2934. case WUStateWait:
  2935. if(returnOnWaitState)
  2936. return ret;
  2937. default:
  2938. break;
  2939. }
  2940. return WUStateUnknown;
  2941. }
  2942. Owned<WorkUnitWaiter> waiter = new WorkUnitWaiter(wuid, SubscribeOptionState);
  2943. LocalIAbortHandler abortHandler(*waiter);
  2944. if (conn)
  2945. {
  2946. SessionId agent = -1;
  2947. bool agentSessionStopped = false;
  2948. unsigned start = msTick();
  2949. for (;;)
  2950. {
  2951. ret = (WUState) getEnum(conn->queryRoot(), "@state", states);
  2952. switch (ret)
  2953. {
  2954. case WUStateCompiled:
  2955. case WUStateUploadingFiles:
  2956. if (!compiled)
  2957. break;
  2958. // fall into
  2959. case WUStateCompleted:
  2960. case WUStateFailed:
  2961. case WUStateAborted:
  2962. return ret;
  2963. case WUStateWait:
  2964. if(returnOnWaitState)
  2965. {
  2966. return ret;
  2967. }
  2968. break;
  2969. case WUStateCompiling:
  2970. case WUStateRunning:
  2971. case WUStateDebugPaused:
  2972. case WUStateDebugRunning:
  2973. case WUStateBlocked:
  2974. case WUStateAborting:
  2975. if (agentSessionStopped)
  2976. {
  2977. reportAbnormalTermination(wuid, ret, agent);
  2978. return ret;
  2979. }
  2980. if (queryDaliServerVersion().compare("2.1")>=0)
  2981. {
  2982. agent = conn->queryRoot()->getPropInt64("@agentSession", -1);
  2983. if((agent>0) && querySessionManager().sessionStopped(agent, 0))
  2984. {
  2985. agentSessionStopped = true;
  2986. conn->reload();
  2987. continue;
  2988. }
  2989. }
  2990. break;
  2991. }
  2992. agentSessionStopped = false; // reset for state changes such as WUStateWait then WUStateRunning again
  2993. unsigned waited = msTick() - start;
  2994. if (timeout==-1 || waited + 20000 < timeout)
  2995. {
  2996. waiter->wait(20000); // recheck state every 20 seconds, in case eclagent has crashed.
  2997. if (waiter->aborted)
  2998. {
  2999. ret = WUStateUnknown; // MORE - throw an exception?
  3000. break;
  3001. }
  3002. }
  3003. else if (waited > timeout || !waiter->wait(timeout-waited))
  3004. {
  3005. ret = WUStateUnknown; // MORE - throw an exception?
  3006. break;
  3007. }
  3008. conn->reload();
  3009. }
  3010. }
  3011. return ret;
  3012. }
  3013. virtual WUAction waitForWorkUnitAction(const char * wuid, WUAction original)
  3014. {
  3015. Owned<WorkUnitWaiter> waiter = new WorkUnitWaiter(wuid, SubscribeOptionAction);
  3016. LocalIAbortHandler abortHandler(*waiter);
  3017. WUAction ret = WUActionUnknown;
  3018. StringBuffer wuRoot;
  3019. getXPath(wuRoot, wuid);
  3020. Owned<IRemoteConnection> conn = sdsManager->connect(wuRoot.str(), session, 0, SDS_LOCK_TIMEOUT);
  3021. if (conn)
  3022. {
  3023. unsigned start = msTick();
  3024. for (;;)
  3025. {
  3026. ret = (WUAction) getEnum(conn->queryRoot(), "Action", actions);
  3027. if (ret != original)
  3028. break;
  3029. unsigned waited = msTick() - start;
  3030. waiter->wait(20000); // recheck state every 20 seconds even if no timeout, in case eclagent has crashed.
  3031. if (waiter->aborted)
  3032. {
  3033. ret = WUActionUnknown; // MORE - throw an exception?
  3034. break;
  3035. }
  3036. conn->reload();
  3037. }
  3038. }
  3039. waiter->unsubscribe();
  3040. return ret;
  3041. }
  3042. protected:
  3043. IConstWorkUnitIterator * _getWorkUnitsByXPath(const char *xpath, ISecManager *secmgr, ISecUser *secuser)
  3044. {
  3045. Owned<IRemoteConnection> conn = sdsManager->connect("/WorkUnits", session, 0, SDS_LOCK_TIMEOUT);
  3046. if (conn)
  3047. {
  3048. CDaliVersion serverVersionNeeded("3.2");
  3049. Owned<IPropertyTreeIterator> iter(queryDaliServerVersion().compare(serverVersionNeeded) < 0 ?
  3050. conn->queryRoot()->getElements(xpath) :
  3051. conn->getElements(xpath));
  3052. return createSecureConstWUIterator(iter.getClear(), secmgr, secuser);
  3053. }
  3054. else
  3055. return NULL;
  3056. }
  3057. ISDSManager *sdsManager;
  3058. SessionId session;
  3059. };
  3060. extern WORKUNIT_API IConstWorkUnitIterator *createSecureConstWUIterator(IConstWorkUnitIterator *iter, ISecManager *secmgr, ISecUser *secuser)
  3061. {
  3062. if (secmgr)
  3063. return new CSecureConstWUIterator(iter, secmgr, secuser);
  3064. else
  3065. return iter;
  3066. }
  3067. extern WORKUNIT_API IConstWorkUnitIterator *createSecureConstWUIterator(IPropertyTreeIterator *iter, ISecManager *secmgr, ISecUser *secuser)
  3068. {
  3069. if (secmgr)
  3070. return new CSecureConstWUIterator(new CConstWUIterator(iter), secmgr, secuser);
  3071. else
  3072. return new CConstWUIterator(iter);
  3073. }
  3074. static CriticalSection factoryCrit;
  3075. static Owned<IWorkUnitFactory> factory;
  3076. void CDaliWorkUnitFactory::clientShutdown()
  3077. {
  3078. CriticalBlock b(factoryCrit);
  3079. factory.clear();
  3080. }
  3081. void clientShutdownWorkUnit()
  3082. {
  3083. CriticalBlock b(factoryCrit);
  3084. factory.clear();
  3085. }
  3086. extern WORKUNIT_API void setWorkUnitFactory(IWorkUnitFactory * _factory)
  3087. {
  3088. CriticalBlock b(factoryCrit);
  3089. factory.setown(_factory);
  3090. }
  3091. extern WORKUNIT_API IWorkUnitFactory * getWorkUnitFactory()
  3092. {
  3093. if (!factory)
  3094. {
  3095. CriticalBlock b(factoryCrit);
  3096. if (!factory) // NOTE - this "double test" paradigm is not guaranteed threadsafe on modern systems/compilers - I think in this instance that is harmless even in the (extremely) unlikely event that it resulted in the setown being called twice.
  3097. {
  3098. const char *forceEnv = getenv("FORCE_DALI_WORKUNITS");
  3099. bool forceDali = forceEnv && !strieq(forceEnv, "off") && !strieq(forceEnv, "0");
  3100. Owned<IRemoteConnection> env = querySDS().connect("/Environment", myProcessSession(), 0, SDS_LOCK_TIMEOUT);
  3101. IPropertyTree *pluginInfo = NULL;
  3102. if (env)
  3103. {
  3104. SocketEndpoint targetDali = queryCoven().queryGroup().queryNode(0).endpoint();
  3105. IPropertyTree *daliInfo = findDaliProcess(env->queryRoot(), targetDali);
  3106. if (daliInfo)
  3107. {
  3108. const char *daliName = daliInfo->queryProp("@name");
  3109. if (daliName)
  3110. {
  3111. VStringBuffer xpath("Software/DaliServerPlugin[@type='WorkunitServer'][@daliServers='%s']", daliName);
  3112. pluginInfo = env->queryRoot()->queryPropTree(xpath);
  3113. }
  3114. if (!pluginInfo)
  3115. pluginInfo = daliInfo->queryPropTree("Plugin[@type='WorkunitServer']"); // Compatibility with early betas of 6.0 ...
  3116. }
  3117. }
  3118. if (pluginInfo && !forceDali)
  3119. factory.setown( (IWorkUnitFactory *) loadPlugin(pluginInfo));
  3120. else
  3121. factory.setown(new CDaliWorkUnitFactory());
  3122. }
  3123. }
  3124. return factory.getLink();
  3125. }
  3126. extern WORKUNIT_API IWorkUnitFactory * getDaliWorkUnitFactory()
  3127. {
  3128. if (!factory)
  3129. {
  3130. CriticalBlock b(factoryCrit);
  3131. if (!factory) // NOTE - this "double test" paradigm is not guaranteed threadsafe on modern systems/compilers - I think in this instance that is harmless even in the (extremely) unlikely event that it resulted in the setown being called twice.
  3132. factory.setown(new CDaliWorkUnitFactory());
  3133. }
  3134. return factory.getLink();
  3135. }
  3136. // A SecureWorkUnitFactory allows the security params to be supplied once to the factory rather than being supplied to each call.
  3137. // They can still be supplied if you want...
  3138. class CSecureWorkUnitFactory : implements IWorkUnitFactory, public CInterface
  3139. {
  3140. public:
  3141. IMPLEMENT_IINTERFACE;
  3142. CSecureWorkUnitFactory(IWorkUnitFactory *_baseFactory, ISecManager *_secMgr, ISecUser *_secUser)
  3143. : baseFactory(_baseFactory), defaultSecMgr(_secMgr), defaultSecUser(_secUser)
  3144. {
  3145. }
  3146. virtual bool initializeStore()
  3147. {
  3148. throwUnexpected(); // Used when loading a plugin factory - not applicable here
  3149. }
  3150. virtual IWorkUnitWatcher *getWatcher(IWorkUnitSubscriber *subscriber, WUSubscribeOptions options, const char *wuid) const
  3151. {
  3152. return baseFactory->getWatcher(subscriber, options, wuid);
  3153. }
  3154. virtual unsigned validateRepository(bool fix)
  3155. {
  3156. return baseFactory->validateRepository(fix);
  3157. }
  3158. virtual void deleteRepository(bool recreate)
  3159. {
  3160. return baseFactory->deleteRepository(recreate);
  3161. }
  3162. virtual void createRepository()
  3163. {
  3164. return baseFactory->createRepository();
  3165. }
  3166. virtual const char *queryStoreType() const
  3167. {
  3168. return baseFactory->queryStoreType();
  3169. }
  3170. virtual StringArray &getUniqueValues(WUSortField field, const char *prefix, StringArray &result) const
  3171. {
  3172. return baseFactory->getUniqueValues(field, prefix, result);
  3173. }
  3174. virtual IWorkUnit* createNamedWorkUnit(const char *wuid, const char *app, const char *user, ISecManager *secMgr, ISecUser *secUser)
  3175. {
  3176. if (!secMgr) secMgr = defaultSecMgr.get();
  3177. if (!secUser) secUser = defaultSecUser.get();
  3178. return baseFactory->createNamedWorkUnit(wuid, app, user, secMgr, secUser);
  3179. }
  3180. virtual IWorkUnit* createWorkUnit(const char *app, const char *user, ISecManager *secMgr, ISecUser *secUser)
  3181. {
  3182. if (!secMgr) secMgr = defaultSecMgr.get();
  3183. if (!secUser) secUser = defaultSecUser.get();
  3184. return baseFactory->createWorkUnit(app, user, secMgr, secUser);
  3185. }
  3186. virtual bool deleteWorkUnit(const char * wuid, ISecManager *secMgr, ISecUser *secUser)
  3187. {
  3188. if (!secMgr) secMgr = defaultSecMgr.get();
  3189. if (!secUser) secUser = defaultSecUser.get();
  3190. return baseFactory->deleteWorkUnit(wuid, secMgr, secUser);
  3191. }
  3192. virtual IConstWorkUnit* openWorkUnit(const char *wuid, ISecManager *secMgr, ISecUser *secUser)
  3193. {
  3194. if (!secMgr) secMgr = defaultSecMgr.get();
  3195. if (!secUser) secUser = defaultSecUser.get();
  3196. return baseFactory->openWorkUnit(wuid, secMgr, secUser);
  3197. }
  3198. virtual IWorkUnit* updateWorkUnit(const char *wuid, ISecManager *secMgr, ISecUser *secUser)
  3199. {
  3200. if (!secMgr) secMgr = defaultSecMgr.get();
  3201. if (!secUser) secUser = defaultSecUser.get();
  3202. return baseFactory->updateWorkUnit(wuid, secMgr, secUser);
  3203. }
  3204. virtual bool restoreWorkUnit(const char *base, const char *wuid, bool restoreAssociated)
  3205. {
  3206. return baseFactory->restoreWorkUnit(base, wuid, restoreAssociated);
  3207. }
  3208. virtual IWorkUnit * getGlobalWorkUnit(ISecManager *secMgr, ISecUser *secUser)
  3209. {
  3210. if (!secMgr) secMgr = defaultSecMgr.get();
  3211. if (!secUser) secUser = defaultSecUser.get();
  3212. return baseFactory->getGlobalWorkUnit(secMgr, secUser);
  3213. }
  3214. virtual IConstWorkUnitIterator * getWorkUnitsByOwner(const char * owner, ISecManager *secMgr, ISecUser *secUser)
  3215. {
  3216. if (!secMgr) secMgr = defaultSecMgr.get();
  3217. if (!secUser) secUser = defaultSecUser.get();
  3218. return baseFactory->getWorkUnitsByOwner(owner, secMgr, secUser);
  3219. }
  3220. virtual IConstWorkUnitIterator * getScheduledWorkUnits(ISecManager *secMgr, ISecUser *secUser)
  3221. {
  3222. if (!secMgr) secMgr = defaultSecMgr.get();
  3223. if (!secUser) secUser = defaultSecUser.get();
  3224. return baseFactory->getScheduledWorkUnits(secMgr, secUser);
  3225. }
  3226. virtual void descheduleAllWorkUnits(ISecManager *secMgr, ISecUser *secUser)
  3227. {
  3228. if (!secMgr) secMgr = defaultSecMgr.get();
  3229. if (!secUser) secUser = defaultSecUser.get();
  3230. baseFactory->descheduleAllWorkUnits(secMgr, secUser);
  3231. }
  3232. virtual int setTracingLevel(int newLevel)
  3233. {
  3234. return baseFactory->setTracingLevel(newLevel);
  3235. }
  3236. virtual IConstWorkUnitIterator* getWorkUnitsSorted( WUSortField sortorder, // field to sort by
  3237. WUSortField *filters, // NULL or list of fields to filter on (terminated by WUSFterm)
  3238. const void *filterbuf, // (appended) string values for filters
  3239. unsigned startoffset,
  3240. unsigned maxnum,
  3241. __int64 *cachehint,
  3242. unsigned *total,
  3243. ISecManager *secMgr, ISecUser *secUser)
  3244. {
  3245. if (!secMgr) secMgr = defaultSecMgr.get();
  3246. if (!secUser) secUser = defaultSecUser.get();
  3247. return baseFactory->getWorkUnitsSorted(sortorder,filters,filterbuf,startoffset,maxnum,cachehint, total, secMgr, secUser);
  3248. }
  3249. virtual IConstQuerySetQueryIterator* getQuerySetQueriesSorted( WUQuerySortField *sortorder,
  3250. WUQuerySortField *filters,
  3251. const void *filterbuf,
  3252. unsigned startoffset,
  3253. unsigned maxnum,
  3254. __int64 *cachehint,
  3255. unsigned *total,
  3256. const MapStringTo<bool> *subset)
  3257. {
  3258. // MORE - why no security?
  3259. return baseFactory->getQuerySetQueriesSorted(sortorder,filters,filterbuf,startoffset,maxnum,cachehint,total,subset);
  3260. }
  3261. virtual unsigned numWorkUnits()
  3262. {
  3263. return baseFactory->numWorkUnits();
  3264. }
  3265. virtual bool isAborting(const char *wuid) const
  3266. {
  3267. return baseFactory->isAborting(wuid);
  3268. }
  3269. virtual void clearAborting(const char *wuid)
  3270. {
  3271. baseFactory->clearAborting(wuid);
  3272. }
  3273. virtual WUState waitForWorkUnit(const char * wuid, unsigned timeout, bool compiled, bool returnOnWaitState)
  3274. {
  3275. return baseFactory->waitForWorkUnit(wuid, timeout, compiled, returnOnWaitState);
  3276. }
  3277. virtual WUAction waitForWorkUnitAction(const char * wuid, WUAction original)
  3278. {
  3279. return baseFactory->waitForWorkUnitAction(wuid, original);
  3280. }
  3281. private:
  3282. Owned<IWorkUnitFactory> baseFactory;
  3283. Linked<ISecManager> defaultSecMgr;
  3284. Linked<ISecUser> defaultSecUser;
  3285. };
  3286. extern WORKUNIT_API IWorkUnitFactory * getWorkUnitFactory(ISecManager *secmgr, ISecUser *secuser)
  3287. {
  3288. if (secmgr && secuser)
  3289. return new CSecureWorkUnitFactory(getWorkUnitFactory(), secmgr, secuser);
  3290. else
  3291. return getWorkUnitFactory();
  3292. }
  3293. //==========================================================================================
  3294. class CStringPTreeIterator : implements IStringIterator, public CInterface
  3295. {
  3296. Owned<IPropertyTreeIterator> it;
  3297. public:
  3298. IMPLEMENT_IINTERFACE;
  3299. CStringPTreeIterator(IPropertyTreeIterator *p) : it(p) {};
  3300. virtual bool first() { return it->first(); }
  3301. virtual bool next() { return it->next(); }
  3302. virtual bool isValid() { return it->isValid(); }
  3303. virtual IStringVal & str(IStringVal &s) { s.set(it->query().queryProp(NULL)); return s; }
  3304. };
  3305. class CStringPTreeTagIterator : implements IStringIterator, public CInterface
  3306. {
  3307. Owned<IPropertyTreeIterator> it;
  3308. public:
  3309. IMPLEMENT_IINTERFACE;
  3310. CStringPTreeTagIterator(IPropertyTreeIterator *p) : it(p) {};
  3311. virtual bool first() { return it->first(); }
  3312. virtual bool next() { return it->next(); }
  3313. virtual bool isValid() { return it->isValid(); }
  3314. virtual IStringVal & str(IStringVal &s) { s.set(it->query().queryName()); return s; }
  3315. };
  3316. class CStringPTreeAttrIterator : implements IStringIterator, public CInterface
  3317. {
  3318. Owned<IPropertyTreeIterator> it;
  3319. StringAttr name;
  3320. public:
  3321. IMPLEMENT_IINTERFACE;
  3322. CStringPTreeAttrIterator(IPropertyTreeIterator *p, const char *_name) : it(p), name(_name) {};
  3323. virtual bool first() { return it->first(); }
  3324. virtual bool next() { return it->next(); }
  3325. virtual bool isValid() { return it->isValid(); }
  3326. virtual IStringVal & str(IStringVal &s) { s.set(it->query().queryProp(name)); return s; }
  3327. };
  3328. //==========================================================================================
  3329. CLocalWorkUnit::CLocalWorkUnit(ISecManager *secmgr, ISecUser *secuser)
  3330. {
  3331. clearCached(false);
  3332. secMgr.set(secmgr);
  3333. secUser.set(secuser);
  3334. workflowIteratorCached = false;
  3335. resultsCached = false;
  3336. graphsCached = false;
  3337. temporariesCached = false;
  3338. variablesCached = false;
  3339. exceptionsCached = false;
  3340. pluginsCached = false;
  3341. librariesCached = false;
  3342. activitiesCached = false;
  3343. webServicesInfoCached = false;
  3344. roxieQueryInfoCached = false;
  3345. }
  3346. void CLocalWorkUnit::clearCached(bool clearTree)
  3347. {
  3348. query.clear();
  3349. webServicesInfo.clear();
  3350. workflowIterator.clear();
  3351. graphs.kill();
  3352. results.kill();
  3353. variables.kill();
  3354. plugins.kill();
  3355. libraries.kill();
  3356. exceptions.kill();
  3357. temporaries.kill();
  3358. statistics.kill();
  3359. appvalues.kill();
  3360. if (clearTree)
  3361. p.clear();
  3362. workflowIteratorCached = false;
  3363. resultsCached = false;
  3364. graphsCached = false;
  3365. temporariesCached = false;
  3366. variablesCached = false;
  3367. exceptionsCached = false;
  3368. pluginsCached = false;
  3369. librariesCached = false;
  3370. activitiesCached = false;
  3371. webServicesInfoCached = false;
  3372. roxieQueryInfoCached = false;
  3373. }
  3374. void CLocalWorkUnit::loadPTree(IPropertyTree *ptree)
  3375. {
  3376. clearCached(false);
  3377. p.setown(ptree);
  3378. }
  3379. void CLocalWorkUnit::beforeDispose()
  3380. {
  3381. try
  3382. {
  3383. unsubscribe();
  3384. clearCached(true);
  3385. userDesc.clear();
  3386. secMgr.clear();
  3387. secUser.clear();
  3388. }
  3389. catch (IException *E) { LOG(MCexception(E, MSGCLS_warning), E, "Exception during ~CLocalWorkUnit"); E->Release(); }
  3390. }
  3391. void CLocalWorkUnit::cleanupAndDelete(bool deldll, bool deleteOwned, const StringArray *deleteExclusions)
  3392. {
  3393. MTIME_SECTION(queryActiveTimer(), "WUDELETE cleanupAndDelete total");
  3394. // Delete any related things in SDS etc that might otherwise be forgotten
  3395. if (p->getPropBool("@protected", false))
  3396. throw MakeStringException(WUERR_WorkunitProtected, "%s: Workunit is protected",p->queryName());
  3397. switch (getState())
  3398. {
  3399. case WUStateAborted:
  3400. case WUStateCompleted:
  3401. case WUStateFailed:
  3402. case WUStateArchived:
  3403. break;
  3404. case WUStateCompiled:
  3405. if (getAction()==WUActionRun || getAction()==WUActionUnknown)
  3406. throw MakeStringException(WUERR_WorkunitActive, "%s: Workunit is active. Please abort before deleting this workunit.",p->queryName());
  3407. break;
  3408. case WUStateWait:
  3409. throw MakeStringException(WUERR_WorkunitScheduled, "%s: Workunit is scheduled",p->queryName());
  3410. default:
  3411. throw MakeStringException(WUERR_WorkunitActive, "%s: Workunit is active. Please abort before deleting this workunit.",p->queryName());
  3412. break;
  3413. }
  3414. if (getIsQueryService())
  3415. {
  3416. Owned<IPropertyTree> registry = getQueryRegistryRoot();
  3417. if (registry)
  3418. {
  3419. VStringBuffer xpath("QuerySet/Query[@wuid='%s']", p->queryName());
  3420. if (registry->hasProp(xpath.str()))
  3421. throw MakeStringException(WUERR_WorkunitPublished, "%s: Workunit is published",p->queryName());
  3422. }
  3423. }
  3424. try
  3425. {
  3426. if (deldll && !p->getPropBool("@isClone", false))
  3427. {
  3428. Owned<IConstWUQuery> q = getQuery();
  3429. if (q)
  3430. {
  3431. Owned<IConstWUAssociatedFileIterator> iter = &q->getAssociatedFiles();
  3432. SCMStringBuffer name;
  3433. ForEach(*iter)
  3434. {
  3435. IConstWUAssociatedFile & cur = iter->query();
  3436. cur.getNameTail(name);
  3437. if (!deleteExclusions || (NotFound == deleteExclusions->find(name.str())))
  3438. {
  3439. Owned<IDllEntry> entry = queryDllServer().getEntry(name.str());
  3440. if (entry.get())
  3441. asyncRemoveDll(name.str());
  3442. else
  3443. {
  3444. SCMStringBuffer ip, localPath;
  3445. cur.getName(localPath);
  3446. cur.getIp(ip);
  3447. asyncRemoveFile(ip.str(), localPath.str());
  3448. }
  3449. }
  3450. }
  3451. }
  3452. }
  3453. factory->clearAborting(queryWuid());
  3454. deleteTempFiles(NULL, deleteOwned, true); // all, any remaining.
  3455. }
  3456. catch(IException *E)
  3457. {
  3458. StringBuffer s;
  3459. LOG(MCexception(E, MSGCLS_warning), E, s.append("Exception during cleanupAndDelete: ").append(p->queryName()).str());
  3460. E->Release();
  3461. }
  3462. catch (...)
  3463. {
  3464. WARNLOG("Unknown exception during cleanupAndDelete: %s", p->queryName());
  3465. }
  3466. }
  3467. void CLocalWorkUnit::setTimeScheduled(const IJlibDateTime &val)
  3468. {
  3469. SCMStringBuffer strval;
  3470. val.getGmtString(strval);
  3471. p->setProp("@timescheduled",strval.str());
  3472. }
  3473. IJlibDateTime & CLocalWorkUnit::getTimeScheduled(IJlibDateTime &val) const
  3474. {
  3475. StringBuffer str;
  3476. p->getProp("@timescheduled",str);
  3477. if(str.length())
  3478. val.setGmtString(str.str());
  3479. return val;
  3480. }
  3481. bool modifyAndWriteWorkUnitXML(char const * wuid, StringBuffer & buf, StringBuffer & extra, IFileIO * fileio)
  3482. {
  3483. // kludge in extra chunks of XML such as GraphProgress and GeneratedDlls
  3484. if(extra.length())
  3485. {
  3486. size32_t l = (size32_t)strlen(wuid);
  3487. size32_t p = buf.length()-l-4; // bit of a kludge
  3488. assertex(memcmp(buf.str()+p+2,wuid,l)==0);
  3489. StringAttr tail(buf.str()+p);
  3490. buf.setLength(p);
  3491. buf.append(extra);
  3492. buf.append(tail);
  3493. }
  3494. return (fileio->write(0,buf.length(),buf.str()) == buf.length());
  3495. }
  3496. bool CLocalWorkUnit::archiveWorkUnit(const char *base, bool del, bool ignoredllerrors, bool deleteOwned, bool exportAssociatedFiles)
  3497. {
  3498. CriticalBlock block(crit);
  3499. StringBuffer path(base);
  3500. if (!p)
  3501. return false;
  3502. const char *wuid = p->queryName();
  3503. if (!wuid||!*wuid)
  3504. return false;
  3505. addPathSepChar(path).append(wuid).append(".xml");
  3506. Owned<IFile> file = createIFile(path.str());
  3507. if (!file)
  3508. return false;
  3509. Owned<IFileIO> fileio = file->open(IFOcreate);
  3510. if (!fileio)
  3511. return false;
  3512. StringBuffer buf;
  3513. exportWorkUnitToXML(this, buf, false, false, true);
  3514. StringBuffer extraWorkUnitXML;
  3515. Owned<IPTree> graphProgress = getGraphProgressTree();
  3516. if (graphProgress)
  3517. {
  3518. toXML(graphProgress,extraWorkUnitXML,1,XML_Format);
  3519. graphProgress.clear();
  3520. }
  3521. Owned<IConstWUQuery> q = getQuery();
  3522. if (!q)
  3523. {
  3524. if (!modifyAndWriteWorkUnitXML(wuid, buf, extraWorkUnitXML, fileio))
  3525. return false;
  3526. if (del)
  3527. {
  3528. if (getState()==WUStateUnknown)
  3529. setState(WUStateArchived); // to allow delete
  3530. cleanupAndDelete(false,deleteOwned); // no query, may as well delete
  3531. }
  3532. return false;
  3533. }
  3534. StringArray deleteExclusions; // associated files not to delete, added if failure to copy
  3535. Owned<IConstWUAssociatedFileIterator> iter = &q->getAssociatedFiles();
  3536. Owned<IPropertyTree> generatedDlls = createPTree("GeneratedDlls");
  3537. ForEach(*iter)
  3538. {
  3539. IConstWUAssociatedFile & cur = iter->query();
  3540. SCMStringBuffer name;
  3541. cur.getNameTail(name);
  3542. if (name.length())
  3543. {
  3544. Owned<IDllEntry> entry = queryDllServer().getEntry(name.str());
  3545. SCMStringBuffer curPath, curIp;
  3546. cur.getName(curPath);
  3547. cur.getIp(curIp);
  3548. SocketEndpoint curEp(curIp.str());
  3549. RemoteFilename curRfn;
  3550. curRfn.setPath(curEp, curPath.str());
  3551. StringBuffer dst(base);
  3552. addPathSepChar(dst);
  3553. curRfn.getTail(dst);
  3554. Owned<IFile> dstFile = createIFile(dst.str());
  3555. if (entry.get())
  3556. {
  3557. Owned<IException> exception;
  3558. Owned<IDllLocation> loc;
  3559. Owned<IPropertyTree> generatedDllBranch = createPTree();
  3560. generatedDllBranch->setProp("@name", entry->queryName());
  3561. generatedDllBranch->setProp("@kind", entry->queryKind());
  3562. if (exportAssociatedFiles)
  3563. {
  3564. try
  3565. {
  3566. loc.setown(entry->getBestLocation()); //throws exception if no readable locations
  3567. }
  3568. catch(IException * e)
  3569. {
  3570. exception.setown(e);
  3571. loc.setown(entry->getBestLocationCandidate()); //this will be closest of the unreadable locations
  3572. }
  3573. RemoteFilename filename;
  3574. loc->getDllFilename(filename);
  3575. if (!exception)
  3576. {
  3577. Owned<IFile> srcfile = createIFile(filename);
  3578. try
  3579. {
  3580. if (dstFile->exists())
  3581. {
  3582. if (streq(srcfile->queryFilename(), dstFile->queryFilename()))
  3583. deleteExclusions.append(name.str()); // restored workunit, referencing archive location for query dll (no longer true post HPCC-11191 fix)
  3584. // still want to delete if already archived but there are source file copies
  3585. }
  3586. else
  3587. copyFile(dstFile, srcfile);
  3588. }
  3589. catch(IException * e)
  3590. {
  3591. exception.setown(e);
  3592. }
  3593. }
  3594. if (exception)
  3595. {
  3596. if (ignoredllerrors)
  3597. {
  3598. EXCLOG(exception.get(), "archiveWorkUnit (copying associated file)");
  3599. //copy failed, so don't delete the registered dll files
  3600. deleteExclusions.append(name.str());
  3601. }
  3602. else
  3603. throw exception.getClear();
  3604. }
  3605. }
  3606. // Record Associated path to restore back to
  3607. StringBuffer restorePath;
  3608. curRfn.getRemotePath(restorePath);
  3609. generatedDllBranch->setProp("@location", restorePath.str());
  3610. generatedDlls->addPropTree("GeneratedDll", generatedDllBranch.getClear());
  3611. }
  3612. else if (exportAssociatedFiles) // no generated dll entry
  3613. {
  3614. Owned<IFile> srcFile = createIFile(curRfn);
  3615. try
  3616. {
  3617. copyFile(dstFile, srcFile);
  3618. }
  3619. catch (IException *e)
  3620. {
  3621. VStringBuffer msg("Failed to archive associated file '%s' to destination '%s'", srcFile->queryFilename(), dstFile->queryFilename());
  3622. EXCLOG(e, msg.str());
  3623. e->Release();
  3624. deleteExclusions.append(name.str());
  3625. }
  3626. }
  3627. }
  3628. }
  3629. iter.clear();
  3630. if (generatedDlls->numChildren())
  3631. toXML(generatedDlls, extraWorkUnitXML, 1, XML_Format);
  3632. if (!modifyAndWriteWorkUnitXML(wuid, buf, extraWorkUnitXML, fileio))
  3633. return false;
  3634. if (del)
  3635. {
  3636. //setState(WUStateArchived); // this isn't useful as about to delete it!
  3637. q.clear();
  3638. cleanupAndDelete(true, deleteOwned, &deleteExclusions);
  3639. }
  3640. return true;
  3641. }
  3642. void CLocalWorkUnit::loadXML(const char *xml)
  3643. {
  3644. CriticalBlock block(crit);
  3645. clearCached(true);
  3646. assertex(xml);
  3647. p.setown(createPTreeFromXMLString(xml,ipt_lowmem));
  3648. }
  3649. void CLocalWorkUnit::serialize(MemoryBuffer &tgt)
  3650. {
  3651. CriticalBlock block(crit);
  3652. StringBuffer x;
  3653. tgt.append(exportWorkUnitToXML(this, x, false, false, false).str());
  3654. }
  3655. void CLocalWorkUnit::deserialize(MemoryBuffer &src)
  3656. {
  3657. CriticalBlock block(crit);
  3658. StringAttr value;
  3659. src.read(value);
  3660. loadXML(value);
  3661. }
  3662. void CLocalWorkUnit::requestAbort()
  3663. {
  3664. CriticalBlock block(crit);
  3665. abortWorkUnit(p->queryName());
  3666. }
  3667. void CLocalWorkUnit::unlockRemote()
  3668. {
  3669. CriticalBlock block(crit);
  3670. locked.unlock();
  3671. _unlockRemote();
  3672. }
  3673. IWorkUnit &CLocalWorkUnit::lockRemote(bool commit)
  3674. {
  3675. if (secMgr)
  3676. checkWuSecAccess(*this, secMgr.get(), secUser.get(), SecAccess_Write, "write lock", true, true);
  3677. locked.lock();
  3678. CriticalBlock block(crit);
  3679. if (commit)
  3680. {
  3681. try
  3682. {
  3683. _lockRemote();
  3684. }
  3685. catch (IException *E)
  3686. {
  3687. StringBuffer s;
  3688. PrintLog("Failed to get write lock on workunit: %s", E->errorMessage(s).str());
  3689. locked.unlock();
  3690. throw;
  3691. }
  3692. }
  3693. return *new CLockedWorkUnit(LINK(this));
  3694. }
  3695. void CLocalWorkUnit::commit()
  3696. {
  3697. // Nothing to do if not backed by a persistent store
  3698. }
  3699. IWorkUnit& CLocalWorkUnit::lock()
  3700. {
  3701. return lockRemote(true);
  3702. }
  3703. const char *CLocalWorkUnit::queryWuid() const
  3704. {
  3705. CriticalBlock block(crit);
  3706. return p->queryName();
  3707. }
  3708. unsigned CLocalWorkUnit::getDebugAgentListenerPort() const
  3709. {
  3710. CriticalBlock block(crit);
  3711. return p->getPropInt("@DebugListenerPort", 0);
  3712. }
  3713. unsigned CLocalWorkUnit::getTotalThorTime() const
  3714. {
  3715. CriticalBlock block(crit);
  3716. return (unsigned)nanoToMilli(extractTimeCollatable(p->queryProp("@totalThorTime"), false));
  3717. }
  3718. void CLocalWorkUnit::setDebugAgentListenerPort(unsigned port)
  3719. {
  3720. CriticalBlock block(crit);
  3721. p->setPropInt("@DebugListenerPort", port);
  3722. }
  3723. IStringVal& CLocalWorkUnit::getDebugAgentListenerIP(IStringVal &ip) const
  3724. {
  3725. CriticalBlock block(crit);
  3726. ip.set(p->queryProp("@DebugListenerIP"));
  3727. return ip;
  3728. }
  3729. void CLocalWorkUnit::setDebugAgentListenerIP(const char * ip)
  3730. {
  3731. CriticalBlock block(crit);
  3732. p->setProp("@DebugListenerIP", ip);
  3733. }
  3734. IStringVal& CLocalWorkUnit::getSecurityToken(IStringVal &str) const
  3735. {
  3736. CriticalBlock block(crit);
  3737. str.set(p->queryProp("@token"));
  3738. return str;
  3739. }
  3740. void CLocalWorkUnit::setSecurityToken(const char *value)
  3741. {
  3742. CriticalBlock block(crit);
  3743. p->setProp("@token", value);
  3744. }
  3745. bool CLocalWorkUnit::getRunningGraph(IStringVal &graphName, WUGraphIDType &subId) const
  3746. {
  3747. // Only implemented in derived classes
  3748. return false;
  3749. }
  3750. void CLocalWorkUnit::setJobName(const char *value)
  3751. {
  3752. CriticalBlock block(crit);
  3753. p->setProp("@jobName", value);
  3754. }
  3755. const char *CLocalWorkUnit::queryJobName() const
  3756. {
  3757. CriticalBlock block(crit);
  3758. const char *ret = p->queryProp("@jobName");
  3759. if (!ret)
  3760. ret = "";
  3761. return ret;
  3762. }
  3763. void CLocalWorkUnit::setClusterName(const char *value)
  3764. {
  3765. CriticalBlock block(crit);
  3766. p->setProp("@clusterName", value);
  3767. }
  3768. const char *CLocalWorkUnit::queryClusterName() const
  3769. {
  3770. CriticalBlock block(crit);
  3771. const char *ret = p->queryProp("@clusterName");
  3772. if (!ret)
  3773. ret = "";
  3774. return ret;
  3775. }
  3776. void CLocalWorkUnit::setAllowedClusters(const char *value)
  3777. {
  3778. setDebugValue("allowedclusters",value, true);
  3779. }
  3780. IStringVal& CLocalWorkUnit::getAllowedClusters(IStringVal &str) const
  3781. {
  3782. CriticalBlock block(crit);
  3783. getDebugValue("allowedclusters",str);
  3784. if (str.length()!=0)
  3785. return str;
  3786. str.set(p->queryProp("@clusterName"));
  3787. return str;
  3788. }
  3789. void CLocalWorkUnit::setAllowAutoQueueSwitch(bool val)
  3790. {
  3791. setDebugValueInt("allowautoqueueswitch",val?1:0,true);
  3792. }
  3793. bool CLocalWorkUnit::getAllowAutoQueueSwitch() const
  3794. {
  3795. CriticalBlock block(crit);
  3796. return getDebugValueBool("allowautoqueueswitch",false);
  3797. }
  3798. void CLocalWorkUnit::setLibraryInformation(const char * name, unsigned interfaceHash, unsigned definitionHash)
  3799. {
  3800. StringBuffer suffix;
  3801. if (name && *name)
  3802. setApplicationValue("LibraryModule", "name", name, true);
  3803. setApplicationValueInt("LibraryModule", "interfaceHash", interfaceHash, true);
  3804. setApplicationValueInt("LibraryModule", "definitionHash", definitionHash, true);
  3805. setApplicationValue("LibraryModule", "platform", appendLibrarySuffix(suffix).str(), true);
  3806. }
  3807. void CLocalWorkUnit::remoteCheckAccess(IUserDescriptor *user, bool writeaccess) const
  3808. {
  3809. unsigned auditflags = DALI_LDAP_AUDIT_REPORT|DALI_LDAP_READ_WANTED;
  3810. if (writeaccess)
  3811. auditflags |= DALI_LDAP_WRITE_WANTED;
  3812. SecAccessFlags perm = SecAccess_Full;
  3813. const char *scopename = p->queryProp("@scope");
  3814. if (scopename&&*scopename) {
  3815. if (!user)
  3816. user = queryUserDescriptor();
  3817. perm = querySessionManager().getPermissionsLDAP("workunit",scopename,user,auditflags);
  3818. if (perm<0) {
  3819. if (perm == SecAccess_Unavailable)
  3820. perm = SecAccess_Full;
  3821. else
  3822. perm = SecAccess_None;
  3823. }
  3824. }
  3825. if (!HASREADPERMISSION(perm))
  3826. throw MakeStringException(WUERR_WorkunitAccessDenied, "Read access denied for workunit %s", queryWuid());
  3827. if (writeaccess && !HASWRITEPERMISSION(perm))
  3828. throw MakeStringException(WUERR_WorkunitAccessDenied, "Write access denied for workunit %s", queryWuid());
  3829. }
  3830. void CLocalWorkUnit::setUser(const char * value)
  3831. {
  3832. CriticalBlock block(crit);
  3833. p->setProp("@submitID", value);
  3834. }
  3835. const char *CLocalWorkUnit::queryUser() const
  3836. {
  3837. CriticalBlock block(crit);
  3838. const char *ret = p->queryProp("@submitID");
  3839. if (!ret)
  3840. ret = "";
  3841. return ret;
  3842. }
  3843. void CLocalWorkUnit::setWuScope(const char * value)
  3844. {
  3845. if (value && *value)
  3846. {
  3847. if (checkWuScopeSecAccess(value, secMgr.get(), secUser.get(), SecAccess_Write, "Change Scope", true, true))
  3848. {
  3849. CriticalBlock block(crit);
  3850. p->setProp("@scope", value);
  3851. }
  3852. }
  3853. }
  3854. const char *CLocalWorkUnit::queryWuScope() const
  3855. {
  3856. CriticalBlock block(crit);
  3857. const char *ret = p->queryProp("@scope");
  3858. if (!ret)
  3859. ret = "";
  3860. return ret;
  3861. }
  3862. void CLocalWorkUnit::setPriority(WUPriorityClass cls)
  3863. {
  3864. CriticalBlock block(crit);
  3865. setEnum(p, "@priorityClass", cls, priorityClasses);
  3866. }
  3867. WUPriorityClass CLocalWorkUnit::getPriority() const
  3868. {
  3869. CriticalBlock block(crit);
  3870. return (WUPriorityClass) getEnum(p, "@priorityClass", priorityClasses);
  3871. }
  3872. const char *CLocalWorkUnit::queryPriorityDesc() const
  3873. {
  3874. return getEnumText(getPriority(), priorityClasses);
  3875. }
  3876. void CLocalWorkUnit::setState(WUState value)
  3877. {
  3878. if (value==WUStateAborted || value==WUStatePaused || value==WUStateCompleted || value==WUStateFailed || value==WUStateSubmitted || value==WUStateWait)
  3879. {
  3880. if (factory)
  3881. factory->clearAborting(queryWuid());
  3882. }
  3883. CriticalBlock block(crit);
  3884. setEnum(p, "@state", value, states); // For historical reasons, we use state to store the state
  3885. setEnum(p, "State", value, states); // But we can only subscribe to elements, not attributes
  3886. if (getDebugValueBool("monitorWorkunit", false))
  3887. {
  3888. switch(value)
  3889. {
  3890. case WUStateAborted:
  3891. FLLOG(MCoperatorWarning, "Workunit %s aborted", p->queryName());
  3892. break;
  3893. case WUStateCompleted:
  3894. FLLOG(MCoperatorProgress, "Workunit %s completed", p->queryName());
  3895. break;
  3896. case WUStateFailed:
  3897. FLLOG(MCoperatorProgress, "Workunit %s failed", p->queryName());
  3898. break;
  3899. }
  3900. }
  3901. p->removeProp("@stateEx");
  3902. }
  3903. void CLocalWorkUnit::setStateEx(const char * text)
  3904. {
  3905. CriticalBlock block(crit);
  3906. p->setProp("@stateEx", text);
  3907. }
  3908. void CLocalWorkUnit::setAgentSession(__int64 sessionId)
  3909. {
  3910. CriticalBlock block(crit);
  3911. p->setPropInt64("@agentSession", sessionId);
  3912. }
  3913. bool CLocalWorkUnit::getIsQueryService() const
  3914. {
  3915. CriticalBlock block(crit);
  3916. return p->getPropBool("@isQueryService", false);
  3917. }
  3918. void CLocalWorkUnit::setIsQueryService(bool value)
  3919. {
  3920. CriticalBlock block(crit);
  3921. p->setPropBool("@isQueryService", value);
  3922. }
  3923. void CLocalWorkUnit::checkAgentRunning(WUState & state)
  3924. {
  3925. if (queryDaliServerVersion().compare("2.1")<0)
  3926. return;
  3927. switch(state)
  3928. {
  3929. case WUStateRunning:
  3930. case WUStateDebugPaused:
  3931. case WUStateDebugRunning:
  3932. case WUStateBlocked:
  3933. case WUStateAborting:
  3934. case WUStateCompiling:
  3935. case WUStatePaused:
  3936. {
  3937. SessionId agent = getAgentSession();
  3938. if((agent>0) && querySessionManager().sessionStopped(agent, 0))
  3939. {
  3940. forceReload();
  3941. state = (WUState) getEnum(p, "@state", states);
  3942. bool isecl=state==WUStateCompiling;
  3943. if (aborting())
  3944. state = WUStateAborted;
  3945. else if (state==WUStateRunning || state==WUStatePaused || state==WUStateDebugPaused || state==WUStateDebugRunning || state==WUStateBlocked || state==WUStateCompiling)
  3946. state = WUStateFailed;
  3947. else
  3948. return;
  3949. WARNLOG("checkAgentRunning terminated: %" I64F "d state = %d",(__int64)agent,(int)state);
  3950. Owned<IWorkUnit> w = &lock();
  3951. w->setState(state);
  3952. Owned<IWUException> e = w->createException();
  3953. WUAction action = w->getAction();
  3954. switch (action)
  3955. {
  3956. case WUActionPause:
  3957. case WUActionPauseNow:
  3958. case WUActionResume:
  3959. w->setAction(WUActionUnknown);
  3960. }
  3961. if(isecl)
  3962. {
  3963. e->setExceptionCode(1001);
  3964. e->setExceptionMessage("EclServer terminated unexpectedly");
  3965. }
  3966. else
  3967. {
  3968. e->setExceptionCode(1000);
  3969. e->setExceptionMessage("Workunit terminated unexpectedly");
  3970. }
  3971. }
  3972. }
  3973. }
  3974. }
  3975. WUState CLocalWorkUnit::getState() const
  3976. {
  3977. CriticalBlock block(crit);
  3978. WUState state = (WUState) getEnum(p, "@state", states);
  3979. switch (state)
  3980. {
  3981. case WUStateRunning:
  3982. case WUStateDebugPaused:
  3983. case WUStateDebugRunning:
  3984. case WUStateBlocked:
  3985. case WUStateCompiling:
  3986. if (aborting())
  3987. state = WUStateAborting;
  3988. break;
  3989. case WUStateSubmitted:
  3990. if (aborting())
  3991. state = WUStateAborted;
  3992. break;
  3993. }
  3994. const_cast<CLocalWorkUnit *>(this)->checkAgentRunning(state); //need const_cast as will change state if agent has died
  3995. return state;
  3996. }
  3997. IStringVal& CLocalWorkUnit::getStateEx(IStringVal & str) const
  3998. {
  3999. CriticalBlock block(crit);
  4000. str.set(p->queryProp("@stateEx"));
  4001. return str;
  4002. }
  4003. __int64 CLocalWorkUnit::getAgentSession() const
  4004. {
  4005. CriticalBlock block(crit);
  4006. return p->getPropInt64("@agentSession", -1);
  4007. }
  4008. unsigned CLocalWorkUnit::getAgentPID() const
  4009. {
  4010. CriticalBlock block(crit);
  4011. return p->getPropInt("@agentPID", -1);
  4012. }
  4013. const char * CLocalWorkUnit::queryStateDesc() const
  4014. {
  4015. // MORE - not sure about this - may prefer a separate interface
  4016. CriticalBlock block(crit);
  4017. try
  4018. {
  4019. return getEnumText(getState(), states);
  4020. }
  4021. catch (...)
  4022. {
  4023. return "???";
  4024. }
  4025. }
  4026. void CLocalWorkUnit::setAction(WUAction value)
  4027. {
  4028. CriticalBlock block(crit);
  4029. setEnum(p, "Action", value, actions);
  4030. }
  4031. WUAction CLocalWorkUnit::getAction() const
  4032. {
  4033. CriticalBlock block(crit);
  4034. return (WUAction) getEnum(p, "Action", actions);
  4035. }
  4036. const char *CLocalWorkUnit::queryActionDesc() const
  4037. {
  4038. CriticalBlock block(crit);
  4039. return p->queryProp("Action");
  4040. }
  4041. IStringVal& CLocalWorkUnit::getApplicationValue(const char *app, const char *propname, IStringVal &str) const
  4042. {
  4043. CriticalBlock block(crit);
  4044. StringBuffer prop("Application/");
  4045. prop.append(app).append('/').append(propname);
  4046. str.set(p->queryProp(prop.str()));
  4047. return str;
  4048. }
  4049. int CLocalWorkUnit::getApplicationValueInt(const char *app, const char *propname, int defVal) const
  4050. {
  4051. CriticalBlock block(crit);
  4052. StringBuffer prop("Application/");
  4053. prop.append(app).append('/').append(propname);
  4054. return p->getPropInt(prop.str(), defVal);
  4055. }
  4056. IConstWUAppValueIterator& CLocalWorkUnit::getApplicationValues() const
  4057. {
  4058. CriticalBlock block(crit);
  4059. appvalues.load(p,"Application/*");
  4060. return *new CArrayIteratorOf<IConstWUAppValue,IConstWUAppValueIterator> (appvalues, 0, (IConstWorkUnit *) this);
  4061. }
  4062. void CLocalWorkUnit::setApplicationValue(const char *app, const char *propname, const char *value, bool overwrite)
  4063. {
  4064. CriticalBlock block(crit);
  4065. StringBuffer prop("Application/");
  4066. prop.append(app).append('/').append(propname);
  4067. if (overwrite || !p->hasProp(prop.str()))
  4068. {
  4069. StringBuffer sp;
  4070. p->setProp(sp.append("Application").str(), "");
  4071. p->setProp(sp.append('/').append(app).str(), "");
  4072. p->setProp(prop.str(), value);
  4073. }
  4074. }
  4075. void CLocalWorkUnit::setApplicationValueInt(const char *app, const char *propname, int value, bool overwrite)
  4076. {
  4077. VStringBuffer s("%d", value);
  4078. setApplicationValue(app, propname, s, overwrite);
  4079. }
  4080. void CLocalWorkUnit::setPriorityLevel(int level)
  4081. {
  4082. CriticalBlock block(crit);
  4083. p->setPropInt("PriorityFlag", level);
  4084. }
  4085. int CLocalWorkUnit::getPriorityLevel() const
  4086. {
  4087. CriticalBlock block(crit);
  4088. return p->getPropInt("PriorityFlag");
  4089. }
  4090. int calcPriorityValue(const IPropertyTree * p)
  4091. {
  4092. int priority = p->getPropInt("PriorityFlag");
  4093. switch((WUPriorityClass) getEnum(p, "@priorityClass", priorityClasses))
  4094. {
  4095. case PriorityClassLow:
  4096. priority -= 100;
  4097. break;
  4098. case PriorityClassHigh:
  4099. priority += 100;
  4100. break;
  4101. }
  4102. return priority;
  4103. }
  4104. int CLocalWorkUnit::getPriorityValue() const
  4105. {
  4106. CriticalBlock block(crit);
  4107. return calcPriorityValue(p);
  4108. }
  4109. void CLocalWorkUnit::setRescheduleFlag(bool value)
  4110. {
  4111. CriticalBlock block(crit);
  4112. p->setPropInt("RescheduleFlag", (int) value);
  4113. }
  4114. bool CLocalWorkUnit::getRescheduleFlag() const
  4115. {
  4116. CriticalBlock block(crit);
  4117. return p->getPropInt("RescheduleFlag") != 0;
  4118. }
  4119. class NullIStringIterator : implements IStringIterator, public CInterface
  4120. {
  4121. public:
  4122. IMPLEMENT_IINTERFACE;
  4123. bool first() { return false; }
  4124. bool next() { return false; }
  4125. bool isValid() { return false; }
  4126. IStringVal & str(IStringVal & str) { return str; }
  4127. };
  4128. ClusterType getClusterType(const char * platform, ClusterType dft)
  4129. {
  4130. if (stricmp(platform, "thor") == 0)
  4131. return ThorLCRCluster;
  4132. if (stricmp(platform, "thorlcr") == 0)
  4133. return ThorLCRCluster;
  4134. if (stricmp(platform, "hthor") == 0)
  4135. return HThorCluster;
  4136. if (stricmp(platform, "roxie") == 0)
  4137. return RoxieCluster;
  4138. return dft;
  4139. }
  4140. const char *clusterTypeString(ClusterType clusterType, bool lcrSensitive)
  4141. {
  4142. switch (clusterType)
  4143. {
  4144. case ThorLCRCluster:
  4145. if (lcrSensitive)
  4146. return "thorlcr";
  4147. return "thor";
  4148. case RoxieCluster:
  4149. return "roxie";
  4150. case HThorCluster:
  4151. return "hthor";
  4152. }
  4153. throwUnexpected();
  4154. }
  4155. IPropertyTree *queryRoxieProcessTree(IPropertyTree *environment, const char *process)
  4156. {
  4157. if (!process || !*process)
  4158. return NULL;
  4159. VStringBuffer xpath("Software/RoxieCluster[@name=\"%s\"]", process);
  4160. return environment->queryPropTree(xpath.str());
  4161. }
  4162. void getRoxieProcessServers(IPropertyTree *roxie, SocketEndpointArray &endpoints)
  4163. {
  4164. if (!roxie)
  4165. return;
  4166. Owned<IPropertyTreeIterator> servers = roxie->getElements("RoxieServerProcess");
  4167. ForEach(*servers)
  4168. {
  4169. IPropertyTree &server = servers->query();
  4170. const char *netAddress = server.queryProp("@netAddress");
  4171. if (netAddress && *netAddress)
  4172. {
  4173. SocketEndpoint ep(netAddress, server.getPropInt("@port", 9876));
  4174. endpoints.append(ep);
  4175. }
  4176. }
  4177. }
  4178. void getRoxieProcessServers(const char *process, SocketEndpointArray &servers)
  4179. {
  4180. Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
  4181. Owned<IConstEnvironment> env = factory->openEnvironment();
  4182. if (!env)
  4183. return;
  4184. Owned<IPropertyTree> root = &env->getPTree();
  4185. getRoxieProcessServers(queryRoxieProcessTree(root, process), servers);
  4186. }
  4187. class CEnvironmentClusterInfo: implements IConstWUClusterInfo, public CInterface
  4188. {
  4189. StringAttr name;
  4190. StringAttr alias;
  4191. StringAttr serverQueue;
  4192. StringAttr agentQueue;
  4193. StringAttr roxieProcess;
  4194. SocketEndpointArray roxieServers;
  4195. StringAttr thorQueue;
  4196. StringArray thorProcesses;
  4197. StringArray primaryThorProcesses;
  4198. StringAttr prefix;
  4199. StringAttr ldapUser;
  4200. StringBuffer ldapPassword;
  4201. ClusterType platform;
  4202. unsigned clusterWidth;
  4203. unsigned roxieRedundancy;
  4204. unsigned channelsPerNode;
  4205. unsigned numberOfSlaveLogs;
  4206. int roxieReplicateOffset;
  4207. public:
  4208. IMPLEMENT_IINTERFACE;
  4209. CEnvironmentClusterInfo(const char *_name, const char *_prefix, const char *_alias, IPropertyTree *agent, IArrayOf<IPropertyTree> &thors, IPropertyTree *roxie)
  4210. : name(_name), prefix(_prefix), alias(_alias), roxieRedundancy(0), channelsPerNode(0), numberOfSlaveLogs(0), roxieReplicateOffset(1)
  4211. {
  4212. StringBuffer queue;
  4213. if (thors.ordinality())
  4214. {
  4215. thorQueue.set(getClusterThorQueueName(queue.clear(), name));
  4216. clusterWidth = 0;
  4217. bool isMultiThor = (thors.length() > 1);
  4218. ForEachItemIn(i,thors)
  4219. {
  4220. IPropertyTree &thor = thors.item(i);
  4221. const char* thorName = thor.queryProp("@name");
  4222. thorProcesses.append(thorName);
  4223. if (!isMultiThor)
  4224. primaryThorProcesses.append(thorName);
  4225. else
  4226. {
  4227. const char *nodeGroup = thor.queryProp("@nodeGroup");
  4228. if (!nodeGroup || strieq(nodeGroup, thorName))
  4229. primaryThorProcesses.append(thorName);
  4230. }
  4231. unsigned nodes = thor.getCount("ThorSlaveProcess");
  4232. if (!nodes)
  4233. throw MakeStringException(WUERR_MismatchClusterSize,"CEnvironmentClusterInfo: Thor cluster can not have 0 slave processes");
  4234. unsigned slavesPerNode = thor.getPropInt("@slavesPerNode", 1);
  4235. unsigned channelsPerSlave = thor.getPropInt("@channelsPerSlave", 1);
  4236. unsigned ts = nodes * slavesPerNode * channelsPerSlave;
  4237. numberOfSlaveLogs = nodes * slavesPerNode;
  4238. if (clusterWidth && (ts!=clusterWidth))
  4239. throw MakeStringException(WUERR_MismatchClusterSize,"CEnvironmentClusterInfo: mismatched thor sizes in cluster");
  4240. clusterWidth = ts;
  4241. bool islcr = !thor.getPropBool("@Legacy");
  4242. if (!islcr)
  4243. throw MakeStringException(WUERR_MismatchThorType,"CEnvironmentClusterInfo: Legacy Thor no longer supported");
  4244. }
  4245. platform = ThorLCRCluster;
  4246. }
  4247. else if (roxie)
  4248. {
  4249. roxieProcess.set(roxie->queryProp("@name"));
  4250. platform = RoxieCluster;
  4251. getRoxieProcessServers(roxie, roxieServers);
  4252. clusterWidth = roxieServers.length();
  4253. ldapUser.set(roxie->queryProp("@ldapUser"));
  4254. StringBuffer encPassword = roxie->queryProp("@ldapPassword");
  4255. if (encPassword.length())
  4256. decrypt(ldapPassword, encPassword);
  4257. const char *redundancyMode = roxie->queryProp("@slaveConfig");
  4258. if (redundancyMode && *redundancyMode)
  4259. {
  4260. unsigned dataCopies = roxie->getPropInt("@numDataCopies", 1);
  4261. if (strieq(redundancyMode, "overloaded"))
  4262. channelsPerNode = roxie->getPropInt("@channelsPernode", 1);
  4263. else if (strieq(redundancyMode, "full redundancy"))
  4264. {
  4265. roxieRedundancy = dataCopies-1;
  4266. roxieReplicateOffset = 0;
  4267. }
  4268. else if (strieq(redundancyMode, "cyclic redundancy"))
  4269. {
  4270. roxieRedundancy = dataCopies-1;
  4271. channelsPerNode = dataCopies;
  4272. roxieReplicateOffset = roxie->getPropInt("@cyclicOffset", 1);
  4273. }
  4274. }
  4275. }
  4276. else
  4277. {
  4278. clusterWidth = 1;
  4279. platform = HThorCluster;
  4280. }
  4281. if (agent)
  4282. {
  4283. assertex(!roxie);
  4284. agentQueue.set(getClusterEclAgentQueueName(queue.clear(), name));
  4285. }
  4286. else if (roxie)
  4287. agentQueue.set(getClusterRoxieQueueName(queue.clear(), name));
  4288. // MORE - does this need to be conditional?
  4289. serverQueue.set(getClusterEclCCServerQueueName(queue.clear(), name));
  4290. }
  4291. IStringVal & getName(IStringVal & str) const
  4292. {
  4293. str.set(name.get());
  4294. return str;
  4295. }
  4296. const char *getAlias() const
  4297. {
  4298. return alias;
  4299. }
  4300. IStringVal & getScope(IStringVal & str) const
  4301. {
  4302. str.set(prefix.get());
  4303. return str;
  4304. }
  4305. IStringVal & getAgentQueue(IStringVal & str) const
  4306. {
  4307. str.set(agentQueue);
  4308. return str;
  4309. }
  4310. virtual IStringVal & getServerQueue(IStringVal & str) const
  4311. {
  4312. str.set(serverQueue);
  4313. return str;
  4314. }
  4315. IStringVal & getThorQueue(IStringVal & str) const
  4316. {
  4317. str.set(thorQueue);
  4318. return str;
  4319. }
  4320. unsigned getSize() const
  4321. {
  4322. return clusterWidth;
  4323. }
  4324. unsigned getNumberOfSlaveLogs() const
  4325. {
  4326. return numberOfSlaveLogs;
  4327. }
  4328. virtual ClusterType getPlatform() const
  4329. {
  4330. return platform;
  4331. }
  4332. IStringVal & getRoxieProcess(IStringVal & str) const
  4333. {
  4334. str.set(roxieProcess.get());
  4335. return str;
  4336. }
  4337. const StringArray & getThorProcesses() const
  4338. {
  4339. return thorProcesses;
  4340. }
  4341. const StringArray & getPrimaryThorProcesses() const
  4342. {
  4343. return primaryThorProcesses;
  4344. }
  4345. const SocketEndpointArray & getRoxieServers() const
  4346. {
  4347. return roxieServers;
  4348. }
  4349. unsigned getRoxieRedundancy() const
  4350. {
  4351. return roxieRedundancy;
  4352. }
  4353. unsigned getChannelsPerNode() const
  4354. {
  4355. return channelsPerNode;
  4356. }
  4357. int getRoxieReplicateOffset() const
  4358. {
  4359. return roxieReplicateOffset;
  4360. }
  4361. const char *getLdapUser() const
  4362. {
  4363. return ldapUser.get();
  4364. }
  4365. virtual const char *getLdapPassword() const
  4366. {
  4367. return ldapPassword.str();
  4368. }
  4369. };
  4370. IStringVal &getProcessQueueNames(IStringVal &ret, const char *process, const char *type, const char *suffix)
  4371. {
  4372. if (process)
  4373. {
  4374. Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
  4375. Owned<IConstEnvironment> env = factory->openEnvironment();
  4376. if (env)
  4377. {
  4378. Owned<IPropertyTree> root = &env->getPTree();
  4379. StringBuffer queueNames;
  4380. StringBuffer xpath;
  4381. xpath.appendf("%s[@process=\"%s\"]", type, process);
  4382. Owned<IPropertyTreeIterator> targets = root->getElements("Software/Topology/Cluster");
  4383. ForEach(*targets)
  4384. {
  4385. IPropertyTree &target = targets->query();
  4386. if (target.hasProp(xpath))
  4387. {
  4388. if (queueNames.length())
  4389. queueNames.append(',');
  4390. queueNames.append(target.queryProp("@name")).append(suffix);
  4391. }
  4392. }
  4393. ret.set(queueNames);
  4394. }
  4395. }
  4396. return ret;
  4397. }
  4398. #define ROXIE_QUEUE_EXT ".roxie"
  4399. #define THOR_QUEUE_EXT ".thor"
  4400. #define ECLCCSERVER_QUEUE_EXT ".eclserver"
  4401. #define ECLSERVER_QUEUE_EXT ECLCCSERVER_QUEUE_EXT
  4402. #define ECLSCHEDULER_QUEUE_EXT ".eclscheduler"
  4403. #define ECLAGENT_QUEUE_EXT ".agent"
  4404. extern WORKUNIT_API void getDFUServerQueueNames(StringArray &ret, const char *process)
  4405. {
  4406. Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
  4407. Owned<IConstEnvironment> env = factory->openEnvironment();
  4408. if (!env)
  4409. return;
  4410. StringBuffer xpath = "Software/DfuServerProcess";
  4411. if (!isEmptyString(process))
  4412. xpath.appendf("[@name=\"%s\"]", process);
  4413. Owned<IPropertyTree> root = &env->getPTree();
  4414. Owned<IPropertyTreeIterator> targets = root->getElements(xpath.str());
  4415. ForEach(*targets)
  4416. {
  4417. IPropertyTree &target = targets->query();
  4418. if (target.hasProp("@queue"))
  4419. ret.appendListUniq(target.queryProp("@queue"), ",");
  4420. }
  4421. return;
  4422. }
  4423. extern WORKUNIT_API IStringVal &getEclCCServerQueueNames(IStringVal &ret, const char *process)
  4424. {
  4425. return getProcessQueueNames(ret, process, "EclCCServerProcess", ECLCCSERVER_QUEUE_EXT);
  4426. }
  4427. extern WORKUNIT_API IStringVal &getEclServerQueueNames(IStringVal &ret, const char *process)
  4428. {
  4429. return getProcessQueueNames(ret, process, "EclServerProcess", ECLSERVER_QUEUE_EXT); // shares queue name with EclCCServer
  4430. }
  4431. extern WORKUNIT_API IStringVal &getEclSchedulerQueueNames(IStringVal &ret, const char *process)
  4432. {
  4433. return getProcessQueueNames(ret, process, "EclSchedulerProcess", ECLSCHEDULER_QUEUE_EXT); // Shares deployment/config with EclCCServer
  4434. }
  4435. extern WORKUNIT_API IStringVal &getAgentQueueNames(IStringVal &ret, const char *process)
  4436. {
  4437. return getProcessQueueNames(ret, process, "EclAgentProcess", ECLAGENT_QUEUE_EXT);
  4438. }
  4439. extern WORKUNIT_API IStringVal &getRoxieQueueNames(IStringVal &ret, const char *process)
  4440. {
  4441. return getProcessQueueNames(ret, process, "RoxieCluster", ROXIE_QUEUE_EXT);
  4442. }
  4443. extern WORKUNIT_API IStringVal &getThorQueueNames(IStringVal &ret, const char *process)
  4444. {
  4445. return getProcessQueueNames(ret, process, "ThorCluster", THOR_QUEUE_EXT);
  4446. }
  4447. extern WORKUNIT_API StringBuffer &getClusterThorQueueName(StringBuffer &ret, const char *cluster)
  4448. {
  4449. return ret.append(cluster).append(THOR_QUEUE_EXT);
  4450. }
  4451. extern WORKUNIT_API StringBuffer &getClusterThorGroupName(StringBuffer &ret, const char *cluster)
  4452. {
  4453. Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
  4454. Owned<IConstEnvironment> env = factory->openEnvironment();
  4455. if (env)
  4456. {
  4457. Owned<IPropertyTree> root = &env->getPTree();
  4458. StringBuffer path;
  4459. path.append("Software/ThorCluster[@name=\"").append(cluster).append("\"]");
  4460. IPropertyTree * child = root->queryPropTree(path);
  4461. if (child)
  4462. getClusterGroupName(*child, ret);
  4463. }
  4464. return ret;
  4465. }
  4466. extern WORKUNIT_API StringBuffer &getClusterRoxieQueueName(StringBuffer &ret, const char *cluster)
  4467. {
  4468. return ret.append(cluster).append(ROXIE_QUEUE_EXT);
  4469. }
  4470. extern WORKUNIT_API StringBuffer &getClusterEclCCServerQueueName(StringBuffer &ret, const char *cluster)
  4471. {
  4472. return ret.append(cluster).append(ECLCCSERVER_QUEUE_EXT);
  4473. }
  4474. extern WORKUNIT_API StringBuffer &getClusterEclServerQueueName(StringBuffer &ret, const char *cluster)
  4475. {
  4476. return ret.append(cluster).append(ECLSERVER_QUEUE_EXT);
  4477. }
  4478. extern WORKUNIT_API StringBuffer &getClusterEclAgentQueueName(StringBuffer &ret, const char *cluster)
  4479. {
  4480. return ret.append(cluster).append(ECLAGENT_QUEUE_EXT);
  4481. }
  4482. extern WORKUNIT_API IStringIterator *getTargetClusters(const char *processType, const char *processName)
  4483. {
  4484. Owned<CStringArrayIterator> ret = new CStringArrayIterator;
  4485. Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
  4486. Owned<IConstEnvironment> env = factory->openEnvironment();
  4487. if (env)
  4488. {
  4489. Owned<IPropertyTree> root = &env->getPTree();
  4490. StringBuffer xpath;
  4491. xpath.appendf("%s", processType ? processType : "*");
  4492. if (processName && *processName)
  4493. xpath.appendf("[@process=\"%s\"]", processName);
  4494. Owned<IPropertyTreeIterator> targets = root->getElements("Software/Topology/Cluster");
  4495. ForEach(*targets)
  4496. {
  4497. IPropertyTree &target = targets->query();
  4498. if (target.hasProp(xpath))
  4499. {
  4500. ret->append(target.queryProp("@name"));
  4501. }
  4502. }
  4503. }
  4504. return ret.getClear();
  4505. }
  4506. extern WORKUNIT_API bool isProcessCluster(const char *process)
  4507. {
  4508. if (!process || !*process)
  4509. return false;
  4510. Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
  4511. Owned<IConstEnvironment> env = factory->openEnvironment();
  4512. if (!env)
  4513. return false;
  4514. Owned<IPropertyTree> root = &env->getPTree();
  4515. VStringBuffer xpath("Software/*Cluster[@name=\"%s\"]", process);
  4516. return root->hasProp(xpath.str());
  4517. }
  4518. extern WORKUNIT_API bool isProcessCluster(const char *remoteDali, const char *process)
  4519. {
  4520. if (!remoteDali || !*remoteDali)
  4521. return isProcessCluster(process);
  4522. if (!process || !*process)
  4523. return false;
  4524. Owned<INode> remote = createINode(remoteDali, 7070);
  4525. if (!remote)
  4526. return false;
  4527. //Cannot use getEnvironmentFactory() since it is using a remotedali
  4528. VStringBuffer xpath("Environment/Software/*Cluster[@name=\"%s\"]/@name", process);
  4529. try
  4530. {
  4531. Owned<IPropertyTreeIterator> clusters = querySDS().getElementsRaw(xpath, remote, 1000*60*1);
  4532. return clusters->first();
  4533. }
  4534. catch (IException *E)
  4535. {
  4536. StringBuffer msg;
  4537. E->errorMessage(msg);
  4538. DBGLOG("Exception validating cluster %s/%s: %s", remoteDali, xpath.str(), msg.str());
  4539. E->Release();
  4540. }
  4541. return true;
  4542. }
  4543. IConstWUClusterInfo* getTargetClusterInfo(IPropertyTree *environment, IPropertyTree *cluster)
  4544. {
  4545. const char *clustname = cluster->queryProp("@name");
  4546. // MORE - at the moment configenf specifies eclagent and thor queues by (in effect) placing an 'example' thor or eclagent in the topology
  4547. // that uses the queue that will be used.
  4548. // We should and I hope will change that, at which point the code below gets simpler
  4549. StringBuffer prefix(cluster->queryProp("@prefix"));
  4550. prefix.toLowerCase();
  4551. StringBuffer xpath;
  4552. StringBuffer querySetName;
  4553. IPropertyTree *agent = NULL;
  4554. const char *agentName = cluster->queryProp("EclAgentProcess/@process");
  4555. if (agentName)
  4556. {
  4557. xpath.clear().appendf("Software/EclAgentProcess[@name=\"%s\"]", agentName);
  4558. agent = environment->queryPropTree(xpath.str());
  4559. }
  4560. Owned<IPropertyTreeIterator> ti = cluster->getElements("ThorCluster");
  4561. IArrayOf<IPropertyTree> thors;
  4562. ForEach(*ti)
  4563. {
  4564. const char *thorName = ti->query().queryProp("@process");
  4565. if (thorName)
  4566. {
  4567. xpath.clear().appendf("Software/ThorCluster[@name=\"%s\"]", thorName);
  4568. thors.append(*environment->getPropTree(xpath.str()));
  4569. }
  4570. }
  4571. const char *roxieName = cluster->queryProp("RoxieCluster/@process");
  4572. return new CEnvironmentClusterInfo(clustname, prefix, cluster->queryProp("@alias"), agent, thors, queryRoxieProcessTree(environment, roxieName));
  4573. }
  4574. IPropertyTree* getTopologyCluster(Owned<IPropertyTree> &envRoot, const char *clustname)
  4575. {
  4576. if (!clustname || !*clustname)
  4577. return NULL;
  4578. Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
  4579. Owned<IConstEnvironment> env = factory->openEnvironment();
  4580. if (!env)
  4581. return NULL;
  4582. envRoot.setown(&env->getPTree());
  4583. StringBuffer xpath;
  4584. xpath.appendf("Software/Topology/Cluster[@name=\"%s\"]", clustname);
  4585. return envRoot->getPropTree(xpath.str());
  4586. }
  4587. bool validateTargetClusterName(const char *clustname)
  4588. {
  4589. Owned<IPropertyTree> envRoot;
  4590. Owned<IPropertyTree> cluster = getTopologyCluster(envRoot, clustname);
  4591. return (cluster.get()!=NULL);
  4592. }
  4593. IConstWUClusterInfo* getTargetClusterInfo(const char *clustname)
  4594. {
  4595. Owned<IPropertyTree> envRoot;
  4596. Owned<IPropertyTree> cluster = getTopologyCluster(envRoot, clustname);
  4597. if (!cluster)
  4598. return NULL;
  4599. return getTargetClusterInfo(envRoot, cluster);
  4600. }
  4601. unsigned getEnvironmentClusterInfo(CConstWUClusterInfoArray &clusters)
  4602. {
  4603. Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
  4604. Owned<IConstEnvironment> env = factory->openEnvironment();
  4605. if (!env)
  4606. return 0;
  4607. Owned<IPropertyTree> root = &env->getPTree();
  4608. return getEnvironmentClusterInfo(root, clusters);
  4609. }
  4610. unsigned getEnvironmentClusterInfo(IPropertyTree* environmentRoot, CConstWUClusterInfoArray &clusters)
  4611. {
  4612. if (!environmentRoot)
  4613. return 0;
  4614. Owned<IPropertyTreeIterator> clusterIter = environmentRoot->getElements("Software/Topology/Cluster");
  4615. ForEach(*clusterIter)
  4616. {
  4617. IPropertyTree &node = clusterIter->query();
  4618. Owned<IConstWUClusterInfo> cluster = getTargetClusterInfo(environmentRoot, &node);
  4619. clusters.append(*cluster.getClear());
  4620. }
  4621. return clusters.ordinality();
  4622. }
  4623. const char *getTargetClusterComponentName(const char *clustname, const char *processType, StringBuffer &name)
  4624. {
  4625. if (!clustname)
  4626. return NULL;
  4627. Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
  4628. Owned<IConstEnvironment> env = factory->openEnvironment();
  4629. if (!env)
  4630. return NULL;
  4631. Owned<IPropertyTree> root = &env->getPTree();
  4632. StringBuffer xpath;
  4633. xpath.appendf("Software/Topology/Cluster[@name=\"%s\"]", clustname);
  4634. Owned<IPropertyTree> cluster = root->getPropTree(xpath.str());
  4635. if (!cluster)
  4636. return NULL;
  4637. StringBuffer xpath1;
  4638. xpath1.appendf("%s/@process", processType);
  4639. name.append(cluster->queryProp(xpath1.str()));
  4640. return name.str();
  4641. }
  4642. unsigned getEnvironmentThorClusterNames(StringArray &thorNames, StringArray &groupNames, StringArray &targetNames, StringArray &queueNames)
  4643. {
  4644. Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
  4645. Owned<IConstEnvironment> env = factory->openEnvironment();
  4646. if (!env)
  4647. return 0;
  4648. Owned<IPropertyTree> root = &env->getPTree();
  4649. Owned<IPropertyTreeIterator> allTargets = root->getElements("Software/Topology/Cluster");
  4650. ForEach(*allTargets)
  4651. {
  4652. IPropertyTree &target = allTargets->query();
  4653. const char *targetName = target.queryProp("@name");
  4654. if (targetName && *targetName)
  4655. {
  4656. Owned<IPropertyTreeIterator> thorClusters = target.getElements("ThorCluster");
  4657. ForEach(*thorClusters)
  4658. {
  4659. const char *thorName = thorClusters->query().queryProp("@process");
  4660. VStringBuffer query("Software/ThorCluster[@name=\"%s\"]",thorName);
  4661. IPropertyTree *thorCluster = root->queryPropTree(query.str());
  4662. if (thorCluster)
  4663. {
  4664. const char *groupName = thorCluster->queryProp("@nodeGroup");
  4665. if (!groupName||!*groupName)
  4666. groupName = thorName;
  4667. thorNames.append(thorName);
  4668. groupNames.append(groupName);
  4669. targetNames.append(targetName);
  4670. StringBuffer queueName(targetName);
  4671. queueNames.append(queueName.append(THOR_QUEUE_EXT));
  4672. }
  4673. }
  4674. }
  4675. }
  4676. return thorNames.ordinality();
  4677. }
  4678. unsigned getEnvironmentHThorClusterNames(StringArray &eclAgentNames, StringArray &groupNames, StringArray &targetNames)
  4679. {
  4680. Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
  4681. Owned<IConstEnvironment> env = factory->openEnvironment();
  4682. if (!env)
  4683. return 0;
  4684. Owned<IPropertyTree> root = &env->getPTree();
  4685. Owned<IPropertyTreeIterator> allEclAgents = root->getElements("Software/EclAgentProcess");
  4686. ForEach(*allEclAgents)
  4687. {
  4688. IPropertyTree &eclAgent = allEclAgents->query();
  4689. const char *eclAgentName = eclAgent.queryProp("@name");
  4690. if (eclAgentName && *eclAgentName)
  4691. {
  4692. Owned<IPropertyTreeIterator> allTargets = root->getElements("Software/Topology/Cluster");
  4693. ForEach(*allTargets)
  4694. {
  4695. IPropertyTree &target = allTargets->query();
  4696. const char *targetName = target.queryProp("@name");
  4697. if (targetName && *targetName)
  4698. {
  4699. StringBuffer xpath;
  4700. xpath.appendf("EclAgentProcess[@process=\"%s\"]", eclAgentName);
  4701. if (target.hasProp(xpath) && !target.hasProp("ThorCluster"))
  4702. {
  4703. StringBuffer groupName("hthor__");
  4704. groupName.append(eclAgentName);
  4705. groupNames.append(groupName);
  4706. eclAgentNames.append(eclAgentName);
  4707. targetNames.append(targetName);
  4708. }
  4709. }
  4710. }
  4711. }
  4712. }
  4713. return eclAgentNames.ordinality();
  4714. }
  4715. IStringVal& CLocalWorkUnit::getScope(IStringVal &str) const
  4716. {
  4717. CriticalBlock block(crit);
  4718. if (p->hasProp("Debug/ForceScope"))
  4719. {
  4720. StringBuffer prefix(p->queryProp("Debug/ForceScope"));
  4721. str.set(prefix.toLowerCase().str());
  4722. }
  4723. else
  4724. {
  4725. Owned <IConstWUClusterInfo> ci = getTargetClusterInfo(p->queryProp("@clusterName"));
  4726. if (ci)
  4727. ci->getScope(str);
  4728. else
  4729. str.clear();
  4730. }
  4731. return str;
  4732. }
  4733. //Queries
  4734. void CLocalWorkUnit::setCodeVersion(unsigned codeVersion, const char * buildVersion, const char * eclVersion)
  4735. {
  4736. CriticalBlock block(crit);
  4737. p->setPropInt("@codeVersion", codeVersion);
  4738. p->setProp("@buildVersion", buildVersion);
  4739. p->setProp("@eclVersion", eclVersion);
  4740. }
  4741. unsigned CLocalWorkUnit::getCodeVersion() const
  4742. {
  4743. CriticalBlock block(crit);
  4744. return p->getPropInt("@codeVersion");
  4745. }
  4746. unsigned CLocalWorkUnit::getWuidVersion() const
  4747. {
  4748. CriticalBlock block(crit);
  4749. return p->getPropInt("@wuidVersion");
  4750. }
  4751. void CLocalWorkUnit::getBuildVersion(IStringVal & buildVersion, IStringVal & eclVersion) const
  4752. {
  4753. CriticalBlock block(crit);
  4754. buildVersion.set(p->queryProp("@buildVersion"));
  4755. eclVersion.set(p->queryProp("@eclVersion"));
  4756. }
  4757. void CLocalWorkUnit::setCloneable(bool value)
  4758. {
  4759. CriticalBlock block(crit);
  4760. p->setPropInt("@cloneable", value);
  4761. }
  4762. void CLocalWorkUnit::setIsClone(bool value)
  4763. {
  4764. CriticalBlock block(crit);
  4765. p->setPropInt("@isClone", value);
  4766. }
  4767. bool CLocalWorkUnit::getCloneable() const
  4768. {
  4769. CriticalBlock block(crit);
  4770. return p->getPropBool("@cloneable", false);
  4771. }
  4772. IUserDescriptor *CLocalWorkUnit::queryUserDescriptor() const
  4773. {
  4774. CriticalBlock block(crit);
  4775. if (!userDesc)
  4776. {
  4777. SCMStringBuffer token, user, password;
  4778. getSecurityToken(token);
  4779. extractToken(token.str(), queryWuid(), user, password);
  4780. userDesc.setown(createUserDescriptor());
  4781. userDesc->set(user.str(), password.str());
  4782. }
  4783. return userDesc;
  4784. }
  4785. bool CLocalWorkUnit::isProtected() const
  4786. {
  4787. CriticalBlock block(crit);
  4788. return p->getPropBool("@protected", false);
  4789. }
  4790. bool CLocalWorkUnit::isPausing() const
  4791. {
  4792. CriticalBlock block(crit);
  4793. if (WUActionPause == getAction())
  4794. {
  4795. switch (getState())
  4796. {
  4797. case WUStateRunning:
  4798. case WUStateAborting:
  4799. return true;
  4800. }
  4801. }
  4802. return false;
  4803. }
  4804. void CLocalWorkUnit::protect(bool protectMode)
  4805. {
  4806. CriticalBlock block(crit);
  4807. p->setPropBool("@protected", protectMode);
  4808. }
  4809. void CLocalWorkUnit::setResultLimit(unsigned value)
  4810. {
  4811. CriticalBlock block(crit);
  4812. p->setPropInt("resultLimit", value);
  4813. }
  4814. unsigned CLocalWorkUnit::getResultLimit() const
  4815. {
  4816. CriticalBlock block(crit);
  4817. return p->getPropInt("resultLimit");
  4818. }
  4819. IStringVal & CLocalWorkUnit::getSnapshot(IStringVal & str) const
  4820. {
  4821. CriticalBlock block(crit);
  4822. str.set(p->queryProp("SNAPSHOT"));
  4823. return str;
  4824. }
  4825. void CLocalWorkUnit::setSnapshot(const char * val)
  4826. {
  4827. CriticalBlock block(crit);
  4828. p->setProp("SNAPSHOT", val);
  4829. }
  4830. const static mapEnums warningSeverityMap[] =
  4831. {
  4832. { SeverityInformation, "info" },
  4833. { SeverityWarning, "warning" },
  4834. { SeverityError, "error" },
  4835. { SeverityAlert, "alert" },
  4836. { SeverityIgnore, "ignore" },
  4837. { SeverityFatal, "fatal" },
  4838. { SeverityUnknown, NULL }
  4839. };
  4840. ErrorSeverity CLocalWorkUnit::getWarningSeverity(unsigned code, ErrorSeverity defaultSeverity) const
  4841. {
  4842. StringBuffer xpath;
  4843. xpath.append("OnWarnings/OnWarning[@code='").append(code).append("']");
  4844. CriticalBlock block(crit);
  4845. IPropertyTree * mapping = p->queryPropTree(xpath);
  4846. if (mapping)
  4847. return (ErrorSeverity) getEnum(mapping, "@severity", warningSeverityMap);
  4848. return defaultSeverity;
  4849. }
  4850. void CLocalWorkUnit::setWarningSeverity(unsigned code, ErrorSeverity severity)
  4851. {
  4852. StringBuffer xpath;
  4853. xpath.append("OnWarnings/OnWarning[@code='").append(code).append("']");
  4854. CriticalBlock block(crit);
  4855. IPropertyTree * mapping = p->queryPropTree(xpath);
  4856. if (!mapping)
  4857. {
  4858. IPropertyTree * onWarnings = ensurePTree(p, "OnWarnings");
  4859. mapping = onWarnings->addPropTree("OnWarning");
  4860. mapping->setPropInt("@code", code);
  4861. }
  4862. setEnum(mapping, "@severity", severity, warningSeverityMap);
  4863. }
  4864. static int comparePropTrees(IInterface * const *ll, IInterface * const *rr)
  4865. {
  4866. IPropertyTree *l = (IPropertyTree *) *ll;
  4867. IPropertyTree *r = (IPropertyTree *) *rr;
  4868. return stricmp(l->queryName(), r->queryName());
  4869. };
  4870. unsigned CLocalWorkUnit::calculateHash(unsigned crc)
  4871. {
  4872. // Any other values in the WU that could affect generated code should be crc'ed here
  4873. IPropertyTree *tree = p->queryBranch("Debug");
  4874. if (tree)
  4875. {
  4876. Owned<IPropertyTreeIterator> sub = tree->getElements("*");
  4877. ICopyArrayOf<IPropertyTree> subs;
  4878. for(sub->first(); sub->isValid(); sub->next())
  4879. subs.append(sub->query());
  4880. subs.sort(comparePropTrees);
  4881. ForEachItemIn(idx, subs)
  4882. {
  4883. const char *name = subs.item(idx).queryName();
  4884. const char *val = subs.item(idx).queryProp(NULL);
  4885. crc = crc32(name, (size32_t)strlen(name), crc);
  4886. if (val)
  4887. crc = crc32(val, (size32_t)strlen(val), crc);
  4888. }
  4889. }
  4890. Owned<IConstWUPluginIterator> plugins = &getPlugins();
  4891. for (plugins->first();plugins->isValid();plugins->next())
  4892. {
  4893. IConstWUPlugin &thisplugin = plugins->query();
  4894. SCMStringBuffer version;
  4895. thisplugin.getPluginVersion(version);
  4896. crc = crc32(version.str(), version.length(), crc);
  4897. }
  4898. return crc;
  4899. }
  4900. static void updateProp(IPropertyTree * to, const IPropertyTree * from, const char * xpath)
  4901. {
  4902. if (!to->hasProp(xpath) && from->hasProp(xpath))
  4903. to->setProp(xpath, from->queryProp(xpath));
  4904. }
  4905. static void setProp(IPropertyTree * to, const IPropertyTree * from, const char * xpath)
  4906. {
  4907. if (from->hasProp(xpath))
  4908. to->setProp(xpath, from->queryProp(xpath));
  4909. }
  4910. static void copyTree(IPropertyTree * to, const IPropertyTree * from, const char * xpath)
  4911. {
  4912. IPropertyTree * match = from->getBranch(xpath);
  4913. if (match)
  4914. to->setPropTree(xpath, match);
  4915. }
  4916. IPropertyTree *CLocalWorkUnit::queryPTree() const
  4917. {
  4918. return p;
  4919. }
  4920. void CLocalWorkUnit::copyWorkUnit(IConstWorkUnit *cached, bool all)
  4921. {
  4922. CLocalWorkUnit *from = QUERYINTERFACE(cached, CLocalWorkUnit);
  4923. if (!from)
  4924. {
  4925. CLockedWorkUnit *fl = QUERYINTERFACE(cached, CLockedWorkUnit);
  4926. if (!fl)
  4927. throw MakeStringException(WUERR_InternalUnknownImplementation, "Cached workunit not created using workunit dll");
  4928. from = fl->c;
  4929. }
  4930. // Need to copy the query, the results, and the graphs from the cached query.
  4931. // The cache is made before the query is executed so there is no need to clear them.
  4932. if (!cached->getCloneable())
  4933. throw MakeStringException(WUERR_CannotCloneWorkunit, "Source work unit not marked as clonable");
  4934. const IPropertyTree * fromP = from->p;
  4935. IPropertyTree *pt;
  4936. CriticalBlock block(crit);
  4937. clearCached(false);
  4938. query.clear();
  4939. updateProp(p, fromP, "@jobName");
  4940. copyTree(p, fromP, "Query");
  4941. pt = fromP->getBranch("Application/LibraryModule");
  4942. if (pt)
  4943. {
  4944. ensurePTree(p, "Application");
  4945. p->setPropTree("Application/LibraryModule", pt);
  4946. }
  4947. pt = fromP->queryBranch("Debug");
  4948. if (pt)
  4949. {
  4950. IPropertyTree *curDebug = p->queryPropTree("Debug");
  4951. if (curDebug)
  4952. {
  4953. Owned<IPropertyTreeIterator> elems = pt->getElements("*");
  4954. ForEach(*elems)
  4955. {
  4956. IPropertyTree *elem = &elems->query();
  4957. if (!curDebug->hasProp(elem->queryName()))
  4958. curDebug->setPropTree(elem->queryName(),LINK(elem));
  4959. }
  4960. }
  4961. else
  4962. p->setPropTree("Debug", LINK(pt));
  4963. }
  4964. copyTree(p, fromP, "OnWarnings");
  4965. copyTree(p, fromP, "Plugins");
  4966. copyTree(p, fromP, "Libraries");
  4967. copyTree(p, fromP, "Results");
  4968. copyTree(p, fromP, "Graphs");
  4969. copyTree(p, fromP, "Workflow");
  4970. copyTree(p, fromP, "WebServicesInfo");
  4971. if (all)
  4972. {
  4973. // 'all' mode is used when setting up a dali WU from the embedded wu in a workunit dll
  4974. // Merge timing info from both branches
  4975. pt = fromP->getBranch("Timings");
  4976. if (pt)
  4977. {
  4978. IPropertyTree *tgtTimings = ensurePTree(p, "Timings");
  4979. mergePTree(tgtTimings, pt);
  4980. pt->Release();
  4981. }
  4982. pt = fromP->getBranch("Statistics");
  4983. if (pt)
  4984. {
  4985. IPropertyTree *tgtStatistics = ensurePTree(p, "Statistics");
  4986. mergePTree(tgtStatistics, pt);
  4987. pt->Release();
  4988. }
  4989. }
  4990. updateProp(p, fromP, "@clusterName");
  4991. updateProp(p, fromP, "allowedclusters");
  4992. updateProp(p, fromP, "@submitID");
  4993. updateProp(p, fromP, "SNAPSHOT");
  4994. setProp(p, fromP, "@eventScheduledCount");
  4995. //MORE: This is very adhoc. All options that should be cloned should really be in a common branch
  4996. if (all)
  4997. {
  4998. setProp(p, fromP, "PriorityFlag");
  4999. setProp(p, fromP, "@priorityClass");
  5000. setProp(p, fromP, "@protected");
  5001. setProp(p, fromP, "@clusterName");
  5002. updateProp(p, fromP, "@scope");
  5003. }
  5004. //Variables may have been set up as parameters to the query - so need to preserve any values that were supplied.
  5005. pt = fromP->getBranch("Variables");
  5006. if (pt)
  5007. {
  5008. IPropertyTree *ptTgtVariables = ensurePTree(p, "Variables");
  5009. Owned<IPropertyTreeIterator> ptiVariable = pt->getElements("Variable");
  5010. for (ptiVariable->first(); ptiVariable->isValid(); ptiVariable->next())
  5011. {
  5012. IPropertyTree *ptSrcVariable = &ptiVariable->query();
  5013. const char *name = ptSrcVariable->queryProp("@name");
  5014. assertex(name);
  5015. StringBuffer xpath;
  5016. xpath.append("Variable[@name='").append(name).append("']");
  5017. IPropertyTree *ptTgtVariable = ptTgtVariables->queryPropTree(xpath.str());
  5018. IPropertyTree *merged = createPTreeFromIPT(ptSrcVariable); // clone entire source info...
  5019. merged->removeProp("Value"); // except value and status
  5020. merged->setProp("@status", "undefined");
  5021. if (!merged->getPropBool("@isScalar"))
  5022. merged->removeProp("totalRowCount");
  5023. merged->removeProp("rowCount");
  5024. // If there are any other fields that get set ONLY by eclagent, strip them out here...
  5025. if (ptTgtVariable)
  5026. {
  5027. // copy status and Value from what is already set in target
  5028. merged->setProp("@status", ptTgtVariable->queryProp("@status"));
  5029. MemoryBuffer value;
  5030. if (ptTgtVariable->getPropBin("Value", value))
  5031. merged->setPropBin("Value", value.length(), value.toByteArray());
  5032. ptTgtVariable->removeProp(xpath.str());
  5033. // If there are any other fields in a variable that get set by ws_ecl before submitting, copy them across here...
  5034. }
  5035. ptTgtVariables->addPropTree("Variable", merged);
  5036. }
  5037. pt->Release();
  5038. }
  5039. p->setProp("@codeVersion", fromP->queryProp("@codeVersion"));
  5040. p->setProp("@buildVersion", fromP->queryProp("@buildVersion"));
  5041. p->setProp("@eclVersion", fromP->queryProp("@eclVersion"));
  5042. p->setProp("@hash", fromP->queryProp("@hash"));
  5043. p->setPropBool("@cloneable", true);
  5044. p->setPropBool("@isClone", true);
  5045. resetWorkflow(); // the source Workflow section may have had some parts already executed...
  5046. Owned<IPropertyTreeIterator> results = p->getElements("Results/Result");
  5047. ForEach(*results)
  5048. {
  5049. CLocalWUResult result(LINK(&results->query()));
  5050. result.setResultStatus(ResultStatusUndefined);
  5051. }
  5052. copyTree(p, fromP, "usedsources"); // field usage
  5053. }
  5054. bool CLocalWorkUnit::hasDebugValue(const char *propname) const
  5055. {
  5056. StringBuffer lower;
  5057. lower.append(propname).toLowerCase();
  5058. CriticalBlock block(crit);
  5059. StringBuffer prop("Debug/");
  5060. return p->hasProp(prop.append(lower));
  5061. }
  5062. IStringVal& CLocalWorkUnit::getDebugValue(const char *propname, IStringVal &str) const
  5063. {
  5064. StringBuffer lower;
  5065. lower.append(propname).toLowerCase();
  5066. CriticalBlock block(crit);
  5067. StringBuffer prop("Debug/");
  5068. str.set(p->queryProp(prop.append(lower).str()));
  5069. return str;
  5070. }
  5071. IStringIterator& CLocalWorkUnit::getDebugValues() const
  5072. {
  5073. return getDebugValues(NULL);
  5074. }
  5075. IStringIterator& CLocalWorkUnit::getDebugValues(const char *prop) const
  5076. {
  5077. CriticalBlock block(crit);
  5078. StringBuffer path("Debug/");
  5079. if (prop)
  5080. {
  5081. StringBuffer lower;
  5082. lower.append(prop).toLowerCase();
  5083. path.append(lower);
  5084. }
  5085. else
  5086. path.append("*");
  5087. return *new CStringPTreeTagIterator(p->getElements(path.str()));
  5088. }
  5089. int CLocalWorkUnit::getDebugValueInt(const char *propname, int defVal) const
  5090. {
  5091. StringBuffer lower;
  5092. lower.append(propname).toLowerCase();
  5093. CriticalBlock block(crit);
  5094. StringBuffer prop("Debug/");
  5095. prop.append(lower);
  5096. return p->getPropInt(prop.str(), defVal);
  5097. }
  5098. __int64 CLocalWorkUnit::getDebugValueInt64(const char *propname, __int64 defVal) const
  5099. {
  5100. StringBuffer lower;
  5101. lower.append(propname).toLowerCase();
  5102. CriticalBlock block(crit);
  5103. StringBuffer prop("Debug/");
  5104. prop.append(lower);
  5105. return p->getPropInt64(prop.str(), defVal);
  5106. }
  5107. bool CLocalWorkUnit::getDebugValueBool(const char * propname, bool defVal) const
  5108. {
  5109. StringBuffer lower;
  5110. lower.append(propname).toLowerCase();
  5111. CriticalBlock block(crit);
  5112. StringBuffer prop("Debug/");
  5113. prop.append(lower);
  5114. return p->getPropBool(prop.str(), defVal);
  5115. }
  5116. IStringIterator *CLocalWorkUnit::getLogs(const char *type, const char *instance) const
  5117. {
  5118. VStringBuffer xpath("Process/%s/", type);
  5119. if (instance)
  5120. xpath.append(instance);
  5121. else
  5122. xpath.append("*");
  5123. CriticalBlock block(crit);
  5124. if (p->getPropInt("@wuidVersion") < 1) // legacy wuid
  5125. {
  5126. // NB: instance unused
  5127. if (streq("EclAgent", type))
  5128. return new CStringPTreeIterator(p->getElements("Debug/eclagentlog"));
  5129. else if (streq("Thor", type))
  5130. return new CStringPTreeIterator(p->getElements("Debug/thorlog*"));
  5131. VStringBuffer xpath("Debug/%s", type);
  5132. return new CStringPTreeIterator(p->getElements(xpath.str()));
  5133. }
  5134. else
  5135. return new CStringPTreeAttrIterator(p->getElements(xpath.str()), "@log");
  5136. }
  5137. IPropertyTreeIterator* CLocalWorkUnit::getProcesses(const char *type, const char *instance) const
  5138. {
  5139. VStringBuffer xpath("Process/%s/", type);
  5140. if (instance)
  5141. xpath.append(instance);
  5142. else
  5143. xpath.append("*");
  5144. CriticalBlock block(crit);
  5145. return p->getElements(xpath.str());
  5146. }
  5147. IStringIterator *CLocalWorkUnit::getProcesses(const char *type) const
  5148. {
  5149. VStringBuffer xpath("Process/%s/*", type);
  5150. CriticalBlock block(crit);
  5151. return new CStringPTreeTagIterator(p->getElements(xpath.str()));
  5152. }
  5153. void CLocalWorkUnit::addProcess(const char *type, const char *instance, unsigned pid, const char *log)
  5154. {
  5155. VStringBuffer processType("Process/%s", type);
  5156. VStringBuffer xpath("%s/%s", processType.str(), instance);
  5157. if (log)
  5158. xpath.appendf("[@log=\"%s\"]", log);
  5159. CriticalBlock block(crit);
  5160. if (!p->hasProp(xpath))
  5161. {
  5162. IPropertyTree *node = ensurePTree(p, processType.str());
  5163. node = node->addPropTree(instance);
  5164. node->setProp("@log", log);
  5165. node->setPropInt("@pid", pid);
  5166. }
  5167. }
  5168. void CLocalWorkUnit::setDebugValue(const char *propname, const char *value, bool overwrite)
  5169. {
  5170. StringBuffer lower;
  5171. lower.append(propname).toLowerCase();
  5172. CriticalBlock block(crit);
  5173. StringBuffer prop("Debug/");
  5174. prop.append(lower);
  5175. if (overwrite || !p->hasProp(prop.str()))
  5176. {
  5177. // MORE - not sure this line should be needed....
  5178. p->setProp("Debug", "");
  5179. p->setProp(prop.str(), value);
  5180. }
  5181. }
  5182. void CLocalWorkUnit::setDebugValueInt(const char *propname, int value, bool overwrite)
  5183. {
  5184. StringBuffer lower;
  5185. lower.append(propname).toLowerCase();
  5186. CriticalBlock block(crit);
  5187. StringBuffer prop("Debug/");
  5188. prop.append(lower);
  5189. if (overwrite || !p->hasProp(prop.str()))
  5190. {
  5191. // MORE - not sure this line should be needed....
  5192. p->setProp("Debug", "");
  5193. p->setPropInt(prop.str(), value);
  5194. }
  5195. }
  5196. void CLocalWorkUnit::setTracingValue(const char *propname, const char *value)
  5197. {
  5198. CriticalBlock block(crit);
  5199. // MORE - not sure this line should be needed....
  5200. p->setProp("Tracing", "");
  5201. StringBuffer prop("Tracing/");
  5202. p->setProp(prop.append(propname).str(), value);
  5203. }
  5204. void CLocalWorkUnit::setTracingValueInt(const char *propname, int value)
  5205. {
  5206. CriticalBlock block(crit);
  5207. StringBuffer prop("Tracing/");
  5208. p->setPropInt(prop.append(propname).str(), value);
  5209. }
  5210. void CLocalWorkUnit::setTracingValueInt64(const char *propname, __int64 value)
  5211. {
  5212. CriticalBlock block(crit);
  5213. VStringBuffer prop("Tracing/%s", propname);
  5214. p->setPropInt64(prop.str(), value);
  5215. }
  5216. IConstWUQuery* CLocalWorkUnit::getQuery() const
  5217. {
  5218. // For this to be legally called, we must have the read-able interface. So we are already locked for (at least) read.
  5219. CriticalBlock block(crit);
  5220. if (!query)
  5221. {
  5222. IPropertyTree *s = p->getPropTree("Query");
  5223. if (s)
  5224. query.setown(new CLocalWUQuery(s)); // NB takes ownership of 's'
  5225. }
  5226. return query.getLink();
  5227. }
  5228. IWUQuery* CLocalWorkUnit::updateQuery()
  5229. {
  5230. // For this to be legally called, we must have the write-able interface. So we are already locked for write.
  5231. CriticalBlock block(crit);
  5232. if (!query)
  5233. {
  5234. IPropertyTree *s = p->queryPropTree("Query");
  5235. if (!s)
  5236. s = p->addPropTree("Query", createPTreeFromXMLString("<Query fetchEntire='1'/>")); // Is this really desirable (the fetchEntire) ?
  5237. s->Link();
  5238. query.setown(new CLocalWUQuery(s));
  5239. }
  5240. return query.getLink();
  5241. }
  5242. void CLocalWorkUnit::loadPlugins() const
  5243. {
  5244. CriticalBlock block(crit);
  5245. if (!pluginsCached)
  5246. {
  5247. assertex(plugins.length() == 0);
  5248. Owned<IPropertyTreeIterator> r = p->getElements("Plugins/Plugin");
  5249. for (r->first(); r->isValid(); r->next())
  5250. {
  5251. IPropertyTree *rp = &r->query();
  5252. rp->Link();
  5253. plugins.append(*new CLocalWUPlugin(rp));
  5254. }
  5255. pluginsCached = true;
  5256. }
  5257. }
  5258. IConstWUPluginIterator& CLocalWorkUnit::getPlugins() const
  5259. {
  5260. CriticalBlock block(crit);
  5261. loadPlugins();
  5262. return *new CArrayIteratorOf<IConstWUPlugin,IConstWUPluginIterator> (plugins, 0, (IConstWorkUnit *) this);
  5263. }
  5264. void CLocalWorkUnit::loadLibraries() const
  5265. {
  5266. CriticalBlock block(crit);
  5267. if (!librariesCached)
  5268. {
  5269. assertex(libraries.length() == 0);
  5270. Owned<IPropertyTreeIterator> r = p->getElements("Libraries/Library");
  5271. ForEach(*r)
  5272. {
  5273. IPropertyTree *rp = &r->query();
  5274. rp->Link();
  5275. libraries.append(*new CLocalWULibrary(rp));
  5276. }
  5277. librariesCached = true;
  5278. }
  5279. }
  5280. IConstWULibraryIterator& CLocalWorkUnit::getLibraries() const
  5281. {
  5282. CriticalBlock block(crit);
  5283. loadLibraries();
  5284. return *new CArrayIteratorOf<IConstWULibrary,IConstWULibraryIterator> (libraries, 0, (IConstWorkUnit *) this);
  5285. }
  5286. IConstWULibrary * CLocalWorkUnit::getLibraryByName(const char * search) const
  5287. {
  5288. CriticalBlock block(crit);
  5289. loadLibraries();
  5290. ForEachItemIn(idx, libraries)
  5291. {
  5292. SCMStringBuffer name;
  5293. IConstWULibrary &cur = libraries.item(idx);
  5294. cur.getName(name);
  5295. if (stricmp(name.str(), search)==0)
  5296. return &OLINK(cur);
  5297. }
  5298. return NULL;
  5299. }
  5300. StringBuffer &formatGraphTimerLabel(StringBuffer &str, const char *graphName, unsigned subGraphNum, unsigned __int64 subId)
  5301. {
  5302. str.append("Graph ").append(graphName);
  5303. if (subGraphNum) str.append(" - ").append(subGraphNum).append(" (").append(subId).append(")");
  5304. else if (subId) str.append(" - id(").append(subId).append(")");
  5305. return str;
  5306. }
  5307. StringBuffer &formatGraphTimerScope(StringBuffer &str, const char *graphName, unsigned subGraphNum, unsigned __int64 subId)
  5308. {
  5309. str.append(graphName);
  5310. if (subId) str.append(":sg").append(subId);
  5311. return str;
  5312. }
  5313. bool parseGraphTimerLabel(const char *label, StringAttr &graphName, unsigned & graphNum, unsigned &subGraphNum, unsigned &subId)
  5314. {
  5315. // expects format: "Graph <graphname>[ - <subgraphnum> (<subgraphid>)]"
  5316. unsigned len = (size32_t)strlen(label);
  5317. if (len < 6 || (0 != memcmp(label, "Graph ", 6)))
  5318. return false;
  5319. graphNum = 0;
  5320. subGraphNum = 0;
  5321. subId = 0;
  5322. const char *finger = label+6;
  5323. const char *finger2 = strchr(finger, '-');
  5324. if (NULL == finger2) // just graphName
  5325. graphName.set(finger);
  5326. else
  5327. {
  5328. graphName.set(finger, (size32_t)((finger2-1)-finger));
  5329. finger = finger2+2; // skip '-' and space
  5330. finger2 = strchr(finger, ' ');
  5331. if (finger2)
  5332. {
  5333. subGraphNum = atoi_l(finger, (size32_t)(finger2-finger));
  5334. finger = finger2+2; // skip space and '('
  5335. finger2 = strchr(finger, ')');
  5336. if (finger2)
  5337. subId = atoi_l(finger, (size32_t)(finger2-finger));
  5338. }
  5339. else if (((len-(finger-label))>3) && 0 == memcmp(finger, "id(", 3)) // subgraph id only, new format.
  5340. {
  5341. finger += 3;
  5342. finger2 = strchr(finger, ')');
  5343. if (finger2)
  5344. subId = atoi_l(finger, (size32_t)(finger2-finger));
  5345. }
  5346. }
  5347. if (graphName && !memicmp(graphName, "graph", 5))
  5348. graphNum = atoi(graphName + 5);
  5349. return true;
  5350. }
  5351. bool parseGraphScope(const char *scope, StringAttr &graphName, unsigned & graphNum, unsigned &subGraphId)
  5352. {
  5353. if (!MATCHES_CONST_PREFIX(scope, GraphScopePrefix))
  5354. return false;
  5355. graphNum = atoi(scope + CONST_STRLEN(GraphScopePrefix));
  5356. subGraphId = 0;
  5357. const char * colon = strchr(scope, ':');
  5358. if (!colon)
  5359. {
  5360. graphName.set(scope);
  5361. return true;
  5362. }
  5363. const char * subgraph = colon+1;
  5364. graphName.set(scope, (size32_t)(colon - scope));
  5365. if (MATCHES_CONST_PREFIX(subgraph, SubGraphScopePrefix))
  5366. subGraphId = atoi(subgraph+CONST_STRLEN(SubGraphScopePrefix));
  5367. return true;
  5368. }
  5369. class WorkUnitStatisticsIterator : public CArrayIteratorOf<IConstWUStatistic,IConstWUStatisticIterator>
  5370. {
  5371. typedef CArrayIteratorOf<IConstWUStatistic,IConstWUStatisticIterator> PARENT;
  5372. public:
  5373. WorkUnitStatisticsIterator(const IArray &a, aindex_t start, IInterface *owner, const IStatisticsFilter * _filter)
  5374. : PARENT(a,start, owner), filter(_filter)
  5375. {
  5376. }
  5377. virtual bool first()
  5378. {
  5379. if (!PARENT::first())
  5380. return false;
  5381. if (matchesFilter())
  5382. return true;
  5383. return next();
  5384. }
  5385. virtual bool next()
  5386. {
  5387. for (;;)
  5388. {
  5389. if (!PARENT::next())
  5390. return false;
  5391. if (matchesFilter())
  5392. return true;
  5393. }
  5394. }
  5395. protected:
  5396. bool matchesFilter()
  5397. {
  5398. if (!filter)
  5399. return true;
  5400. return query().matches(filter);
  5401. }
  5402. protected:
  5403. Linked<const IStatisticsFilter> filter;
  5404. };
  5405. void CLocalWorkUnit::setStatistic(StatisticCreatorType creatorType, const char * creator, StatisticScopeType scopeType, const char * scope, StatisticKind kind, const char * optDescription, unsigned __int64 value, unsigned __int64 count, unsigned __int64 maxValue, StatsMergeAction mergeAction)
  5406. {
  5407. if (!scope || !*scope) scope = GLOBAL_SCOPE;
  5408. const char * kindName = queryStatisticName(kind);
  5409. StatisticMeasure measure = queryMeasure(kind);
  5410. //creator. scope and name must all be present, and must not contain semi colons.
  5411. assertex(creator && scope);
  5412. CriticalBlock block(crit);
  5413. IPropertyTree * stats = p->queryPropTree("Statistics");
  5414. if (!stats)
  5415. stats = p->addPropTree("Statistics");
  5416. IPropertyTree * statTree = NULL;
  5417. if (mergeAction != StatsMergeAppend)
  5418. {
  5419. StringBuffer xpath;
  5420. xpath.append("Statistic[@creator='").append(creator).append("'][@scope='").append(scope).append("'][@kind='").append(kindName).append("']");
  5421. statTree = stats->queryPropTree(xpath.str());
  5422. }
  5423. if (!statTree)
  5424. {
  5425. statTree = stats->addPropTree("Statistic");
  5426. statTree->setProp("@creator", creator);
  5427. statTree->setProp("@scope", scope);
  5428. statTree->setProp("@kind", kindName);
  5429. //These items are primarily here to facilitate filtering.
  5430. statTree->setProp("@unit", queryMeasureName(measure));
  5431. statTree->setProp("@c", queryCreatorTypeName(creatorType));
  5432. statTree->setProp("@s", queryScopeTypeName(scopeType));
  5433. statTree->setPropInt64("@ts", getTimeStampNowValue());
  5434. if (optDescription)
  5435. statTree->setProp("@desc", optDescription);
  5436. if (statistics.cached)
  5437. statistics.append(LINK(statTree));
  5438. mergeAction = StatsMergeAppend;
  5439. }
  5440. if (mergeAction != StatsMergeAppend) // RKC->GH Is this right??
  5441. {
  5442. unsigned __int64 oldValue = statTree->getPropInt64("@value", 0);
  5443. unsigned __int64 oldCount = statTree->getPropInt64("@count", 0);
  5444. unsigned __int64 oldMax = statTree->getPropInt64("@max", 0);
  5445. if (oldMax < oldValue)
  5446. oldMax = oldValue;
  5447. statTree->setPropInt64("@value", mergeStatisticValue(oldValue, value, mergeAction));
  5448. statTree->setPropInt64("@count", count + oldCount);
  5449. if (maxValue > oldMax)
  5450. statTree->setPropInt64("@max", maxValue);
  5451. }
  5452. else
  5453. {
  5454. statTree->setPropInt64("@value", value);
  5455. statTree->setPropInt64("@count", count);
  5456. if (maxValue)
  5457. statTree->setPropInt64("@max", maxValue);
  5458. else
  5459. statTree->removeProp("@max");
  5460. }
  5461. if (creatorType==SCTsummary && kind==StTimeElapsed && strsame(scope, GLOBAL_SCOPE))
  5462. {
  5463. StringBuffer t;
  5464. formatTimeCollatable(t, value, false);
  5465. p->setProp("@totalThorTime", t);
  5466. }
  5467. }
  5468. void CLocalWorkUnit::_loadStatistics() const
  5469. {
  5470. statistics.load(p,"Statistics/*");
  5471. }
  5472. IConstWUStatisticIterator& CLocalWorkUnit::getStatistics(const IStatisticsFilter * filter) const
  5473. {
  5474. CriticalBlock block(crit);
  5475. //This should be deleted in version 6.0 when support for 4.x is no longer required
  5476. legacyTimings.loadBranch(p,"Timings");
  5477. if (legacyTimings.ordinality())
  5478. return *new WorkUnitStatisticsIterator(legacyTimings, 0, (IConstWorkUnit *) this, filter);
  5479. statistics.loadBranch(p,"Statistics");
  5480. Owned<IConstWUStatisticIterator> localStats = new WorkUnitStatisticsIterator(statistics, 0, (IConstWorkUnit *) this, filter);
  5481. if (!filter->recurseChildScopes(SSTgraph, nullptr))
  5482. return *localStats.getClear();
  5483. const char * wuid = p->queryName();
  5484. Owned<IConstWUStatisticIterator> graphStats = new CConstGraphProgressStatisticsIterator(wuid, filter);
  5485. return * new CCompoundIteratorOf<IConstWUStatisticIterator, IConstWUStatistic>(localStats, graphStats);
  5486. }
  5487. IConstWUStatistic * CLocalWorkUnit::getStatistic(const char * creator, const char * scope, StatisticKind kind) const
  5488. {
  5489. //MORE: Optimize this....
  5490. StatisticsFilter filter;
  5491. filter.setCreator(creator);
  5492. filter.setScope(scope);
  5493. filter.setKind(kind);
  5494. Owned<IConstWUStatisticIterator> stats = &getStatistics(&filter);
  5495. if (stats->first())
  5496. return LINK(&stats->query());
  5497. return NULL;
  5498. }
  5499. IWUPlugin* CLocalWorkUnit::updatePluginByName(const char *qname)
  5500. {
  5501. CriticalBlock block(crit);
  5502. IConstWUPlugin *existing = getPluginByName(qname);
  5503. if (existing)
  5504. return (IWUPlugin *) existing;
  5505. if (!plugins.length())
  5506. p->addPropTree("Plugins");
  5507. IPropertyTree *pl = p->queryPropTree("Plugins");
  5508. IPropertyTree *s = pl->addPropTree("Plugin");
  5509. s->Link();
  5510. IWUPlugin* q = new CLocalWUPlugin(s);
  5511. q->Link();
  5512. plugins.append(*q);
  5513. q->setPluginName(qname);
  5514. return q;
  5515. }
  5516. IConstWUPlugin* CLocalWorkUnit::getPluginByName(const char *qname) const
  5517. {
  5518. CriticalBlock block(crit);
  5519. loadPlugins();
  5520. ForEachItemIn(idx, plugins)
  5521. {
  5522. SCMStringBuffer name;
  5523. IConstWUPlugin &cur = plugins.item(idx);
  5524. cur.getPluginName(name);
  5525. if (stricmp(name.str(), qname)==0)
  5526. {
  5527. cur.Link();
  5528. return &cur;
  5529. }
  5530. }
  5531. return NULL;
  5532. }
  5533. IWULibrary* CLocalWorkUnit::updateLibraryByName(const char *qname)
  5534. {
  5535. CriticalBlock block(crit);
  5536. IConstWULibrary *existing = getLibraryByName(qname);
  5537. if (existing)
  5538. return (IWULibrary *) existing;
  5539. if (!libraries.length())
  5540. p->addPropTree("Libraries");
  5541. IPropertyTree *pl = p->queryPropTree("Libraries");
  5542. IPropertyTree *s = pl->addPropTree("Library");
  5543. s->Link();
  5544. IWULibrary* q = new CLocalWULibrary(s);
  5545. q->Link();
  5546. libraries.append(*q);
  5547. q->setName(qname);
  5548. return q;
  5549. }
  5550. void CLocalWorkUnit::_loadExceptions() const
  5551. {
  5552. assertex(exceptions.length() == 0);
  5553. Owned<IPropertyTreeIterator> r = p->getElements("Exceptions/Exception");
  5554. for (r->first(); r->isValid(); r->next())
  5555. {
  5556. IPropertyTree *rp = &r->query();
  5557. rp->Link();
  5558. exceptions.append(*new CLocalWUException(rp));
  5559. }
  5560. }
  5561. void CLocalWorkUnit::loadExceptions() const
  5562. {
  5563. CriticalBlock block(crit);
  5564. if (!exceptionsCached)
  5565. {
  5566. _loadExceptions();
  5567. exceptionsCached = true;
  5568. }
  5569. }
  5570. IConstWUExceptionIterator& CLocalWorkUnit::getExceptions() const
  5571. {
  5572. CriticalBlock block(crit);
  5573. loadExceptions();
  5574. return *new CArrayIteratorOf<IConstWUException,IConstWUExceptionIterator> (exceptions, 0, (IConstWorkUnit *) this);
  5575. }
  5576. unsigned CLocalWorkUnit::getExceptionCount() const
  5577. {
  5578. CriticalBlock block(crit);
  5579. loadExceptions();
  5580. return exceptions.length();
  5581. }
  5582. void CLocalWorkUnit::clearExceptions()
  5583. {
  5584. CriticalBlock block(crit);
  5585. // For this to be legally called, we must have the write-able interface. So we are already locked for write.
  5586. loadExceptions();
  5587. ForEachItemInRev(idx, exceptions)
  5588. {
  5589. IWUException &e = exceptions.item(idx);
  5590. SCMStringBuffer s;
  5591. e.getExceptionSource(s);
  5592. if (strieq(s.s, "eclcc") || strieq(s.s, "eclccserver") || strieq(s.s, "eclserver") )
  5593. break;
  5594. VStringBuffer xpath("Exceptions/Exception[@sequence='%d']", e.getSequence());
  5595. p->removeProp(xpath);
  5596. exceptions.remove(idx);
  5597. }
  5598. }
  5599. IWUException* CLocalWorkUnit::createException()
  5600. {
  5601. CriticalBlock block(crit);
  5602. // For this to be legally called, we must have the write-able interface. So we are already locked for write.
  5603. loadExceptions();
  5604. if (!exceptions.length())
  5605. p->addPropTree("Exceptions");
  5606. IPropertyTree *r = p->queryPropTree("Exceptions");
  5607. IPropertyTree *s = r->addPropTree("Exception");
  5608. s->setPropInt("@sequence", exceptions.ordinality());
  5609. IWUException* q = new CLocalWUException(LINK(s));
  5610. exceptions.append(*LINK(q));
  5611. Owned<IJlibDateTime> now = createDateTimeNow();
  5612. SCMStringBuffer temp;
  5613. now->getString(temp);
  5614. q->setTimeStamp(temp.str());
  5615. return q;
  5616. }
  5617. IConstWUWebServicesInfo* CLocalWorkUnit::getWebServicesInfo() const
  5618. {
  5619. // For this to be legally called, we must have the read-able interface. So we are already locked for (at least) read.
  5620. CriticalBlock block(crit);
  5621. if (!webServicesInfoCached)
  5622. {
  5623. assertex(!webServicesInfo);
  5624. IPropertyTree *s = p->getPropTree("WebServicesInfo");
  5625. if (s)
  5626. webServicesInfo.setown(new CLocalWUWebServicesInfo(s)); // NB takes ownership of 's'
  5627. webServicesInfoCached = true;
  5628. }
  5629. return webServicesInfo.getLink();
  5630. }
  5631. IWUWebServicesInfo* CLocalWorkUnit::updateWebServicesInfo(bool create)
  5632. {
  5633. // For this to be legally called, we must have the write-able interface. So we are already locked for write.
  5634. CriticalBlock block(crit);
  5635. if (!webServicesInfoCached)
  5636. {
  5637. IPropertyTree *s = p->queryPropTree("WebServicesInfo");
  5638. if (!s)
  5639. {
  5640. if (create)
  5641. s = p->addPropTree("WebServicesInfo");
  5642. else
  5643. return NULL;
  5644. }
  5645. s->Link();
  5646. webServicesInfo.setown(new CLocalWUWebServicesInfo(s));
  5647. webServicesInfoCached = true;
  5648. }
  5649. return webServicesInfo.getLink();
  5650. }
  5651. static int compareResults(IInterface * const *ll, IInterface * const *rr)
  5652. {
  5653. CLocalWUResult *l = (CLocalWUResult *) *ll;
  5654. CLocalWUResult *r = (CLocalWUResult *) *rr;
  5655. return l->getResultSequence() - r->getResultSequence();
  5656. }
  5657. void CLocalWorkUnit::_loadResults() const
  5658. {
  5659. Owned<IPropertyTreeIterator> r = p->getElements("Results/Result");
  5660. for (r->first(); r->isValid(); r->next())
  5661. {
  5662. IPropertyTree *rp = &r->query();
  5663. rp->Link();
  5664. results.append(*new CLocalWUResult(rp));
  5665. }
  5666. }
  5667. void CLocalWorkUnit::loadResults() const
  5668. {
  5669. if (!resultsCached)
  5670. {
  5671. assertex(results.length() == 0);
  5672. _loadResults();
  5673. results.sort(compareResults);
  5674. resultsCached = true;
  5675. }
  5676. }
  5677. void CLocalWorkUnit::_loadVariables() const
  5678. {
  5679. Owned<IPropertyTreeIterator> r = p->getElements("Variables/Variable");
  5680. for (r->first(); r->isValid(); r->next())
  5681. {
  5682. IPropertyTree *rp = &r->query();
  5683. rp->Link();
  5684. variables.append(*new CLocalWUResult(rp));
  5685. }
  5686. }
  5687. void CLocalWorkUnit::loadVariables() const
  5688. {
  5689. if (!variablesCached)
  5690. {
  5691. assertex(variables.length() == 0);
  5692. _loadVariables();
  5693. variablesCached = true;
  5694. }
  5695. }
  5696. void CLocalWorkUnit::_loadTemporaries() const
  5697. {
  5698. Owned<IPropertyTreeIterator> r = p->getElements("Temporaries/Variable");
  5699. for (r->first(); r->isValid(); r->next())
  5700. {
  5701. IPropertyTree *rp = &r->query();
  5702. rp->Link();
  5703. temporaries.append(*new CLocalWUResult(rp));
  5704. }
  5705. }
  5706. void CLocalWorkUnit::loadTemporaries() const
  5707. {
  5708. if (!temporariesCached)
  5709. {
  5710. assertex(temporaries.length() == 0);
  5711. _loadTemporaries();
  5712. temporariesCached = true;
  5713. }
  5714. }
  5715. void CLocalWorkUnit::deleteTemporaries()
  5716. {
  5717. CriticalBlock block(crit);
  5718. if (temporariesCached)
  5719. {
  5720. temporaries.kill();
  5721. temporariesCached = false;
  5722. }
  5723. p->removeProp("Temporaries");
  5724. }
  5725. IWUResult* CLocalWorkUnit::createResult()
  5726. {
  5727. CriticalBlock block(crit);
  5728. // For this to be legally called, we must have the write-able interface. So we are already locked for write.
  5729. loadResults();
  5730. if (!results.length())
  5731. p->addPropTree("Results");
  5732. IPropertyTree *r = p->queryPropTree("Results");
  5733. IPropertyTree *s = r->addPropTree("Result");
  5734. s->Link();
  5735. IWUResult* q = new CLocalWUResult(s);
  5736. q->Link();
  5737. results.append(*q);
  5738. return q;
  5739. }
  5740. IWUResult* CLocalWorkUnit::updateResultByName(const char *qname)
  5741. {
  5742. CriticalBlock block(crit);
  5743. IConstWUResult *existing = getResultByName(qname);
  5744. if (existing)
  5745. return (IWUResult *) existing;
  5746. IWUResult* q = createResult();
  5747. q->setResultName(qname);
  5748. return q;
  5749. }
  5750. IWUResult* CLocalWorkUnit::updateResultBySequence(unsigned seq)
  5751. {
  5752. CriticalBlock block(crit);
  5753. IConstWUResult *existing = getResultBySequence(seq);
  5754. if (existing)
  5755. return (IWUResult *) existing;
  5756. IWUResult* q = createResult();
  5757. q->setResultSequence(seq);
  5758. return q;
  5759. }
  5760. IConstWUResultIterator& CLocalWorkUnit::getResults() const
  5761. {
  5762. CriticalBlock block(crit);
  5763. loadResults();
  5764. return *new CArrayIteratorOf<IConstWUResult,IConstWUResultIterator> (results, 0, (IConstWorkUnit *) this);
  5765. }
  5766. IConstWUResult* CLocalWorkUnit::getResultByName(const char *qname) const
  5767. {
  5768. CriticalBlock block(crit);
  5769. loadResults();
  5770. ForEachItemIn(idx, results)
  5771. {
  5772. SCMStringBuffer name;
  5773. IConstWUResult &cur = results.item(idx);
  5774. cur.getResultName(name);
  5775. if (stricmp(name.str(), qname)==0)
  5776. {
  5777. cur.Link();
  5778. return &cur;
  5779. }
  5780. }
  5781. return NULL;
  5782. }
  5783. IConstWUResult* CLocalWorkUnit::getResultBySequence(unsigned seq) const
  5784. {
  5785. CriticalBlock block(crit);
  5786. loadResults();
  5787. ForEachItemIn(idx, results)
  5788. {
  5789. IConstWUResult &cur = results.item(idx);
  5790. if (cur.getResultSequence() == seq)
  5791. {
  5792. cur.Link();
  5793. return &cur;
  5794. }
  5795. }
  5796. return NULL;
  5797. }
  5798. IConstWUResultIterator& CLocalWorkUnit::getVariables() const
  5799. {
  5800. CriticalBlock block(crit);
  5801. loadVariables();
  5802. return *new CArrayIteratorOf<IConstWUResult,IConstWUResultIterator> (variables, 0, (IConstWorkUnit *) this);
  5803. }
  5804. IConstWUResult* CLocalWorkUnit::getGlobalByName(const char *qname) const
  5805. {
  5806. CriticalBlock block(crit);
  5807. if (strcmp(p->queryName(), GLOBAL_WORKUNIT)==0)
  5808. return getVariableByName(qname);
  5809. Owned <IWorkUnit> global = factory->getGlobalWorkUnit(secMgr, secUser);
  5810. return global->getVariableByName(qname);
  5811. }
  5812. IWUResult* CLocalWorkUnit::updateGlobalByName(const char *qname)
  5813. {
  5814. CriticalBlock block(crit);
  5815. if (strcmp(p->queryName(), GLOBAL_WORKUNIT)==0)
  5816. return updateVariableByName(qname);
  5817. Owned <IWorkUnit> global = factory->getGlobalWorkUnit(secMgr, secUser);
  5818. return global->updateVariableByName(qname);
  5819. }
  5820. IConstWUResult* CLocalWorkUnit::getVariableByName(const char *qname) const
  5821. {
  5822. CriticalBlock block(crit);
  5823. loadVariables();
  5824. ForEachItemIn(idx, variables)
  5825. {
  5826. SCMStringBuffer name;
  5827. IConstWUResult &cur = variables.item(idx);
  5828. cur.getResultName(name);
  5829. if (stricmp(name.str(), qname)==0)
  5830. {
  5831. cur.Link();
  5832. return &cur;
  5833. }
  5834. }
  5835. return NULL;
  5836. }
  5837. IConstWUResult* CLocalWorkUnit::getTemporaryByName(const char *qname) const
  5838. {
  5839. CriticalBlock block(crit);
  5840. loadTemporaries();
  5841. ForEachItemIn(idx, temporaries)
  5842. {
  5843. SCMStringBuffer name;
  5844. IConstWUResult &cur = temporaries.item(idx);
  5845. cur.getResultName(name);
  5846. if (stricmp(name.str(), qname)==0)
  5847. {
  5848. cur.Link();
  5849. return &cur;
  5850. }
  5851. }
  5852. return NULL;
  5853. }
  5854. IConstWUResultIterator& CLocalWorkUnit::getTemporaries() const
  5855. {
  5856. CriticalBlock block(crit);
  5857. loadTemporaries();
  5858. return *new CArrayIteratorOf<IConstWUResult,IConstWUResultIterator> (temporaries, 0, (IConstWorkUnit *) this);
  5859. }
  5860. IWUResult* CLocalWorkUnit::updateTemporaryByName(const char *qname)
  5861. {
  5862. CriticalBlock block(crit);
  5863. IConstWUResult *existing = getTemporaryByName(qname);
  5864. if (existing)
  5865. return (IWUResult *) existing;
  5866. IPropertyTree *vars = (temporaries.length()) ? p->queryPropTree("Temporaries") : p->addPropTree("Temporaries");
  5867. IPropertyTree *s = vars->addPropTree("Variable");
  5868. s->Link();
  5869. IWUResult* q = new CLocalWUResult(s);
  5870. q->Link();
  5871. temporaries.append(*q);
  5872. q->setResultName(qname);
  5873. q->setResultSequence(ResultSequenceInternal);
  5874. return q;
  5875. }
  5876. IWUResult* CLocalWorkUnit::updateVariableByName(const char *qname)
  5877. {
  5878. CriticalBlock block(crit);
  5879. IConstWUResult *existing = getVariableByName(qname);
  5880. if (existing)
  5881. return (IWUResult *) existing;
  5882. if (!variables.length())
  5883. p->addPropTree("Variables");
  5884. IPropertyTree *vars = p->queryPropTree("Variables");
  5885. IPropertyTree *s = vars->addPropTree("Variable");
  5886. s->Link();
  5887. IWUResult* q = new CLocalWUResult(s);
  5888. q->Link();
  5889. variables.append(*q);
  5890. q->setResultName(qname);
  5891. q->setResultSequence(ResultSequenceStored);
  5892. return q;
  5893. }
  5894. void CLocalWorkUnit::deleteTempFiles(const char *graph, bool deleteOwned, bool deleteJobOwned)
  5895. {
  5896. CriticalBlock block(crit);
  5897. IPropertyTree *files = p->queryPropTree("Files");
  5898. if (!files) return;
  5899. Owned<IPropertyTreeIterator> iter = files->getElements("File");
  5900. ICopyArrayOf<IPropertyTree> toRemove;
  5901. ForEach (*iter)
  5902. {
  5903. IPropertyTree &file = iter->query();
  5904. WUFileKind fileKind = (WUFileKind) file.getPropInt("@kind", WUFileStandard);
  5905. if(file.getPropBool("@temporary")) fileKind = WUFileTemporary; // @temporary, legacy check
  5906. bool needDelete;
  5907. switch(fileKind)
  5908. {
  5909. case WUFileTemporary:
  5910. if(graph==NULL)
  5911. needDelete = true;
  5912. else
  5913. {
  5914. const char *graphOwner = file.queryProp("@graph");
  5915. needDelete = ((graphOwner==NULL) || (strcmp(graph, graphOwner)==0));
  5916. }
  5917. break;
  5918. case WUFileJobOwned:
  5919. needDelete = ((graph==NULL) && deleteJobOwned);
  5920. break;
  5921. case WUFileOwned:
  5922. needDelete = ((graph==NULL) && deleteOwned);
  5923. break;
  5924. default:
  5925. needDelete = false;
  5926. }
  5927. if(needDelete)
  5928. {
  5929. const char *name = file.queryProp("@name");
  5930. LOG(MCdebugProgress, unknownJob, "Removing workunit file %s from DFS", name);
  5931. queryDistributedFileDirectory().removeEntry(name, queryUserDescriptor());
  5932. toRemove.append(file);
  5933. }
  5934. }
  5935. ForEachItemIn(r, toRemove) files->removeTree(&toRemove.item(r));
  5936. }
  5937. static void _noteFileRead(IDistributedFile *file, IPropertyTree *filesRead)
  5938. {
  5939. IDistributedSuperFile *super = file->querySuperFile();
  5940. StringBuffer fname;
  5941. file->getLogicalName(fname);
  5942. if (fname.length())
  5943. {
  5944. StringBuffer path("File[@name=\"");
  5945. path.append(fname).append("\"]");
  5946. IPropertyTree *fileTree = filesRead->queryPropTree(path.str());
  5947. if (fileTree)
  5948. fileTree->setPropInt("@useCount", fileTree->getPropInt("@useCount")+1);
  5949. else
  5950. {
  5951. StringBuffer cluster;
  5952. file->getClusterName(0,cluster);
  5953. fileTree = createPTree();
  5954. fileTree->setProp("@name", fname.str());
  5955. fileTree->setProp("@cluster", cluster.str());
  5956. fileTree->setPropInt("@useCount", 1);
  5957. fileTree = filesRead->addPropTree("File", fileTree);
  5958. }
  5959. if (super)
  5960. {
  5961. fileTree->setPropBool("@super", true);
  5962. Owned<IDistributedFileIterator> iter = super->getSubFileIterator(false);
  5963. ForEach (*iter)
  5964. {
  5965. IDistributedFile &file = iter->query();
  5966. StringBuffer fname;
  5967. file.getLogicalName(fname);
  5968. fileTree->addPropTree("Subfile")->setProp("@name", fname.str());
  5969. _noteFileRead(&file, filesRead);
  5970. }
  5971. }
  5972. }
  5973. }
  5974. void CLocalWorkUnit::_loadFilesRead() const
  5975. {
  5976. // Nothing to do
  5977. }
  5978. void CLocalWorkUnit::noteFileRead(IDistributedFile *file)
  5979. {
  5980. if (file)
  5981. {
  5982. CriticalBlock block(crit);
  5983. IPropertyTree *files = p->queryPropTree("FilesRead");
  5984. if (!files)
  5985. files = p->addPropTree("FilesRead");
  5986. _noteFileRead(file, files);
  5987. }
  5988. }
  5989. void CLocalWorkUnit::noteFieldUsage(IPropertyTree * fieldUsage)
  5990. {
  5991. if (fieldUsage)
  5992. {
  5993. CriticalBlock block(crit);
  5994. p->addPropTree("usedsources", fieldUsage);
  5995. }
  5996. }
  5997. void CLocalWorkUnit::_loadFilesWritten() const
  5998. {
  5999. // Nothing to do
  6000. }
  6001. static void addFile(IPropertyTree *files, const char *fileName, const char *cluster, unsigned usageCount, WUFileKind fileKind, const char *graphOwner)
  6002. {
  6003. StringBuffer path("File[@name=\"");
  6004. path.append(fileName).append("\"]");
  6005. if (cluster)
  6006. path.append("[@cluster=\"").append(cluster).append("\"]");
  6007. IPropertyTree *file = files->queryPropTree(path.str());
  6008. if (file) files->removeTree(file);
  6009. file = createPTree();
  6010. file->setProp("@name", fileName);
  6011. if (cluster)
  6012. file->setProp("@cluster", cluster);
  6013. if (graphOwner)
  6014. file->setProp("@graph", graphOwner);
  6015. file->setPropInt("@kind", (unsigned)fileKind);
  6016. if (WUFileTemporary == fileKind)
  6017. file->setPropInt("@usageCount", usageCount);
  6018. files->addPropTree("File", file);
  6019. }
  6020. void CLocalWorkUnit::addFile(const char *fileName, StringArray *clusters, unsigned usageCount, WUFileKind fileKind, const char *graphOwner)
  6021. {
  6022. CriticalBlock block(crit);
  6023. IPropertyTree *files = p->queryPropTree("Files");
  6024. if (!files)
  6025. files = p->addPropTree("Files");
  6026. if (!clusters)
  6027. ::addFile(files, fileName, NULL, usageCount, fileKind, graphOwner);
  6028. else
  6029. {
  6030. ForEachItemIn(c, *clusters)
  6031. ::addFile(files, fileName, clusters->item(c), usageCount, fileKind, graphOwner);
  6032. }
  6033. }
  6034. void CLocalWorkUnit::releaseFile(const char *fileName)
  6035. {
  6036. StringBuffer path("File[@name=\"");
  6037. path.append(fileName).append("\"]");
  6038. CriticalBlock block(crit);
  6039. IPropertyTree *files = p->queryPropTree("Files");
  6040. if (!files) return;
  6041. Owned<IPropertyTreeIterator> fiter = files->getElements(path.str());
  6042. ForEach (*fiter)
  6043. {
  6044. IPropertyTree *file = &fiter->query();
  6045. unsigned usageCount = file->getPropInt("@usageCount");
  6046. if (usageCount > 1)
  6047. file->setPropInt("@usageCount", usageCount-1);
  6048. else
  6049. {
  6050. StringAttr name(file->queryProp("@name"));
  6051. files->removeTree(file);
  6052. if (!name.isEmpty()&&(1 == usageCount))
  6053. {
  6054. if (queryDistributedFileDirectory().removeEntry(fileName, queryUserDescriptor()))
  6055. LOG(MCdebugProgress, unknownJob, "Removed (released) file %s from DFS", name.get());
  6056. }
  6057. }
  6058. }
  6059. }
  6060. void CLocalWorkUnit::clearGraphProgress() const
  6061. {
  6062. }
  6063. void CLocalWorkUnit::resetBeforeGeneration()
  6064. {
  6065. CriticalBlock block(crit);
  6066. //Remove all associated files
  6067. Owned<IWUQuery> q = updateQuery();
  6068. q->removeAssociatedFiles();
  6069. //Remove any pre-existing workflow information
  6070. workflowIterator.clear();
  6071. p->removeProp("Workflow");
  6072. }
  6073. unsigned CLocalWorkUnit::queryFileUsage(const char *fileName) const
  6074. {
  6075. StringBuffer path("Files/File[@name=\"");
  6076. path.append(fileName).append("\"]/@usageCount");
  6077. CriticalBlock block(crit);
  6078. return p->getPropInt(path.str());
  6079. }
  6080. IConstWUFileUsageIterator * CLocalWorkUnit::getFieldUsage() const
  6081. {
  6082. CriticalBlock block(crit);
  6083. IPropertyTree* fieldUsageTree = p->queryPropTree("usedsources");
  6084. if (!fieldUsageTree)
  6085. return NULL;
  6086. IPropertyTreeIterator* iter = fieldUsageTree->getElements("*");
  6087. return new CConstWUFileUsageIterator(iter);
  6088. }
  6089. bool isFilenameResolved(StringBuffer& filename)
  6090. {
  6091. size32_t length = filename.length();
  6092. // With current implementation, if filename is surrounded by single quotes, it means that the filename was resolved at compile time.
  6093. if (filename.length() >= 2 && filename.charAt(0) == '\'' && filename.charAt(length-1) == '\'')
  6094. return true;
  6095. else
  6096. return false;
  6097. }
  6098. bool CLocalWorkUnit::getFieldUsageArray(StringArray & filenames, StringArray & columnnames, const char * clusterName) const
  6099. {
  6100. bool scopeLoaded = false;
  6101. SCMStringBuffer defaultScope;
  6102. Owned<IConstWUFileUsageIterator> files = getFieldUsage();
  6103. if (!files)
  6104. return false; // this query was not compiled with recordFieldUsage option.
  6105. ForEach(*files)
  6106. {
  6107. Owned<IConstWUFileUsage> file = files->get();
  6108. StringBuffer filename = file->queryName();
  6109. size32_t length = filename.length();
  6110. if (length == 0)
  6111. throw MakeStringException(WUERR_InvalidFieldUsage, "Invalid FieldUsage found in WU. Cannot enforce view security.");
  6112. StringBuffer normalizedFilename;
  6113. // Two cases to handle:
  6114. // 1. Filename was known at compile time, and is surrounded in single quotes (i.e. 'filename').
  6115. // 2. Filename could not be resolved at compile time (i.e. filename is an input to a query),
  6116. // and is a raw expression WITHOUT surrounding single quotes (i.e. STORED('input_filename')).
  6117. if (isFilenameResolved(filename))
  6118. {
  6119. // filename cannot be empty (i.e. empty single quotes '')
  6120. if (length == 2)
  6121. throw MakeStringException(WUERR_InvalidFieldUsage, "Invalid FieldUsage found in WU. Cannot enforce view security.");
  6122. // Remove surrounding single quotes
  6123. StringAttr cleanFilename(filename.str()+1, length-2);
  6124. // When a filename doesn't start with a tilde (~), it means scope is omitted and is relying on a default scope.
  6125. // We need to load a default scope from config and prefix the filename with it.
  6126. if (cleanFilename.str()[0] != '~')
  6127. {
  6128. // loading a default scope from config is expensive, and should be only done once and be reused later.
  6129. if (!scopeLoaded)
  6130. {
  6131. Owned<IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(clusterName);
  6132. if (!clusterInfo)
  6133. throw MakeStringException(WUERR_InvalidCluster, "Unknown cluster %s", clusterName);
  6134. clusterInfo->getScope(defaultScope);
  6135. scopeLoaded = true;
  6136. }
  6137. normalizedFilename.append(defaultScope.str());
  6138. normalizedFilename.append(cleanFilename.str());
  6139. }
  6140. else
  6141. {
  6142. normalizedFilename.append(cleanFilename);
  6143. }
  6144. }
  6145. else
  6146. {
  6147. // When filename is an unresolved expression, simply treat the expression as a "non-existent" filename.
  6148. // It will have an effect of this query accessing a non-existent filename, and will be denied access unconditionally.
  6149. normalizedFilename.append(filename.str());
  6150. }
  6151. Owned<IConstWUFieldUsageIterator> fields = file->getFields();
  6152. ForEach(*fields)
  6153. {
  6154. Owned<IConstWUFieldUsage> field = fields->get();
  6155. filenames.append(normalizedFilename.str());
  6156. columnnames.append(field->queryName());
  6157. }
  6158. }
  6159. return true;
  6160. }
  6161. IPropertyTree *CLocalWorkUnit::getDiskUsageStats()
  6162. {
  6163. return p->getPropTree("DiskUsageStats");
  6164. }
  6165. void CLocalWorkUnit::addDiskUsageStats(__int64 _avgNodeUsage, unsigned _minNode, __int64 _minNodeUsage, unsigned _maxNode, __int64 _maxNodeUsage, __int64 _graphId)
  6166. {
  6167. IPropertyTree *stats = p->queryPropTree("DiskUsageStats");
  6168. offset_t maxNodeUsage;
  6169. if (stats)
  6170. maxNodeUsage = stats->getPropInt64("@maxNodeUsage");
  6171. else
  6172. {
  6173. stats = p->addPropTree("DiskUsageStats");
  6174. maxNodeUsage = 0;
  6175. }
  6176. if ((offset_t)_maxNodeUsage > maxNodeUsage)
  6177. {
  6178. // record all details at time of max node usage.
  6179. stats->setPropInt("@minNode", _minNode);
  6180. stats->setPropInt("@maxNode", _maxNode);
  6181. stats->setPropInt64("@minNodeUsage", _minNodeUsage);
  6182. stats->setPropInt64("@maxNodeUsage", _maxNodeUsage);
  6183. stats->setPropInt64("@graphId", _graphId);
  6184. if (_avgNodeUsage)
  6185. {
  6186. unsigned _skewHi = (unsigned)((100 * (_maxNodeUsage-_avgNodeUsage))/_avgNodeUsage);
  6187. unsigned _skewLo = (unsigned)((100 * (_avgNodeUsage-_minNodeUsage))/_avgNodeUsage);
  6188. stats->setPropInt("@skewHi", _skewHi);
  6189. stats->setPropInt("@skewLo", _skewLo);
  6190. }
  6191. }
  6192. }
  6193. IPropertyTreeIterator & CLocalWorkUnit::getFileIterator() const
  6194. {
  6195. CriticalBlock block(crit);
  6196. _loadFilesWritten();
  6197. return * p->getElements("Files/File");
  6198. }
  6199. IPropertyTreeIterator & CLocalWorkUnit::getFilesReadIterator() const
  6200. {
  6201. CriticalBlock block(crit);
  6202. _loadFilesRead();
  6203. return * p->getElements("FilesRead/File");
  6204. }
  6205. //=================================================================================================
  6206. bool CLocalWorkUnit::switchThorQueue(const char *cluster, IQueueSwitcher *qs)
  6207. {
  6208. CriticalBlock block(crit);
  6209. if (qs->isAuto()&&!getAllowAutoQueueSwitch())
  6210. return false;
  6211. Owned<IConstWUClusterInfo> newci = getTargetClusterInfo(cluster);
  6212. if (!newci)
  6213. return false;
  6214. StringBuffer currentcluster;
  6215. if (!p->getProp("@clusterName",currentcluster))
  6216. return false;
  6217. Owned<IConstWUClusterInfo> curci = getTargetClusterInfo(currentcluster.str());
  6218. if (!curci)
  6219. return false;
  6220. SCMStringBuffer curqname;
  6221. curci->getThorQueue(curqname);
  6222. const char *wuid = p->queryName();
  6223. void *qi = qs->getQ(curqname.str(),wuid);
  6224. if (!qi)
  6225. return false;
  6226. setClusterName(cluster);
  6227. SCMStringBuffer newqname;
  6228. newci->getThorQueue(newqname);
  6229. qs->putQ(newqname.str(),wuid,qi);
  6230. return true;
  6231. }
  6232. //=================================================================================================
  6233. IPropertyTree *CLocalWorkUnit::getUnpackedTree(bool includeProgress) const
  6234. {
  6235. Owned<IPropertyTree> ret = createPTreeFromIPT(p);
  6236. Owned<IConstWUGraphIterator> graphIter = &getGraphs(GraphTypeAny);
  6237. ForEach(*graphIter)
  6238. {
  6239. IConstWUGraph &graph = graphIter->query();
  6240. Owned<IPropertyTree> graphTree = graph.getXGMMLTree(includeProgress);
  6241. SCMStringBuffer gName;
  6242. graph.getName(gName);
  6243. StringBuffer xpath("Graphs/Graph[@name=\"");
  6244. xpath.append(gName.s).append("\"]/xgmml");
  6245. IPropertyTree *xgmml = ret->queryPropTree(xpath.str());
  6246. if (xgmml) // don't know of any reason it shouldn't exist
  6247. {
  6248. xgmml->removeProp("graphBin");
  6249. xgmml->setPropTree("graph", graphTree.getClear());
  6250. }
  6251. }
  6252. return ret.getClear();
  6253. }
  6254. void CLocalWorkUnit::_loadGraphs(bool heavy) const
  6255. {
  6256. Owned<IPropertyTreeIterator> iter = p->getElements("Graphs/Graph");
  6257. ForEach(*iter)
  6258. {
  6259. IPropertyTree &graph = iter->query();
  6260. graphs.append(*new CLocalWUGraph(*this, LINK(&graph)));
  6261. }
  6262. }
  6263. void CLocalWorkUnit::loadGraphs(bool heavy) const
  6264. {
  6265. if (graphsCached < (heavy ? 2 : 1))
  6266. {
  6267. graphs.kill();
  6268. _loadGraphs(heavy);
  6269. graphsCached = (heavy ? 2 : 1);
  6270. }
  6271. }
  6272. mapEnums graphTypes[] = {
  6273. { GraphTypeAny, "unknown" },
  6274. { GraphTypeProgress, "progress" },
  6275. { GraphTypeEcl, "ECL" },
  6276. { GraphTypeActivities, "activities" },
  6277. { GraphTypeSubProgress, "subgraph" },
  6278. { GraphTypeSize, NULL },
  6279. };
  6280. WUGraphType getGraphTypeFromString(const char* type)
  6281. {
  6282. return (WUGraphType) getEnum(type, graphTypes);
  6283. }
  6284. CLocalWUGraph::CLocalWUGraph(const CLocalWorkUnit &_owner, IPropertyTree *props) : p(props), owner(_owner)
  6285. {
  6286. wuidVersion = owner.getWuidVersion();
  6287. }
  6288. IStringVal& CLocalWUGraph::getName(IStringVal &str) const
  6289. {
  6290. str.set(p->queryProp("@name"));
  6291. return str;
  6292. }
  6293. IStringVal& CLocalWUGraph::getLabel(IStringVal &str) const
  6294. {
  6295. if (wuidVersion >= 2)
  6296. {
  6297. str.set(p->queryProp("@label"));
  6298. return str;
  6299. }
  6300. else
  6301. {
  6302. Owned<IPropertyTree> xgmml = getXGMMLTree(false);
  6303. str.set(xgmml->queryProp("@label"));
  6304. return str;
  6305. }
  6306. }
  6307. WUGraphState CLocalWUGraph::getState() const
  6308. {
  6309. return owner.queryGraphState(p->queryProp("@name"));
  6310. }
  6311. IStringVal& CLocalWUGraph::getXGMML(IStringVal &str, bool mergeProgress) const
  6312. {
  6313. Owned<IPropertyTree> xgmml = getXGMMLTree(mergeProgress);
  6314. if (xgmml)
  6315. {
  6316. StringBuffer x;
  6317. toXML(xgmml, x);
  6318. str.set(x.str());
  6319. }
  6320. return str;
  6321. }
  6322. unsigned CLocalWorkUnit::getGraphCount() const
  6323. {
  6324. CriticalBlock block(crit);
  6325. if (p->hasProp("Graphs"))
  6326. {
  6327. return p->queryPropTree("Graphs")->numChildren();
  6328. }
  6329. return 0;
  6330. }
  6331. unsigned CLocalWorkUnit::getSourceFileCount() const
  6332. {
  6333. CriticalBlock block(crit);
  6334. _loadFilesRead();
  6335. if (p->hasProp("FilesRead"))
  6336. {
  6337. return p->queryPropTree("FilesRead")->numChildren();
  6338. }
  6339. return 0;
  6340. }
  6341. unsigned CLocalWorkUnit::getResultCount() const
  6342. {
  6343. CriticalBlock block(crit);
  6344. if (p->hasProp("Results"))
  6345. {
  6346. return p->queryPropTree("Results")->numChildren();
  6347. }
  6348. return 0;
  6349. }
  6350. unsigned CLocalWorkUnit::getVariableCount() const
  6351. {
  6352. CriticalBlock block(crit);
  6353. if (p->hasProp("Variables"))
  6354. {
  6355. return p->queryPropTree("Variables")->numChildren();
  6356. }
  6357. return 0;
  6358. }
  6359. unsigned CLocalWorkUnit::getApplicationValueCount() const
  6360. {
  6361. CriticalBlock block(crit);
  6362. if (p->hasProp("Application"))
  6363. {
  6364. return p->queryPropTree("Application")->numChildren();
  6365. }
  6366. return 0;
  6367. }
  6368. StringBuffer &appendPTreeOpenTag(StringBuffer &s, IPropertyTree *tree, const char *name, unsigned indent, bool hidePasswords)
  6369. {
  6370. appendXMLOpenTag(s, name, NULL, false);
  6371. Owned<IAttributeIterator> attrs = tree->getAttributes(true);
  6372. if (attrs->first())
  6373. {
  6374. unsigned attributeindent = indent + (size32_t) strlen(name);
  6375. unsigned count = attrs->count();
  6376. bool doindent = false;
  6377. ForEach(*attrs)
  6378. {
  6379. if (hidePasswords && streq(attrs->queryName(), "@token"))
  6380. continue;
  6381. if (doindent)
  6382. s.append('\n').appendN(attributeindent, ' ');
  6383. else if (count > 3)
  6384. doindent = true;
  6385. appendXMLAttr(s, attrs->queryName()+1, attrs->queryValue());
  6386. }
  6387. }
  6388. s.append('>');
  6389. return s;
  6390. }
  6391. IStringVal &CLocalWorkUnit::getXmlParams(IStringVal &str, bool hidePasswords) const
  6392. {
  6393. CriticalBlock block(crit);
  6394. IPropertyTree *paramTree = p->queryPropTree("Parameters");
  6395. if (!paramTree)
  6396. return str;
  6397. StringBuffer xml;
  6398. if (!hidePasswords)
  6399. toXML(paramTree, xml);
  6400. else
  6401. {
  6402. appendPTreeOpenTag(xml.append(' '), paramTree, "Parameters", 0, false).append('\n');
  6403. Owned<IPropertyTreeIterator> elems = paramTree->getElements("*");
  6404. ForEach(*elems)
  6405. {
  6406. const char *paramname = elems->query().queryName();
  6407. VStringBuffer xpath("Variables/Variable[@name='%s']/Format/@password", paramname);
  6408. if (p->getPropBool(xpath))
  6409. appendXMLTag(xml.append(" "), paramname, "***").append('\n');
  6410. else
  6411. toXML(&elems->query(), xml, 2);
  6412. }
  6413. appendXMLCloseTag(xml.append(' '), "Parameters").append('\n');
  6414. }
  6415. str.set(xml);
  6416. return str;
  6417. }
  6418. const IPropertyTree *CLocalWorkUnit::getXmlParams() const
  6419. {
  6420. CriticalBlock block(crit);
  6421. return p->getPropTree("Parameters");
  6422. }
  6423. void CLocalWorkUnit::setXmlParams(const char *params)
  6424. {
  6425. CriticalBlock block(crit);
  6426. p->setPropTree("Parameters", createPTreeFromXMLString(params));
  6427. }
  6428. void CLocalWorkUnit::setXmlParams(IPropertyTree *tree)
  6429. {
  6430. CriticalBlock block(crit);
  6431. p->setPropTree("Parameters", tree);
  6432. }
  6433. unsigned __int64 CLocalWorkUnit::getHash() const
  6434. {
  6435. CriticalBlock block(crit);
  6436. return p->getPropInt64("@hash");
  6437. }
  6438. void CLocalWorkUnit::setHash(unsigned __int64 hash)
  6439. {
  6440. CriticalBlock block(crit);
  6441. p->setPropInt64("@hash", hash);
  6442. }
  6443. // getGraphs / getGraphsMeta
  6444. // These are basically the same except for the amount of preloading they do, and the type of the iterator they return...
  6445. // If a type other than any is requested, a postfilter is needed.
  6446. template <class T, class U> class CFilteredGraphIteratorOf : public CInterfaceOf<T>
  6447. {
  6448. WUGraphType type;
  6449. Owned<T> base;
  6450. bool match()
  6451. {
  6452. return base->query().getType()==type;
  6453. }
  6454. public:
  6455. CFilteredGraphIteratorOf<T,U>(T *_base, WUGraphType _type)
  6456. : base(_base), type(_type)
  6457. {
  6458. }
  6459. bool first()
  6460. {
  6461. if (!base->first())
  6462. return false;
  6463. if (match())
  6464. return true;
  6465. return next();
  6466. }
  6467. bool next()
  6468. {
  6469. while (base->next())
  6470. if (match())
  6471. return true;
  6472. return false;
  6473. }
  6474. virtual bool isValid()
  6475. {
  6476. return base->isValid();
  6477. }
  6478. U & query()
  6479. {
  6480. return base->query();
  6481. }
  6482. };
  6483. IConstWUGraphMetaIterator& CLocalWorkUnit::getGraphsMeta(WUGraphType type) const
  6484. {
  6485. /* NB: this method should be 'cheap', loadGraphs() creates IConstWUGraph interfaces to the graphs
  6486. * it does not actually pull the graph data. We only use IConstWUGraphMeta here, which never probes the xgmml.
  6487. */
  6488. CriticalBlock block(crit);
  6489. loadGraphs(false);
  6490. IConstWUGraphMetaIterator *giter = new CArrayIteratorOf<IConstWUGraph,IConstWUGraphMetaIterator> (graphs, 0, (IConstWorkUnit *) this);
  6491. if (type!=GraphTypeAny)
  6492. giter = new CFilteredGraphIteratorOf<IConstWUGraphMetaIterator, IConstWUGraphMeta>(giter,type);
  6493. return *giter;
  6494. }
  6495. IConstWUGraphIterator& CLocalWorkUnit::getGraphs(WUGraphType type) const
  6496. {
  6497. CriticalBlock block(crit);
  6498. loadGraphs(true);
  6499. IConstWUGraphIterator *giter = new CArrayIteratorOf<IConstWUGraph,IConstWUGraphIterator> (graphs, 0, (IConstWorkUnit *) this);
  6500. if (type!=GraphTypeAny)
  6501. giter = new CFilteredGraphIteratorOf<IConstWUGraphIterator, IConstWUGraph>(giter, type);
  6502. return *giter;
  6503. }
  6504. IConstWUGraph* CLocalWorkUnit::getGraph(const char *qname) const
  6505. {
  6506. CriticalBlock block(crit);
  6507. VStringBuffer xpath("Graphs/Graph[@name='%s']", qname);
  6508. // NOTE - this would go wrong if we had other graphs of same name but different type. Ignore for now.
  6509. IPTree *graph = p->queryPropTree(xpath);
  6510. if (graph)
  6511. return new CLocalWUGraph(*this, LINK(graph));
  6512. return NULL;
  6513. }
  6514. void CLocalWorkUnit::createGraph(const char * name, const char *label, WUGraphType type, IPropertyTree *xgmml)
  6515. {
  6516. CriticalBlock block(crit);
  6517. if (!graphs.length())
  6518. p->addPropTree("Graphs");
  6519. IPropertyTree *r = p->queryPropTree("Graphs");
  6520. IPropertyTree *s = r->addPropTree("Graph");
  6521. CLocalWUGraph *q = new CLocalWUGraph(*this, LINK(s));
  6522. q->setName(name);
  6523. q->setLabel(label);
  6524. q->setType(type);
  6525. q->setXGMMLTree(xgmml);
  6526. graphs.append(*q);
  6527. }
  6528. IConstWUGraphProgress *CLocalWorkUnit::getGraphProgress(const char *name) const
  6529. {
  6530. /* Owned<IRemoteConnection> conn = getProgressConnection();
  6531. if (conn)
  6532. {
  6533. IPTree *progress = conn->queryRoot()->queryPropTree(graphName);
  6534. if (progress)
  6535. return new CConstGraphProgress(p->queryName(), graphName, progress);
  6536. }
  6537. */
  6538. return NULL;
  6539. }
  6540. WUGraphState CLocalWorkUnit::queryGraphState(const char *graphName) const
  6541. {
  6542. throwUnexpected(); // Should only be used for persisted workunits
  6543. }
  6544. WUGraphState CLocalWorkUnit::queryNodeState(const char *graphName, WUGraphIDType nodeId) const
  6545. {
  6546. throwUnexpected(); // Should only be used for persisted workunits
  6547. }
  6548. void CLocalWorkUnit::setGraphState(const char *graphName, WUGraphState state) const
  6549. {
  6550. throwUnexpected(); // Should only be used for persisted workunits
  6551. }
  6552. void CLocalWorkUnit::setNodeState(const char *graphName, WUGraphIDType nodeId, WUGraphState state) const
  6553. {
  6554. throwUnexpected(); // Should only be used for persisted workunits
  6555. }
  6556. IWUGraphStats *CLocalWorkUnit::updateStats(const char *graphName, StatisticCreatorType creatorType, const char * creator, unsigned subgraph) const
  6557. {
  6558. return new CWuGraphStats(LINK(p), creatorType, creator, graphName, subgraph);
  6559. }
  6560. void CLocalWUGraph::setName(const char *str)
  6561. {
  6562. p->setProp("@name", str);
  6563. }
  6564. void CLocalWUGraph::setLabel(const char *str)
  6565. {
  6566. p->setProp("@label", str);
  6567. }
  6568. void CLocalWUGraph::setXGMML(const char *str)
  6569. {
  6570. setXGMMLTree(createPTreeFromXMLString(str,ipt_lowmem));
  6571. }
  6572. void CLocalWUGraph::setXGMMLTree(IPropertyTree *_graph)
  6573. {
  6574. assertex(strcmp(_graph->queryName(), "graph")==0);
  6575. IPropertyTree *xgmml = p->setPropTree("xgmml");
  6576. MemoryBuffer mb;
  6577. _graph->serialize(mb);
  6578. // Note - we could compress further but that would introduce compatibility concerns, so don't bother
  6579. // Cassandra workunit code actually lzw compresses the parent anyway
  6580. xgmml->setPropBin("graphBin", mb.length(), mb.toByteArray());
  6581. graph.setown(_graph);
  6582. }
  6583. static void expandAttributes(IPropertyTree & targetNode, IPropertyTree & progressNode)
  6584. {
  6585. Owned<IAttributeIterator> aIter = progressNode.getAttributes();
  6586. ForEach (*aIter)
  6587. {
  6588. const char *aName = aIter->queryName()+1;
  6589. if (0 != stricmp("id", aName)) // "id" reserved.
  6590. {
  6591. IPropertyTree *att = targetNode.addPropTree("att");
  6592. att->setProp("@name", aName);
  6593. att->setProp("@value", aIter->queryValue());
  6594. }
  6595. }
  6596. }
  6597. void CLocalWUGraph::mergeProgress(IPropertyTree &rootNode, IPropertyTree &progressTree, const unsigned &progressV) const
  6598. {
  6599. IPropertyTree *graphNode = rootNode.queryPropTree("att/graph");
  6600. if (!graphNode) return;
  6601. unsigned nodeId = rootNode.getPropInt("@id");
  6602. StringBuffer progressNodePath("node[@id=\"");
  6603. progressNodePath.append(nodeId).append("\"]");
  6604. IPropertyTree *progressNode = progressTree.queryPropTree(progressNodePath.str());
  6605. if (progressNode)
  6606. {
  6607. expandAttributes(*graphNode, *progressNode);
  6608. Owned<IPropertyTreeIterator> edges = progressNode->getElements("edge");
  6609. ForEach (*edges)
  6610. {
  6611. IPropertyTree &edge = edges->query();
  6612. StringBuffer edgePath("edge[@id=\"");
  6613. edgePath.append(edge.queryProp("@id")).append("\"]");
  6614. IPropertyTree *graphEdge = graphNode->queryPropTree(edgePath.str());
  6615. if (graphEdge)
  6616. {
  6617. if (progressV < 1)
  6618. mergePTree(graphEdge, &edge);
  6619. else
  6620. { // must translate to XGMML format
  6621. expandAttributes(*graphEdge, edge);
  6622. // This is really only here, so that our progress format can use non-attribute values, which have different efficiency qualifies (e.g. can be external by dali)
  6623. Owned<IPropertyTreeIterator> iter = edge.getElements("*");
  6624. ForEach (*iter)
  6625. {
  6626. IPropertyTree &t = iter->query();
  6627. IPropertyTree *att = graphEdge->addPropTree("att");
  6628. att->setProp("@name", t.queryName());
  6629. att->setProp("@value", t.queryProp(NULL));
  6630. }
  6631. }
  6632. }
  6633. }
  6634. Owned<IPropertyTreeIterator> nodes = progressNode->getElements("node");
  6635. ForEach (*nodes)
  6636. {
  6637. IPropertyTree &node = nodes->query();
  6638. StringBuffer nodePath("node[@id=\"");
  6639. nodePath.append(node.queryProp("@id")).append("\"]");
  6640. IPropertyTree *_node = graphNode->queryPropTree(nodePath.str());
  6641. if (_node)
  6642. {
  6643. if (progressV < 1)
  6644. mergePTree(_node, &node);
  6645. else
  6646. { // must translate to XGMML format
  6647. expandAttributes(*_node, node);
  6648. }
  6649. }
  6650. }
  6651. }
  6652. Owned<IPropertyTreeIterator> iter = graphNode->getElements("node");
  6653. ForEach (*iter)
  6654. mergeProgress(iter->query(), progressTree, progressV);
  6655. }
  6656. IPropertyTree * CLocalWUGraph::getXGMMLTreeRaw() const
  6657. {
  6658. return p->getPropTree("xgmml");
  6659. }
  6660. IPropertyTree * CLocalWUGraph::getXGMMLTree(bool doMergeProgress) const
  6661. {
  6662. if (!graph)
  6663. {
  6664. // NB: although graphBin introduced in wuidVersion==2,
  6665. // daliadmin can retrospectively compress existing graphs, so need to check for all versions
  6666. MemoryBuffer mb;
  6667. if (p->getPropBin("xgmml/graphBin", mb))
  6668. graph.setown(createPTree(mb, ipt_lowmem));
  6669. else
  6670. graph.setown(p->getBranch("xgmml/graph"));
  6671. if (!graph)
  6672. return NULL;
  6673. }
  6674. if (!doMergeProgress)
  6675. return graph.getLink();
  6676. else
  6677. {
  6678. Owned<IPropertyTree> copy = createPTreeFromIPT(graph, ipt_lowmem);
  6679. Owned<IConstWUGraphProgress> progress = owner.getGraphProgress(p->queryProp("@name"));
  6680. if (progress)
  6681. {
  6682. //MORE: Eventually this should directly access the new stats structure
  6683. unsigned progressV = progress->queryFormatVersion();
  6684. Owned<IPropertyTree> progressTree = progress->getProgressTree();
  6685. Owned<IPropertyTreeIterator> nodeIterator = copy->getElements("node");
  6686. ForEach (*nodeIterator)
  6687. mergeProgress(nodeIterator->query(), *progressTree, progressV);
  6688. }
  6689. return copy.getClear();
  6690. }
  6691. }
  6692. WUGraphType CLocalWUGraph::getType() const
  6693. {
  6694. return (WUGraphType) getEnum(p, "@type", graphTypes);
  6695. }
  6696. IStringVal & CLocalWUGraph::getTypeName(IStringVal &str) const
  6697. {
  6698. str.set(p->queryProp("@type"));
  6699. if (!str.length())
  6700. str.set("unknown");
  6701. return str;
  6702. }
  6703. void CLocalWUGraph::setType(WUGraphType _type)
  6704. {
  6705. setEnum(p, "@type", _type, graphTypes);
  6706. }
  6707. //=================================================================================================
  6708. mapEnums queryFileTypes[] = {
  6709. { FileTypeCpp, "cpp" },
  6710. { FileTypeDll, "dll" },
  6711. { FileTypeResText, "res" },
  6712. { FileTypeHintXml, "hint" },
  6713. { FileTypeXml, "xml" },
  6714. { FileTypeSize, NULL },
  6715. };
  6716. CLocalWUAssociated::CLocalWUAssociated(IPropertyTree *props) : p(props)
  6717. {
  6718. }
  6719. WUFileType CLocalWUAssociated::getType() const
  6720. {
  6721. return (WUFileType)getEnum(p, "@type", queryFileTypes);
  6722. }
  6723. IStringVal & CLocalWUAssociated::getDescription(IStringVal & str) const
  6724. {
  6725. str.set(p->queryProp("@desc"));
  6726. return str;
  6727. }
  6728. IStringVal & CLocalWUAssociated::getIp(IStringVal & str) const
  6729. {
  6730. str.set(p->queryProp("@ip"));
  6731. return str;
  6732. }
  6733. IStringVal & CLocalWUAssociated::getName(IStringVal & str) const
  6734. {
  6735. str.set(p->queryProp("@filename"));
  6736. return str;
  6737. }
  6738. IStringVal & CLocalWUAssociated::getNameTail(IStringVal & str) const
  6739. {
  6740. str.set(pathTail(p->queryProp("@filename")));
  6741. return str;
  6742. }
  6743. unsigned CLocalWUAssociated::getCrc() const
  6744. {
  6745. return p->getPropInt("@crc", 0);
  6746. }
  6747. unsigned CLocalWUAssociated::getMinActivityId() const
  6748. {
  6749. return p->getPropInt("@minActivity", 0);
  6750. }
  6751. unsigned CLocalWUAssociated::getMaxActivityId() const
  6752. {
  6753. return p->getPropInt("@maxActivity", 0);
  6754. }
  6755. //=================================================================================================
  6756. CLocalWUQuery::CLocalWUQuery(IPropertyTree *props) : p(props)
  6757. {
  6758. associatedCached = false;
  6759. }
  6760. mapEnums queryTypes[] = {
  6761. { QueryTypeUnknown, "unknown" },
  6762. { QueryTypeEcl, "ECL" },
  6763. { QueryTypeSql, "SQL" },
  6764. { QueryTypeXml, "XML" },
  6765. { QueryTypeAttribute, "Attribute" },
  6766. { QueryTypeSize, NULL },
  6767. };
  6768. WUQueryType CLocalWUQuery::getQueryType() const
  6769. {
  6770. return (WUQueryType) getEnum(p, "@type", queryTypes);
  6771. }
  6772. void CLocalWUQuery::setQueryType(WUQueryType qt)
  6773. {
  6774. setEnum(p, "@type", qt, queryTypes);
  6775. }
  6776. IStringVal& CLocalWUQuery::getQueryText(IStringVal &str) const
  6777. {
  6778. str.set(p->queryProp("Text"));
  6779. return str;
  6780. }
  6781. IStringVal& CLocalWUQuery::getQueryShortText(IStringVal &str) const
  6782. {
  6783. const char * text = p->queryProp("ShortText");
  6784. if (text)
  6785. str.set(text);
  6786. else
  6787. {
  6788. text = p->queryProp("Text");
  6789. if (isArchiveQuery(text))
  6790. {
  6791. Owned<IPropertyTree> xml = createPTreeFromXMLString(text, ipt_caseInsensitive|ipt_lowmem);
  6792. const char * path = xml->queryProp("Query/@attributePath");
  6793. if (path)
  6794. {
  6795. IPropertyTree * resolved = resolveDefinitionInArchive(xml, path);
  6796. if (resolved)
  6797. str.set(resolved->queryProp(NULL));
  6798. }
  6799. else
  6800. str.set(xml->queryProp("Query"));
  6801. }
  6802. else
  6803. str.set(text);
  6804. }
  6805. return str;
  6806. }
  6807. bool CLocalWUQuery::isArchive() const
  6808. {
  6809. if (p->hasProp("@isArchive"))
  6810. return p->getPropBool("@isArchive");
  6811. const char *text = p->queryProp("Text");
  6812. return isArchiveQuery(text);
  6813. }
  6814. IStringVal& CLocalWUQuery::getQueryName(IStringVal &str) const
  6815. {
  6816. str.set(p->queryProp("@name"));
  6817. return str;
  6818. }
  6819. IStringVal & CLocalWUQuery::getQueryMainDefinition(IStringVal & str) const
  6820. {
  6821. str.set(p->queryProp("@main"));
  6822. return str;
  6823. }
  6824. IStringVal& CLocalWUQuery::getQueryDllName(IStringVal &str) const
  6825. {
  6826. Owned<IConstWUAssociatedFile> entry = getAssociatedFile(FileTypeDll, 0);
  6827. if (entry)
  6828. entry->getNameTail(str);
  6829. return str;
  6830. }
  6831. IStringVal& CLocalWUQuery::getQueryCppName(IStringVal &str) const
  6832. {
  6833. Owned<IConstWUAssociatedFile> entry = getAssociatedFile(FileTypeCpp, 0);
  6834. if (entry)
  6835. entry->getName(str);
  6836. return str;
  6837. }
  6838. IStringVal& CLocalWUQuery::getQueryResTxtName(IStringVal &str) const
  6839. {
  6840. Owned<IConstWUAssociatedFile> entry = getAssociatedFile(FileTypeResText, 0);
  6841. if (entry)
  6842. entry->getName(str);
  6843. return str;
  6844. }
  6845. unsigned CLocalWUQuery::getQueryDllCrc() const
  6846. {
  6847. Owned<IConstWUAssociatedFile> entry = getAssociatedFile(FileTypeDll, 0);
  6848. if (entry)
  6849. return entry->getCrc();
  6850. return 0;
  6851. }
  6852. void CLocalWUQuery::setQueryText(const char *text)
  6853. {
  6854. p->setProp("Text", text);
  6855. bool isArchive = isArchiveQuery(text);
  6856. if (isArchive)
  6857. {
  6858. Owned<IPropertyTree> xml = createPTreeFromXMLString(text, ipt_caseInsensitive|ipt_lowmem);
  6859. const char * path = xml->queryProp("Query/@attributePath");
  6860. if (path)
  6861. {
  6862. IPropertyTree * resolved = resolveDefinitionInArchive(xml, path);
  6863. if (resolved)
  6864. p->setProp("ShortText", resolved->queryProp(NULL));
  6865. }
  6866. else
  6867. p->setProp("ShortText", xml->queryProp("Query"));
  6868. }
  6869. p->setPropBool("@isArchive", isArchive);
  6870. if (isArchive)
  6871. p->setPropBool("@hasArchive", true); //preserved if setQueryText is called multiple times. Should setting this be more explicit?
  6872. }
  6873. void CLocalWUQuery::setQueryName(const char *qname)
  6874. {
  6875. p->setProp("@name", qname);
  6876. }
  6877. void CLocalWUQuery::setQueryMainDefinition(const char * str)
  6878. {
  6879. p->setProp("@main", str);
  6880. }
  6881. void CLocalWUQuery::addAssociatedFile(WUFileType type, const char * name, const char * ip, const char * desc, unsigned crc, unsigned minActivity, unsigned maxActivity)
  6882. {
  6883. CriticalBlock block(crit);
  6884. loadAssociated();
  6885. if (!associated.length())
  6886. p->addPropTree("Associated");
  6887. IPropertyTree *pl = p->queryPropTree("Associated");
  6888. IPropertyTree *s = pl->addPropTree("File");
  6889. setEnum(s, "@type", type, queryFileTypes);
  6890. s->setProp("@filename", name);
  6891. s->setProp("@ip", ip);
  6892. s->setProp("@desc", desc);
  6893. if (crc)
  6894. s->setPropInt("@crc", crc);
  6895. if (minActivity)
  6896. s->setPropInt("@minActivity", minActivity);
  6897. if (maxActivity)
  6898. s->setPropInt("@maxActivity", maxActivity);
  6899. IConstWUAssociatedFile * q = new CLocalWUAssociated(LINK(s));
  6900. associated.append(*q);
  6901. }
  6902. void CLocalWUQuery::removeAssociatedFile(WUFileType type, const char * name, const char * desc)
  6903. {
  6904. CriticalBlock block(crit);
  6905. associatedCached = false;
  6906. associated.kill();
  6907. StringBuffer xpath;
  6908. xpath.append("Associated/File");
  6909. if (type)
  6910. xpath.append("[@type=\"").append(getEnumText(type, queryFileTypes)).append("\"]");
  6911. if (name)
  6912. xpath.append("[@filename=\"").append(name).append("\"]");
  6913. if (desc)
  6914. xpath.append("[@desc=\"").append(desc).append("\"]");
  6915. p->removeProp(xpath.str());
  6916. }
  6917. void CLocalWUQuery::removeAssociatedFiles()
  6918. {
  6919. associatedCached = false;
  6920. associated.kill();
  6921. p->removeProp("Associated");
  6922. }
  6923. IConstWUAssociatedFile * CLocalWUQuery::getAssociatedFile(WUFileType type, unsigned index) const
  6924. {
  6925. CriticalBlock block(crit);
  6926. loadAssociated();
  6927. ForEachItemIn(idx, associated)
  6928. {
  6929. CLocalWUAssociated &cur = static_cast<CLocalWUAssociated &>(associated.item(idx));
  6930. if (cur.getType() == type)
  6931. {
  6932. if (index-- == 0)
  6933. return &OLINK(cur);
  6934. }
  6935. }
  6936. return NULL;
  6937. }
  6938. void CLocalWUQuery::addSpecialCaseAssociated(WUFileType type, const char * propname, unsigned crc) const
  6939. {
  6940. const char * name = p->queryProp(propname);
  6941. if (name)
  6942. {
  6943. IPropertyTree *s = createPTree("File");
  6944. setEnum(s, "@type", type, queryFileTypes);
  6945. s->setProp("@filename", name);
  6946. if (crc)
  6947. s->setPropInt("@crc", crc);
  6948. associated.append(*new CLocalWUAssociated(s));
  6949. }
  6950. }
  6951. void CLocalWUQuery::loadAssociated() const
  6952. {
  6953. CriticalBlock block(crit);
  6954. if (!associatedCached)
  6955. {
  6956. assertex(associated.length() == 0);
  6957. addSpecialCaseAssociated(FileTypeDll, "DllName", p->getPropInt("DllCrc", 0));
  6958. addSpecialCaseAssociated(FileTypeCpp, "CppName", 0);
  6959. addSpecialCaseAssociated(FileTypeResText, "ResTxtName", 0);
  6960. Owned<IPropertyTreeIterator> r = p->getElements("Associated/File");
  6961. for (r->first(); r->isValid(); r->next())
  6962. {
  6963. IPropertyTree *rp = &r->query();
  6964. rp->Link();
  6965. associated.append(*new CLocalWUAssociated(rp));
  6966. }
  6967. associatedCached = true;
  6968. }
  6969. }
  6970. IConstWUAssociatedFileIterator& CLocalWUQuery::getAssociatedFiles() const
  6971. {
  6972. CriticalBlock block(crit);
  6973. loadAssociated();
  6974. return *new CArrayIteratorOf<IConstWUAssociatedFile,IConstWUAssociatedFileIterator> (associated, 0, (IConstWUQuery *) this);
  6975. }
  6976. //========================================================================================
  6977. CLocalWUWebServicesInfo::CLocalWUWebServicesInfo(IPropertyTree *props) : p(props)
  6978. {
  6979. }
  6980. IStringVal& CLocalWUWebServicesInfo::getModuleName(IStringVal &str) const
  6981. {
  6982. str.set(p->queryProp("@module"));
  6983. return str;
  6984. }
  6985. IStringVal& CLocalWUWebServicesInfo::getAttributeName(IStringVal &str) const
  6986. {
  6987. str.set(p->queryProp("@attribute"));
  6988. return str;
  6989. }
  6990. IStringVal& CLocalWUWebServicesInfo::getDefaultName(IStringVal &str) const
  6991. {
  6992. str.set(p->queryProp("@defaultName"));
  6993. return str;
  6994. }
  6995. unsigned CLocalWUWebServicesInfo::getWebServicesCRC() const
  6996. {
  6997. return (unsigned) p->getPropInt("@crc");
  6998. }
  6999. IStringVal& CLocalWUWebServicesInfo::getInfo(const char *name, IStringVal &str) const
  7000. {
  7001. if (!name)
  7002. {
  7003. StringBuffer ws_info;
  7004. ws_info.appendf("<%s ", p->queryName());
  7005. Owned<IAttributeIterator> attrs = p->getAttributes();
  7006. for(attrs->first(); attrs->isValid(); attrs->next())
  7007. {
  7008. const char *name = attrs->queryName()+1;
  7009. const char *value = attrs->queryValue();
  7010. ws_info.appendf("%s='%s' ", name, value);
  7011. }
  7012. ws_info.append("> \n");
  7013. Owned<IPropertyTreeIterator> info = p->getElements("*");
  7014. ForEach(*info)
  7015. {
  7016. IPropertyTree &item = info->query();
  7017. const char *name = item.queryName();
  7018. if (name)
  7019. {
  7020. MemoryBuffer mb;
  7021. bool isbin = p->isBinary(name);
  7022. if (isbin)
  7023. {
  7024. p->getPropBin(name,mb);
  7025. if (mb.length())
  7026. {
  7027. unsigned len = 0;
  7028. mb.read(len);
  7029. StringBuffer encodedString;
  7030. StringBuffer val(len, (const char *) mb.readDirect(len));
  7031. encodeXML(val, encodedString);
  7032. ws_info.appendf("<%s>%s</%s>", name, encodedString.str(), name);
  7033. }
  7034. }
  7035. else
  7036. {
  7037. StringBuffer tmp;
  7038. toXML(&item, tmp);
  7039. ws_info.append(tmp.str());
  7040. }
  7041. }
  7042. }
  7043. ws_info.appendf("</%s>", p->queryName());
  7044. str.setLen(ws_info.str(), ws_info.length());
  7045. }
  7046. else
  7047. {
  7048. MemoryBuffer mb;
  7049. p->getPropBin(name,mb);
  7050. if (mb.length())
  7051. {
  7052. unsigned len;
  7053. mb.read(len);
  7054. str.setLen((const char *) mb.readDirect(len), len);
  7055. }
  7056. }
  7057. return str;
  7058. }
  7059. IStringVal& CLocalWUWebServicesInfo::getText(const char *name, IStringVal &str) const
  7060. {
  7061. str.set(p->queryProp(name));
  7062. return str;
  7063. }
  7064. void CLocalWUWebServicesInfo::setModuleName(const char *mname)
  7065. {
  7066. p->setProp("@module", mname);
  7067. }
  7068. void CLocalWUWebServicesInfo::setAttributeName(const char *aname)
  7069. {
  7070. p->setProp("@attribute", aname);
  7071. }
  7072. void CLocalWUWebServicesInfo::setDefaultName(const char *dname)
  7073. {
  7074. p->setProp("@defaultName", dname);
  7075. }
  7076. void CLocalWUWebServicesInfo::setWebServicesCRC(unsigned crc)
  7077. {
  7078. p->setPropInt("@crc", crc);
  7079. }
  7080. void CLocalWUWebServicesInfo::setInfo(const char *name, const char *info)
  7081. {
  7082. MemoryBuffer m;
  7083. unsigned len = (size32_t)strlen(info);
  7084. serializeLPString(len, info, m);
  7085. p->setPropBin(name, m.length(), m.toByteArray());
  7086. }
  7087. void CLocalWUWebServicesInfo::setText(const char *name, const char *info)
  7088. {
  7089. p->setProp(name, info);
  7090. }
  7091. //========================================================================================
  7092. CLocalWUResult::CLocalWUResult(IPropertyTree *props) : p(props)
  7093. {
  7094. }
  7095. mapEnums resultStatuses[] = {
  7096. { ResultStatusUndefined, "undefined" },
  7097. { ResultStatusCalculated, "calculated" },
  7098. { ResultStatusSupplied, "supplied" },
  7099. { ResultStatusFailed, "failed" },
  7100. { ResultStatusPartial, "partial" },
  7101. { ResultStatusSize, NULL }
  7102. };
  7103. WUResultStatus CLocalWUResult::getResultStatus() const
  7104. {
  7105. return (WUResultStatus ) getEnum(p, "@status", resultStatuses);
  7106. }
  7107. IStringVal& CLocalWUResult::getResultName(IStringVal &str) const
  7108. {
  7109. str.set(p->queryProp("@name"));
  7110. return str;
  7111. }
  7112. int CLocalWUResult::getResultSequence() const
  7113. {
  7114. return p->getPropInt("@sequence", -1);
  7115. }
  7116. bool CLocalWUResult::isResultScalar() const
  7117. {
  7118. return p->getPropInt("@isScalar", 1) != 0;
  7119. }
  7120. bool findSize(int size, IntArray &sizes)
  7121. {
  7122. ForEachItemIn(idx, sizes)
  7123. {
  7124. if (sizes.item(idx)==size)
  7125. return true;
  7126. }
  7127. return false;
  7128. }
  7129. void CLocalWUResult::getSchema(IArrayOf<ITypeInfo> &types, StringAttrArray &names, IStringVal * eclText) const
  7130. {
  7131. MemoryBuffer schema;
  7132. p->getPropBin("SchemaRaw", schema);
  7133. if (schema.length())
  7134. {
  7135. for (;;)
  7136. {
  7137. StringAttr name;
  7138. schema.read(name);
  7139. if (*schema.readDirect(0)==type_void)
  7140. break;
  7141. names.append(*new StringAttrItem(name));
  7142. types.append(*deserializeType(schema)); // MORE - nested records!
  7143. }
  7144. schema.skip(1);
  7145. if (schema.length() != schema.getPos())
  7146. {
  7147. unsigned eclLen;
  7148. schema.read(eclLen);
  7149. const char * schemaData = (const char *)schema.readDirect(eclLen);
  7150. if (eclText)
  7151. {
  7152. eclText->setLen(schemaData, eclLen);
  7153. if ((eclLen == 0) && names.ordinality())
  7154. {
  7155. const char * firstName = names.item(0).text;
  7156. StringBuffer temp;
  7157. temp.append("RECORD ");
  7158. types.item(0).getECLType(temp);
  7159. temp.append(" value{NAMED('").append(firstName).append("')}").append("; END;");
  7160. eclText->set(temp.str());
  7161. }
  7162. }
  7163. }
  7164. }
  7165. }
  7166. void readRow(StringBuffer &out, MemoryBuffer &in, TypeInfoArray &types, StringAttrArray &names)
  7167. {
  7168. ForEachItemIn(idx, types)
  7169. {
  7170. StringAttrItem &name = names.item(idx);
  7171. ITypeInfo &type = types.item(idx);
  7172. unsigned size = type.getSize();
  7173. switch(type.getTypeCode())
  7174. {
  7175. case type_data:
  7176. if (size==UNKNOWN_LENGTH)
  7177. {
  7178. if (in.remaining() < sizeof(int))
  7179. throw MakeStringException(WUERR_CorruptResult, "corrupt workunit information");
  7180. in.read(size);
  7181. }
  7182. outputXmlData(size, in.readDirect(size), name.text, out);
  7183. break;
  7184. case type_string:
  7185. if (size==UNKNOWN_LENGTH)
  7186. {
  7187. if (in.remaining() < sizeof(int))
  7188. throw MakeStringException(WUERR_CorruptResult, "corrupt workunit information");
  7189. in.read(size);
  7190. }
  7191. outputXmlString(size, (const char *) in.readDirect(size), name.text, out);
  7192. break;
  7193. case type_varstring:
  7194. {
  7195. if (size == UNKNOWN_LENGTH)
  7196. size = (size32_t)strlen((const char *) in.readDirect(0))+1;
  7197. const char * text = (const char *) in.readDirect(size);
  7198. outputXmlString((size32_t)strlen(text), text, name.text, out);
  7199. break;
  7200. }
  7201. case type_unicode:
  7202. {
  7203. unsigned len = type.getStringLen();
  7204. if (size==UNKNOWN_LENGTH)
  7205. in.read(len);
  7206. outputXmlUnicode(len, (UChar const *) in.readDirect(len*2), name.text, out);
  7207. }
  7208. break;
  7209. case type_utf8:
  7210. {
  7211. unsigned len = type.getStringLen();
  7212. if (size==UNKNOWN_LENGTH)
  7213. {
  7214. in.read(len);
  7215. size = rtlUtf8Size(len, in.readDirect(0));
  7216. }
  7217. outputXmlUtf8(len, (const char *) in.readDirect(size), name.text, out);
  7218. }
  7219. break;
  7220. case type_qstring:
  7221. {
  7222. unsigned len = type.getStringLen();
  7223. if (size==UNKNOWN_LENGTH)
  7224. in.read(len);
  7225. unsigned outlen;
  7226. char *outstr;
  7227. rtlQStrToStrX(outlen, outstr, len, (const char *) in.readDirect(rtlQStrSize(len)));
  7228. outputXmlString(outlen, outstr, name.text, out);
  7229. free(outstr);
  7230. break;
  7231. }
  7232. case type_int:
  7233. case type_swapint:
  7234. if (type.isSigned())
  7235. {
  7236. const unsigned char *raw = (const unsigned char *) in.readDirect(size);
  7237. unsigned __int64 cval8 = 0;
  7238. //MORE: I think this is wrong - swapped doesn't mean little/big/
  7239. if (type.isSwappedEndian())
  7240. {
  7241. unsigned idx = 0;
  7242. if (raw[idx] & 0x80)
  7243. cval8 = (__int64)-1;
  7244. while (size--)
  7245. cval8 = (cval8 << 8) | raw[idx++];
  7246. }
  7247. else
  7248. {
  7249. if (raw[size-1] & 0x80)
  7250. cval8 = (__int64)-1;
  7251. while (size--)
  7252. cval8 = (cval8 << 8) | raw[size];
  7253. }
  7254. outputXmlInt((__int64) cval8, name.text, out);
  7255. }
  7256. else
  7257. {
  7258. const unsigned char *raw = (const unsigned char *) in.readDirect(size);
  7259. unsigned __int64 cval8 = 0;
  7260. if (type.isSwappedEndian())
  7261. {
  7262. unsigned idx = 0;
  7263. while (size--)
  7264. cval8 = (cval8 << 8) | raw[idx++];
  7265. }
  7266. else
  7267. {
  7268. while (size--)
  7269. cval8 = (cval8 << 8) | raw[size];
  7270. }
  7271. outputXmlUInt(cval8, name.text, out);
  7272. }
  7273. break;
  7274. case type_boolean:
  7275. bool cvalb;
  7276. in.read(cvalb);
  7277. outputXmlBool(cvalb, name.text, out);
  7278. break;
  7279. case type_decimal:
  7280. if (type.isSigned())
  7281. outputXmlDecimal(in.readDirect(size), size, type.getPrecision(), name.text, out);
  7282. else
  7283. outputXmlUDecimal(in.readDirect(size), size, type.getPrecision(), name.text, out);
  7284. break;
  7285. case type_real:
  7286. double cvald;
  7287. switch(size)
  7288. {
  7289. case 4:
  7290. float cvalf;
  7291. in.read(cvalf);
  7292. cvald = cvalf;
  7293. break;
  7294. case 8:
  7295. in.read(cvald);
  7296. break;
  7297. }
  7298. outputXmlReal(cvald, name.text, out);
  7299. break;
  7300. default:
  7301. assertex(!"unexpected type in raw record");
  7302. break;
  7303. }
  7304. }
  7305. }
  7306. IStringVal& CLocalWUResult::getResultXml(IStringVal &str, bool hidePassword) const
  7307. {
  7308. TypeInfoArray types;
  7309. StringAttrArray names;
  7310. getSchema(types, names);
  7311. StringBuffer xml;
  7312. const char * name = p->queryProp("@name");
  7313. if (name)
  7314. xml.appendf("<Dataset name=\'%s\'>\n", name);
  7315. else
  7316. xml.append("<Dataset>\n");
  7317. if (hidePassword && p->getPropBool("Format/@password"))
  7318. {
  7319. xml.append(" <Row>");
  7320. appendXMLTag(xml, name, "****");
  7321. xml.append("</Row>\n");
  7322. }
  7323. else if (p->hasProp("Value"))
  7324. {
  7325. MemoryBuffer raw;
  7326. p->getPropBin("Value", raw);
  7327. unsigned __int64 numrows = getResultRowCount();
  7328. while (numrows--)
  7329. {
  7330. xml.append(" <Row>");
  7331. readRow(xml, raw, types, names);
  7332. xml.append("</Row>\n");
  7333. }
  7334. }
  7335. else if (p->hasProp("xmlValue"))
  7336. {
  7337. xml.append(" <Row>");
  7338. appendXMLTag(xml, name, p->queryProp("xmlValue"));
  7339. xml.append("</Row>\n");
  7340. }
  7341. xml.append("</Dataset>\n");
  7342. str.set(xml.str());
  7343. return str;
  7344. }
  7345. IProperties *CLocalWUResult::queryResultXmlns()
  7346. {
  7347. CriticalBlock block(crit);
  7348. if (xmlns)
  7349. return xmlns;
  7350. xmlns.setown(createProperties());
  7351. Owned<IAttributeIterator> it = p->getAttributes();
  7352. unsigned prefixLen = strlen("@xmlns");
  7353. ForEach(*it)
  7354. {
  7355. const char *name = it->queryName();
  7356. if (!strncmp("@xmlns", name, prefixLen))
  7357. {
  7358. if (name[prefixLen]==':') //normal case
  7359. xmlns->setProp(name+prefixLen+1, it->queryValue());
  7360. else if (!name[prefixLen]) //special case, unprefixed namespace
  7361. xmlns->setProp("xmlns", it->queryValue());
  7362. }
  7363. }
  7364. return xmlns;
  7365. }
  7366. unsigned CLocalWUResult::getResultFetchSize() const
  7367. {
  7368. return p->getPropInt("fetchSize", 100);
  7369. }
  7370. __int64 CLocalWUResult::getResultTotalRowCount() const
  7371. {
  7372. return p->getPropInt64("totalRowCount", -1);
  7373. }
  7374. __int64 CLocalWUResult::getResultRowCount() const
  7375. {
  7376. return p->getPropInt64("rowCount", 0);
  7377. }
  7378. void CLocalWUResult::getResultDataset(IStringVal & ecl, IStringVal & defs) const
  7379. {
  7380. ecl.set(p->queryProp("datasetEcl"));
  7381. defs.set(p->queryProp("datasetEclDefs"));
  7382. }
  7383. IStringVal& CLocalWUResult::getResultLogicalName(IStringVal & val) const
  7384. {
  7385. val.set(p->queryProp("logicalName"));
  7386. return val;
  7387. }
  7388. IStringVal& CLocalWUResult::getResultKeyField(IStringVal & ecl) const
  7389. {
  7390. ecl.set(p->queryProp("keyField"));
  7391. return ecl;
  7392. }
  7393. unsigned CLocalWUResult::getResultRequestedRows() const
  7394. {
  7395. return p->getPropInt("requestedRows", 1);
  7396. }
  7397. IStringVal& CLocalWUResult::getResultEclSchema(IStringVal & str) const
  7398. {
  7399. TypeInfoArray types;
  7400. StringAttrArray names;
  7401. getSchema(types, names, &str);
  7402. return str;
  7403. }
  7404. IStringVal& CLocalWUResult::getResultRecordSizeEntry(IStringVal & str) const
  7405. {
  7406. str.set(p->queryProp("@recordSizeEntry"));
  7407. return str;
  7408. }
  7409. IStringVal& CLocalWUResult::getResultTransformerEntry(IStringVal & str) const
  7410. {
  7411. str.set(p->queryProp("@transformerEntry"));
  7412. return str;
  7413. }
  7414. __int64 CLocalWUResult::getResultRowLimit() const
  7415. {
  7416. return p->getPropInt64("@rowLimit");
  7417. }
  7418. IStringVal& CLocalWUResult::getResultFilename(IStringVal & str) const
  7419. {
  7420. str.set(p->queryProp("@tempFilename"));
  7421. return str;
  7422. }
  7423. IStringVal& CLocalWUResult::getResultFieldOpt(const char *name, IStringVal &str) const
  7424. {
  7425. str.clear();
  7426. if (!name || !*name)
  7427. return str;
  7428. IPropertyTree *format = p->queryPropTree("Format");
  7429. if (!format)
  7430. return str;
  7431. VStringBuffer xpath("@%s", name);
  7432. str.set(format->queryProp(xpath));
  7433. return str;
  7434. }
  7435. void CLocalWUResult::getResultWriteLocation(IStringVal & _graph, unsigned & _activityId) const
  7436. {
  7437. _graph.set(p->queryProp("@graph"));
  7438. _activityId = p->getPropInt("@activity", 0);
  7439. }
  7440. void CLocalWUResult::setResultStatus(WUResultStatus status)
  7441. {
  7442. setEnum(p, "@status", status, resultStatuses);
  7443. if (status==ResultStatusUndefined)
  7444. {
  7445. p->removeProp("Value");
  7446. p->removeProp("totalRowCount");
  7447. p->removeProp("rowCount");
  7448. p->removeProp("@format");
  7449. p->removeProp("@tempFileNmae");
  7450. p->removeProp("logicalName");
  7451. }
  7452. }
  7453. void CLocalWUResult::setResultName(const char *s)
  7454. {
  7455. p->setProp("@name", s);
  7456. }
  7457. void CLocalWUResult::setResultSequence(unsigned seq)
  7458. {
  7459. p->setPropInt("@sequence", seq);
  7460. }
  7461. void CLocalWUResult::setResultSchemaRaw(unsigned size, const void *schema)
  7462. {
  7463. p->setPropBin("SchemaRaw", size, schema);
  7464. }
  7465. void CLocalWUResult::setResultXmlns(const char *prefix, const char *uri)
  7466. {
  7467. StringBuffer xpath("@xmlns");
  7468. if (prefix && *prefix)
  7469. xpath.append(':').append(prefix);
  7470. p->setProp(xpath, uri);
  7471. }
  7472. void CLocalWUResult::setResultFieldOpt(const char *name, const char *value)
  7473. {
  7474. if (!name || !*name)
  7475. return;
  7476. IPropertyTree *format = ensurePTree(p, "Format");
  7477. VStringBuffer xpath("@%s", name);
  7478. format->setProp(xpath, value);
  7479. }
  7480. void CLocalWUResult::setResultWriteLocation(const char * _graph, unsigned _activityId)
  7481. {
  7482. p->setProp("@graph", _graph);
  7483. p->setPropInt("@activity", _activityId);
  7484. }
  7485. void CLocalWUResult::setResultScalar(bool isScalar)
  7486. {
  7487. p->setPropInt("@isScalar", (int) isScalar);
  7488. if (isScalar)
  7489. setResultTotalRowCount(1);
  7490. }
  7491. void CLocalWUResult::setResultRaw(unsigned len, const void *data, WUResultFormat format)
  7492. {
  7493. p->setPropBin("Value", len, data);
  7494. setResultStatus(ResultStatusSupplied);
  7495. setResultFormat(format);
  7496. }
  7497. void CLocalWUResult::setResultFormat(WUResultFormat format)
  7498. {
  7499. switch (format)
  7500. {
  7501. case ResultFormatXml:
  7502. p->setProp("@format","xml");
  7503. break;
  7504. case ResultFormatXmlSet:
  7505. p->setProp("@format","xmlSet");
  7506. break;
  7507. case ResultFormatCsv:
  7508. p->setProp("@format","csv");
  7509. break;
  7510. default:
  7511. p->removeProp("@format");
  7512. break;
  7513. }
  7514. }
  7515. void CLocalWUResult::setResultXML(const char *val)
  7516. {
  7517. p->setProp("xmlValue", val);
  7518. }
  7519. void CLocalWUResult::addResultRaw(unsigned len, const void *data, WUResultFormat format)
  7520. {
  7521. p->appendPropBin("Value", len, data);
  7522. setResultStatus(ResultStatusPartial);
  7523. const char *existingFormat = p->queryProp("@format");
  7524. const char *formatStr = NULL;
  7525. switch (format)
  7526. {
  7527. case ResultFormatXml:
  7528. formatStr = "xml";
  7529. break;
  7530. case ResultFormatXmlSet:
  7531. formatStr = "xmlSet";
  7532. break;
  7533. case ResultFormatCsv:
  7534. formatStr = "csv";
  7535. break;
  7536. default:
  7537. p->removeProp("@format");
  7538. break;
  7539. }
  7540. if (format)
  7541. {
  7542. if (existingFormat)
  7543. {
  7544. if (0 != stricmp(formatStr, existingFormat))
  7545. throw MakeStringException(WUERR_ResultFormatMismatch, "addResult format %s, does not match existing format %s", formatStr, existingFormat);
  7546. }
  7547. else
  7548. p->setProp("@format", formatStr);
  7549. }
  7550. }
  7551. void CLocalWUResult::setResultFetchSize(unsigned rows)
  7552. {
  7553. p->setPropInt("fetchSize", rows);
  7554. }
  7555. void CLocalWUResult::setResultTotalRowCount(__int64 rows)
  7556. {
  7557. p->setPropInt64("totalRowCount", rows);
  7558. }
  7559. void CLocalWUResult::setResultRowCount(__int64 rows)
  7560. {
  7561. p->setPropInt64("rowCount", rows);
  7562. }
  7563. void CLocalWUResult::setResultDataset(const char *ecl, const char *defs)
  7564. {
  7565. p->setProp("datasetEcl", ecl);
  7566. p->setProp("datasetEclDefs", defs);
  7567. }
  7568. void CLocalWUResult::setResultLogicalName(const char *logicalName)
  7569. {
  7570. p->setProp("logicalName", logicalName);
  7571. }
  7572. void CLocalWUResult::setResultKeyField(const char *ecl)
  7573. {
  7574. p->setProp("keyField", ecl);
  7575. }
  7576. void CLocalWUResult::setResultRequestedRows(unsigned rows)
  7577. {
  7578. p->setPropInt("requestedRows", rows);
  7579. }
  7580. void CLocalWUResult::setResultRecordSizeEntry(const char * entry)
  7581. {
  7582. p->setProp("@recordSizeEntry", entry);
  7583. }
  7584. void CLocalWUResult::setResultTransformerEntry(const char * entry)
  7585. {
  7586. p->setProp("@transformerEntry", entry);
  7587. }
  7588. void CLocalWUResult::setResultRowLimit(__int64 value)
  7589. {
  7590. p->setPropInt64("@rowLimit", value);
  7591. }
  7592. void CLocalWUResult::setResultFilename(const char * name)
  7593. {
  7594. p->setProp("@tempFilename", name);
  7595. }
  7596. // MORE - it's an undetected error if we call getResult... of a type that does not match schema
  7597. __int64 CLocalWUResult::getResultInt() const
  7598. {
  7599. __int64 result = 0;
  7600. MemoryBuffer s;
  7601. p->getPropBin("Value", s);
  7602. if (s.length())
  7603. s.read(result);
  7604. else
  7605. result = p->getPropInt64("xmlValue");
  7606. return result;
  7607. }
  7608. bool CLocalWUResult::getResultBool() const
  7609. {
  7610. bool result = false;
  7611. MemoryBuffer s;
  7612. p->getPropBin("Value", s);
  7613. if (s.length())
  7614. s.read(result);
  7615. else
  7616. result = p->getPropBool("xmlValue");
  7617. return result;
  7618. }
  7619. double CLocalWUResult::getResultReal() const
  7620. {
  7621. double result = 0;
  7622. MemoryBuffer s;
  7623. p->getPropBin("Value", s);
  7624. if (s.length())
  7625. s.read(result);
  7626. else
  7627. {
  7628. const char *xmlVal = p->queryProp("xmlValue");
  7629. if (xmlVal)
  7630. result = atof(xmlVal);
  7631. }
  7632. return result;
  7633. }
  7634. void CLocalWUResult::getResultDecimal(void * val, unsigned len, unsigned precision, bool isSigned) const
  7635. {
  7636. MemoryBuffer s;
  7637. p->getPropBin("Value", s);
  7638. if (s.length())
  7639. {
  7640. assertex(s.length() == len);
  7641. s.read(len, val);
  7642. }
  7643. else
  7644. {
  7645. const char *xmlVal = p->queryProp("xmlValue");
  7646. if (xmlVal)
  7647. {
  7648. Decimal d;
  7649. d.setString(strlen(xmlVal), xmlVal);
  7650. if (isSigned)
  7651. d.getDecimal(len, precision, val);
  7652. else
  7653. d.getUDecimal(len, precision, val);
  7654. }
  7655. else
  7656. memset(val, 0, len);
  7657. }
  7658. }
  7659. IStringVal& CLocalWUResult::getResultString(IStringVal & str, bool hidePassword) const
  7660. {
  7661. if (hidePassword && p->getPropBool("@password"))
  7662. {
  7663. str.set("****");
  7664. return str;
  7665. }
  7666. MemoryBuffer s;
  7667. p->getPropBin("Value", s);
  7668. if (s.length())
  7669. {
  7670. unsigned len;
  7671. s.read(len);
  7672. str.setLen((const char *) s.readDirect(len), len);
  7673. }
  7674. else
  7675. {
  7676. p->getPropBin("xmlValue", s);
  7677. if (p->isBinary("xmlValue"))
  7678. str.setLen(s.toByteArray(), s.length());
  7679. else
  7680. {
  7681. char *ascii = rtlUtf8ToVStr(rtlUtf8Length(s.length(), s.toByteArray()), s.toByteArray());
  7682. str.set(ascii);
  7683. rtlFree(ascii);
  7684. }
  7685. }
  7686. return str;
  7687. }
  7688. WUResultFormat CLocalWUResult::getResultFormat() const
  7689. {
  7690. const char * format = p->queryProp("@format");
  7691. if (!format)
  7692. return ResultFormatRaw;
  7693. else if (strcmp(format, "xml") == 0)
  7694. return ResultFormatXml;
  7695. else if (strcmp(format, "xmlSet") == 0)
  7696. return ResultFormatXmlSet;
  7697. else if (strcmp(format, "csv") == 0)
  7698. return ResultFormatCsv;
  7699. else
  7700. throw MakeStringException(WUERR_InvalidResultFormat, "Unrecognised result format %s", format);
  7701. }
  7702. IDataVal& CLocalWUResult::getResultRaw(IDataVal & data, IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const
  7703. {
  7704. MemoryBuffer s;
  7705. p->getPropBin("Value", s);
  7706. unsigned len = s.length();
  7707. if (len)
  7708. {
  7709. WUResultFormat format = getResultFormat();
  7710. if (format == ResultFormatXml || format == ResultFormatXmlSet)
  7711. {
  7712. if (!xmlTransformer)
  7713. throw MakeStringException(WUERR_MissingFormatTranslator, "No transformer supplied to translate XML format result");
  7714. xmlTransformer->transform(data, len, s.readDirect(len), format == ResultFormatXml);
  7715. }
  7716. else if (format == ResultFormatCsv)
  7717. {
  7718. if (!csvTransformer)
  7719. throw MakeStringException(WUERR_MissingFormatTranslator, "No transformer supplied to translate Csv format result");
  7720. csvTransformer->transform(data, len, s.readDirect(len), true);
  7721. }
  7722. else
  7723. data.setLen(s.readDirect(len), len);
  7724. }
  7725. else
  7726. data.clear();
  7727. return data;
  7728. }
  7729. unsigned CLocalWUResult::getResultHash() const
  7730. {
  7731. MemoryBuffer s;
  7732. p->getPropBin("Value", s);
  7733. unsigned len = s.length();
  7734. const byte * data = (const byte *)s.toByteArray();
  7735. return ~hashc(data, len, ~0);
  7736. }
  7737. IDataVal& CLocalWUResult::getResultUnicode(IDataVal & data) const
  7738. {
  7739. MemoryBuffer s;
  7740. p->getPropBin("Value", s);
  7741. if (s.length())
  7742. {
  7743. unsigned len;
  7744. s.read(len);
  7745. data.setLen(s.readDirect(len*2), len*2);
  7746. }
  7747. else
  7748. {
  7749. StringBuffer utf8;
  7750. if (p->getProp("xmlValue", utf8))
  7751. {
  7752. unsigned outlen;
  7753. UChar *out;
  7754. rtlUtf8ToUnicodeX(outlen, out, utf8.length(), utf8.str());
  7755. data.setLen(out, outlen*2);
  7756. rtlFree(out);
  7757. }
  7758. else
  7759. data.clear();
  7760. }
  7761. return data;
  7762. }
  7763. __int64 CLocalWUResult::getResultRawSize(IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const
  7764. {
  7765. WUResultFormat format = getResultFormat();
  7766. if (format == ResultFormatRaw)
  7767. {
  7768. //MORE: This should not load the whole property...
  7769. MemoryBuffer s;
  7770. p->getPropBin("Value", s);
  7771. return s.length();
  7772. }
  7773. else
  7774. {
  7775. MemoryBuffer temp;
  7776. MemoryBuffer2IDataVal adaptor(temp);
  7777. getResultRaw(adaptor, xmlTransformer, csvTransformer);
  7778. return temp.length();
  7779. }
  7780. }
  7781. IDataVal& CLocalWUResult::getResultRaw(IDataVal & data, __int64 from, __int64 length, IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const
  7782. {
  7783. WUResultFormat format = getResultFormat();
  7784. if (format != ResultFormatRaw)
  7785. {
  7786. MemoryBuffer temp;
  7787. MemoryBuffer2IDataVal adaptor(temp);
  7788. getResultRaw(adaptor, xmlTransformer, csvTransformer);
  7789. unsigned len = temp.length();
  7790. if (from > len) from = len;
  7791. if (from + length > len) length = len - from;
  7792. data.setLen(temp.readDirect(len) + from, (size32_t)length);
  7793. return data;
  7794. }
  7795. else
  7796. {
  7797. //MORE: This should not load the whole property, and should be different from the code above...
  7798. MemoryBuffer s;
  7799. p->getPropBin("Value", s);
  7800. unsigned len = s.length();
  7801. if (from > len) from = len;
  7802. if (from + length > len) length = len - from;
  7803. data.setLen(s.readDirect(len) + from, (size32_t)length);
  7804. return data;
  7805. }
  7806. }
  7807. bool CLocalWUResult::getResultIsAll() const
  7808. {
  7809. return p->getPropBool("@isAll", false);
  7810. }
  7811. // MORE - it's an undetected error if we call setResult... of a type that does not match schema
  7812. void CLocalWUResult::setResultInt(__int64 val)
  7813. {
  7814. // Note: we always serialize scalar integer results as int8, and schema must reflect this
  7815. MemoryBuffer m;
  7816. serializeInt8(val, m);
  7817. p->setPropBin("Value", m.length(), m.toByteArray());
  7818. setResultRowCount(1);
  7819. setResultTotalRowCount(1);
  7820. }
  7821. void CLocalWUResult::setResultUInt(unsigned __int64 val)
  7822. {
  7823. setResultInt((__int64) val);
  7824. }
  7825. void CLocalWUResult::setResultReal(double val)
  7826. {
  7827. // Note: we always serialize scalar real results as real8, and schema must reflect this
  7828. MemoryBuffer m;
  7829. serializeReal8(val, m);
  7830. p->setPropBin("Value", m.length(), m.toByteArray());
  7831. setResultRowCount(1);
  7832. setResultTotalRowCount(1);
  7833. }
  7834. void CLocalWUResult::setResultBool(bool val)
  7835. {
  7836. MemoryBuffer m;
  7837. serializeBool(val, m);
  7838. p->setPropBin("Value", m.length(), m.toByteArray());
  7839. setResultRowCount(1);
  7840. setResultTotalRowCount(1);
  7841. }
  7842. void CLocalWUResult::setResultString(const char *val, unsigned len)
  7843. {
  7844. // Note: we always serialize scalar strings with length prefix, and schema must reflect this
  7845. MemoryBuffer m;
  7846. serializeLPString(len, val, m);
  7847. p->setPropBin("Value", m.length(), m.toByteArray());
  7848. setResultRowCount(1);
  7849. setResultTotalRowCount(1);
  7850. }
  7851. void CLocalWUResult::setResultUnicode(const void *val, unsigned len)
  7852. {
  7853. // Note: we always serialize scalar strings with length prefix, and schema must reflect this
  7854. MemoryBuffer m;
  7855. m.append(len).append(len*2, val);
  7856. p->setPropBin("Value", m.length(), m.toByteArray());
  7857. setResultRowCount(1);
  7858. setResultTotalRowCount(1);
  7859. }
  7860. void CLocalWUResult::setResultData(const void *val, unsigned len)
  7861. {
  7862. // Note: we always serialize scalar data with length prefix, and schema must reflect this
  7863. MemoryBuffer m;
  7864. serializeLPString(len, (const char *)val, m);
  7865. p->setPropBin("Value", m.length(), m.toByteArray());
  7866. setResultRowCount(1);
  7867. setResultTotalRowCount(1);
  7868. }
  7869. void CLocalWUResult::setResultDecimal(const void *val, unsigned len)
  7870. {
  7871. // Note: serialized as data but with length known from schema
  7872. MemoryBuffer m;
  7873. serializeFixedData(len, val, m);
  7874. p->setPropBin("Value", m.length(), m.toByteArray());
  7875. setResultRowCount(1);
  7876. setResultTotalRowCount(1);
  7877. }
  7878. void CLocalWUResult::setResultRow(unsigned len, const void * data)
  7879. {
  7880. p->setPropBin("Value", len, data);
  7881. setResultRowCount(1);
  7882. setResultTotalRowCount(1);
  7883. setResultFormat(ResultFormatRaw);
  7884. }
  7885. void CLocalWUResult::setResultIsAll(bool value)
  7886. {
  7887. p->setPropBool("@isAll", value);
  7888. }
  7889. //==========================================================================================
  7890. CLocalWUPlugin::CLocalWUPlugin(IPropertyTree *props) : p(props)
  7891. {
  7892. }
  7893. IStringVal& CLocalWUPlugin::getPluginName(IStringVal &str) const
  7894. {
  7895. str.set(p->queryProp("@dllname"));
  7896. return str;
  7897. }
  7898. IStringVal& CLocalWUPlugin::getPluginVersion(IStringVal &str) const
  7899. {
  7900. str.set(p->queryProp("@version"));
  7901. return str;
  7902. }
  7903. void CLocalWUPlugin::setPluginName(const char *str)
  7904. {
  7905. p->setProp("@dllname", str);
  7906. }
  7907. void CLocalWUPlugin::setPluginVersion(const char *str)
  7908. {
  7909. p->setProp("@version", str);
  7910. }
  7911. //==========================================================================================
  7912. CLocalWULibrary::CLocalWULibrary(IPropertyTree *props) : p(props)
  7913. {
  7914. }
  7915. IStringVal& CLocalWULibrary::getName(IStringVal &str) const
  7916. {
  7917. str.set(p->queryProp("@name"));
  7918. return str;
  7919. }
  7920. void CLocalWULibrary::setName(const char *str)
  7921. {
  7922. p->setProp("@name", str);
  7923. }
  7924. //==========================================================================================
  7925. CLocalWUException::CLocalWUException(IPropertyTree *props) : p(props)
  7926. {
  7927. }
  7928. IStringVal& CLocalWUException::getExceptionSource(IStringVal &str) const
  7929. {
  7930. str.set(p->queryProp("@source"));
  7931. return str;
  7932. }
  7933. IStringVal& CLocalWUException::getExceptionMessage(IStringVal &str) const
  7934. {
  7935. str.set(p->queryProp(NULL));
  7936. return str;
  7937. }
  7938. unsigned CLocalWUException::getExceptionCode() const
  7939. {
  7940. return p->getPropInt("@code", 0);
  7941. }
  7942. ErrorSeverity CLocalWUException::getSeverity() const
  7943. {
  7944. return (ErrorSeverity)p->getPropInt("@severity", SeverityError);
  7945. }
  7946. IStringVal & CLocalWUException::getTimeStamp(IStringVal & dt) const
  7947. {
  7948. dt.set(p->queryProp("@time"));
  7949. return dt;
  7950. }
  7951. IStringVal & CLocalWUException::getExceptionFileName(IStringVal & str) const
  7952. {
  7953. str.set(p->queryProp("@filename"));
  7954. return str;
  7955. }
  7956. unsigned CLocalWUException::getExceptionLineNo() const
  7957. {
  7958. return p->getPropInt("@row", 0);
  7959. }
  7960. unsigned CLocalWUException::getExceptionColumn() const
  7961. {
  7962. return p->getPropInt("@col", 0);
  7963. }
  7964. unsigned CLocalWUException::getActivityId() const
  7965. {
  7966. return p->getPropInt("@activity", 0);
  7967. }
  7968. unsigned CLocalWUException::getSequence() const
  7969. {
  7970. return p->getPropInt("@sequence", 0);
  7971. }
  7972. void CLocalWUException::setExceptionSource(const char *str)
  7973. {
  7974. p->setProp("@source", str);
  7975. }
  7976. void CLocalWUException::setExceptionMessage(const char *str)
  7977. {
  7978. p->setProp(NULL, str);
  7979. }
  7980. void CLocalWUException::setExceptionCode(unsigned code)
  7981. {
  7982. p->setPropInt("@code", code);
  7983. }
  7984. void CLocalWUException::setSeverity(ErrorSeverity level)
  7985. {
  7986. p->setPropInt("@severity", level);
  7987. }
  7988. void CLocalWUException::setTimeStamp(const char *str)
  7989. {
  7990. p->setProp("@time", str);
  7991. }
  7992. void CLocalWUException::setExceptionFileName(const char *str)
  7993. {
  7994. p->setProp("@filename", str);
  7995. }
  7996. void CLocalWUException::setExceptionLineNo(unsigned r)
  7997. {
  7998. p->setPropInt("@row", r);
  7999. }
  8000. void CLocalWUException::setExceptionColumn(unsigned c)
  8001. {
  8002. p->setPropInt("@col", c);
  8003. }
  8004. void CLocalWUException::setActivityId(unsigned _id)
  8005. {
  8006. p->setPropInt("@activity", _id);
  8007. }
  8008. //==========================================================================================
  8009. CLocalWUAppValue::CLocalWUAppValue(IPropertyTree *props, unsigned child) : p(props)
  8010. {
  8011. StringAttrBuilder propPath(prop);
  8012. propPath.append("*[").append(child).append("]");
  8013. }
  8014. const char * CLocalWUAppValue::queryApplication() const
  8015. {
  8016. return p->queryName();
  8017. }
  8018. const char * CLocalWUAppValue::queryName() const
  8019. {
  8020. IPropertyTree* val=p->queryPropTree(prop.str());
  8021. if(val)
  8022. return val->queryName();
  8023. return ""; // Should not happen in normal usage
  8024. }
  8025. const char * CLocalWUAppValue::queryValue() const
  8026. {
  8027. return p->queryProp(prop.str());
  8028. }
  8029. //==========================================================================================
  8030. CLocalWUStatistic::CLocalWUStatistic(IPropertyTree *props) : p(props)
  8031. {
  8032. }
  8033. IStringVal & CLocalWUStatistic::getCreator(IStringVal & str) const
  8034. {
  8035. const char * creator = p->queryProp("@creator");
  8036. str.set(creator);
  8037. return str;
  8038. }
  8039. IStringVal & CLocalWUStatistic::getDescription(IStringVal & str, bool createDefault) const
  8040. {
  8041. const char * desc = p->queryProp("@desc");
  8042. if (desc)
  8043. {
  8044. str.set(desc); // legacy and in case it is overridden
  8045. }
  8046. else if (createDefault)
  8047. {
  8048. StatisticKind kind = getKind();
  8049. assertex(kind != StKindNone);
  8050. const char * scope = p->queryProp("@scope");
  8051. assertex(scope);
  8052. //Clean up the format of the scope when converting it to a description
  8053. StringBuffer descriptionText;
  8054. if (streq(scope, GLOBAL_SCOPE))
  8055. {
  8056. const char * creator = p->queryProp("@creator");
  8057. descriptionText.append(creator).append(":");
  8058. queryLongStatisticName(descriptionText, kind);
  8059. }
  8060. else
  8061. {
  8062. for (;;)
  8063. {
  8064. char c = *scope++;
  8065. if (!c)
  8066. break;
  8067. if (c == ':')
  8068. descriptionText.append(": ");
  8069. else
  8070. descriptionText.append(c);
  8071. }
  8072. if (kind != StTimeElapsed)
  8073. queryLongStatisticName(descriptionText.append(": "), kind);
  8074. }
  8075. str.set(descriptionText);
  8076. }
  8077. else
  8078. str.clear();
  8079. return str;
  8080. }
  8081. IStringVal & CLocalWUStatistic::getType(IStringVal & str) const
  8082. {
  8083. StatisticKind kind = getKind();
  8084. if (kind != StKindNone)
  8085. str.set(queryStatisticName(kind));
  8086. return str;
  8087. }
  8088. IStringVal & CLocalWUStatistic::getFormattedValue(IStringVal & str) const
  8089. {
  8090. StringBuffer formatted;
  8091. formatStatistic(formatted, getValue(), getMeasure());
  8092. str.set(formatted);
  8093. return str;
  8094. }
  8095. StatisticCreatorType CLocalWUStatistic::getCreatorType() const
  8096. {
  8097. return queryCreatorType(p->queryProp("@c"));
  8098. }
  8099. StatisticScopeType CLocalWUStatistic::getScopeType() const
  8100. {
  8101. return queryScopeType(p->queryProp("@s"));
  8102. }
  8103. StatisticKind CLocalWUStatistic::getKind() const
  8104. {
  8105. return queryStatisticKind(p->queryProp("@kind"));
  8106. }
  8107. IStringVal & CLocalWUStatistic::getScope(IStringVal & str) const
  8108. {
  8109. const char * scope = p->queryProp("@scope");
  8110. str.set(scope);
  8111. return str;
  8112. }
  8113. StatisticMeasure CLocalWUStatistic::getMeasure() const
  8114. {
  8115. return queryMeasure(p->queryProp("@unit"));
  8116. }
  8117. unsigned __int64 CLocalWUStatistic::getValue() const
  8118. {
  8119. return p->getPropInt64("@value", 0);
  8120. }
  8121. unsigned __int64 CLocalWUStatistic::getCount() const
  8122. {
  8123. return p->getPropInt64("@count", 0);
  8124. }
  8125. unsigned __int64 CLocalWUStatistic::getMax() const
  8126. {
  8127. return p->getPropInt64("@max", 0);
  8128. }
  8129. unsigned __int64 CLocalWUStatistic::getTimestamp() const
  8130. {
  8131. return p->getPropInt64("@ts", 0);
  8132. }
  8133. bool CLocalWUStatistic::matches(const IStatisticsFilter * filter) const
  8134. {
  8135. if (!filter)
  8136. return true;
  8137. const char * creator = p->queryProp("@creator");
  8138. const char * scope = p->queryProp("@scope");
  8139. return filter->matches(getCreatorType(), creator, getScopeType(), scope, getMeasure(), getKind(), getValue());
  8140. }
  8141. //==========================================================================================
  8142. CLocalWULegacyTiming::CLocalWULegacyTiming(IPropertyTree *props) : p(props)
  8143. {
  8144. }
  8145. IStringVal & CLocalWULegacyTiming::getCreator(IStringVal & str) const
  8146. {
  8147. str.clear();
  8148. return str;
  8149. }
  8150. IStringVal & CLocalWULegacyTiming::getDescription(IStringVal & str, bool createDefault) const
  8151. {
  8152. str.set(p->queryProp("@name"));
  8153. return str;
  8154. }
  8155. IStringVal & CLocalWULegacyTiming::getType(IStringVal & str) const
  8156. {
  8157. str.set(queryStatisticName(StTimeElapsed));
  8158. return str;
  8159. }
  8160. IStringVal & CLocalWULegacyTiming::getFormattedValue(IStringVal & str) const
  8161. {
  8162. StringBuffer formatted;
  8163. formatStatistic(formatted, getValue(), getMeasure());
  8164. str.set(formatted);
  8165. return str;
  8166. }
  8167. StatisticCreatorType CLocalWULegacyTiming::getCreatorType() const
  8168. {
  8169. return SCTunknown;
  8170. }
  8171. StatisticScopeType CLocalWULegacyTiming::getScopeType() const
  8172. {
  8173. return SSTnone;
  8174. }
  8175. StatisticKind CLocalWULegacyTiming::getKind() const
  8176. {
  8177. return StTimeElapsed;
  8178. }
  8179. IStringVal & CLocalWULegacyTiming::getScope(IStringVal & str) const
  8180. {
  8181. str.clear();
  8182. return str;
  8183. }
  8184. StatisticMeasure CLocalWULegacyTiming::getMeasure() const
  8185. {
  8186. return SMeasureTimeNs;
  8187. }
  8188. unsigned __int64 CLocalWULegacyTiming::getValue() const
  8189. {
  8190. return p->getPropInt64("@duration", 0) * 1000000;
  8191. }
  8192. unsigned __int64 CLocalWULegacyTiming::getCount() const
  8193. {
  8194. return p->getPropInt64("@count", 0);
  8195. }
  8196. unsigned __int64 CLocalWULegacyTiming::getMax() const
  8197. {
  8198. return p->getPropInt64("@max", 0);
  8199. }
  8200. unsigned __int64 CLocalWULegacyTiming::getTimestamp() const
  8201. {
  8202. return 0;
  8203. }
  8204. bool CLocalWULegacyTiming::matches(const IStatisticsFilter * filter) const
  8205. {
  8206. if (!filter)
  8207. return true;
  8208. const char * creator = p->queryProp("@creator");
  8209. const char * scope = p->queryProp("@scope");
  8210. return filter->matches(SCTall, NULL, SSTall, NULL, getMeasure(), getKind(), getValue());
  8211. }
  8212. //==========================================================================================
  8213. extern WORKUNIT_API ILocalWorkUnit * createLocalWorkUnit(const char *xml)
  8214. {
  8215. Owned<CLocalWorkUnit> cw = new CLocalWorkUnit((ISecManager *) NULL, NULL);
  8216. if (xml)
  8217. cw->loadPTree(createPTreeFromXMLString(xml, ipt_lowmem));
  8218. else
  8219. {
  8220. Owned<IPropertyTree> p = createPTree("W_LOCAL", ipt_lowmem);
  8221. p->setProp("@xmlns:xsi", "http://www.w3.org/1999/XMLSchema-instance");
  8222. cw->loadPTree(p.getClear());
  8223. }
  8224. ILocalWorkUnit* ret = QUERYINTERFACE(&cw->lockRemote(false), ILocalWorkUnit);
  8225. return ret;
  8226. }
  8227. void exportWorkUnitToXMLWithHiddenPasswords(IPropertyTree *p, IIOStream &out, unsigned extraXmlFlags)
  8228. {
  8229. const char *name = p->queryName();
  8230. if (!name)
  8231. name = "__unnamed__";
  8232. StringBuffer temp;
  8233. writeStringToStream(out, appendPTreeOpenTag(temp, p, name, 1, true));
  8234. Owned<IPropertyTreeIterator> elems = p->getElements("*", iptiter_sort);
  8235. ForEach(*elems)
  8236. {
  8237. IPropertyTree &elem = elems->query();
  8238. if (streq(elem.queryName(), "Parameters"))
  8239. {
  8240. writeStringToStream(out, appendPTreeOpenTag(temp.clear().append(' '), &elem, "Parameters", 2, false).append('\n'));
  8241. Owned<IPropertyTreeIterator> params = elem.getElements("*", iptiter_sort);
  8242. ForEach(*params)
  8243. {
  8244. IPropertyTree &param = params->query();
  8245. const char *paramname = param.queryName();
  8246. VStringBuffer xpath("Variables/Variable[@name='%s']/Format/@password", paramname);
  8247. if (p->getPropBool(xpath))
  8248. writeStringToStream(out, appendXMLTag(temp.clear().append(" "), paramname, "****").append('\n'));
  8249. else
  8250. {
  8251. toXML(&param, out, 2, XML_Format|XML_SortTags|extraXmlFlags);
  8252. }
  8253. }
  8254. writeStringToStream(out, appendXMLCloseTag(temp.clear().append(' '), "Parameters").append('\n'));
  8255. }
  8256. else if (streq(elem.queryName(), "Variables"))
  8257. {
  8258. writeStringToStream(out, appendPTreeOpenTag(temp.clear().append(' '), &elem, "Variables", 2, false).append('\n'));
  8259. Owned<IPropertyTreeIterator> vars = elem.getElements("*", iptiter_sort);
  8260. ForEach(*vars)
  8261. {
  8262. Owned<IPropertyTree> var = LINK(&vars->query());
  8263. if (var->getPropBool("Format/@password"))
  8264. {
  8265. var.setown(createPTreeFromIPT(var)); //copy and remove password values
  8266. var->removeProp("Value");
  8267. var->removeProp("xmlValue");
  8268. }
  8269. toXML(var, out, 2, XML_Format|XML_SortTags|extraXmlFlags);
  8270. }
  8271. writeStringToStream(out, appendXMLCloseTag(temp.clear().append(' '), "Variables").append('\n'));
  8272. }
  8273. else
  8274. toXML(&elem, out, 1, XML_Format|XML_SortTags|extraXmlFlags);
  8275. }
  8276. writeStringToStream(out, appendXMLCloseTag(temp.clear(), name));
  8277. }
  8278. StringBuffer &exportWorkUnitToXMLWithHiddenPasswords(IPropertyTree *p, StringBuffer &str)
  8279. {
  8280. class CAdapter : public CInterface, implements IIOStream
  8281. {
  8282. StringBuffer &out;
  8283. public:
  8284. IMPLEMENT_IINTERFACE;
  8285. CAdapter(StringBuffer &_out) : out(_out) { }
  8286. virtual void flush() { }
  8287. virtual size32_t read(size32_t len, void * data) { UNIMPLEMENTED; return 0; }
  8288. virtual size32_t write(size32_t len, const void * data) { out.append(len, (const char *)data); return len; }
  8289. } adapter(str);
  8290. exportWorkUnitToXMLWithHiddenPasswords(p->queryBranch(NULL), adapter, 0);
  8291. return str;
  8292. }
  8293. void exportWorkUnitToXMLFileWithHiddenPasswords(IPropertyTree *p, const char *filename, unsigned extraXmlFlags)
  8294. {
  8295. OwnedIFile ifile = createIFile(filename);
  8296. OwnedIFileIO ifileio = ifile->open(IFOcreate);
  8297. Owned<IIOStream> stream = createIOStream(ifileio);
  8298. exportWorkUnitToXMLWithHiddenPasswords(p->queryBranch(NULL), *stream, extraXmlFlags);
  8299. }
  8300. extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool unpack, bool includeProgress, bool hidePasswords)
  8301. {
  8302. // MORE - queryPTree isn't really safe without holding CLocalWorkUnit::crit - really need to move these functions into CLocalWorkunit
  8303. const IExtendedWUInterface *ewu = queryExtendedWU(wu);
  8304. if (ewu)
  8305. {
  8306. Linked<IPropertyTree> p;
  8307. if (unpack||includeProgress)
  8308. p.setown(ewu->getUnpackedTree(includeProgress));
  8309. else
  8310. p.set(ewu->queryPTree());
  8311. if (hidePasswords)
  8312. return exportWorkUnitToXMLWithHiddenPasswords(p, str);
  8313. else
  8314. return toXML(p, str, 0, XML_Format|XML_SortTags);
  8315. }
  8316. else
  8317. return str.append("Unrecognized workunit format");
  8318. }
  8319. extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool unpack, bool includeProgress, bool hidePasswords, bool splitStats)
  8320. {
  8321. const IExtendedWUInterface *ewu = queryExtendedWU(wu);
  8322. if (ewu)
  8323. {
  8324. Linked<IPropertyTree> p;
  8325. if (unpack||includeProgress)
  8326. p.setown(ewu->getUnpackedTree(includeProgress));
  8327. else
  8328. p.set(ewu->queryPTree());
  8329. if (hidePasswords)
  8330. return exportWorkUnitToXMLFileWithHiddenPasswords(p, filename, extraXmlFlags);
  8331. if (splitStats)
  8332. {
  8333. StringBuffer statsFilename;
  8334. statsFilename.append(filename).append(".stats");
  8335. IPropertyTree * stats = p->queryPropTree("Statistics");
  8336. if (stats)
  8337. {
  8338. saveXML(statsFilename, stats, 0, (XML_Format|XML_SortTags|extraXmlFlags) & ~XML_LineBreakAttributes);
  8339. p->removeProp("Statistics");
  8340. }
  8341. }
  8342. saveXML(filename, p, 0, XML_Format|XML_SortTags|extraXmlFlags);
  8343. }
  8344. else
  8345. throw makeStringException(0, "Unrecognized workunit format");
  8346. }
  8347. extern WORKUNIT_API void submitWorkUnit(const char *wuid, const char *username, const char *password)
  8348. {
  8349. MemoryBuffer buffer;
  8350. Owned<INamedQueueConnection> conn = createNamedQueueConnection(0); // MORE - security token?
  8351. Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
  8352. Owned<IWorkUnit> workunit = factory->updateWorkUnit(wuid);
  8353. assertex(workunit);
  8354. SCMStringBuffer token;
  8355. createToken(wuid, username, password, token);
  8356. workunit->setSecurityToken(token.str());
  8357. StringAttr clusterName(workunit->queryClusterName());
  8358. if (!clusterName.length())
  8359. throw MakeStringException(WUERR_InvalidCluster, "No target cluster specified");
  8360. workunit->commit();
  8361. workunit.clear();
  8362. Owned<IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(clusterName.str());
  8363. if (!clusterInfo)
  8364. throw MakeStringException(WUERR_InvalidCluster, "Unknown cluster %s", clusterName.str());
  8365. SCMStringBuffer serverQueue;
  8366. clusterInfo->getServerQueue(serverQueue);
  8367. assertex(serverQueue.length());
  8368. Owned<IJobQueue> queue = createJobQueue(serverQueue.str());
  8369. if (!queue.get())
  8370. throw MakeStringException(WUERR_InvalidQueue, "Could not create workunit queue");
  8371. IJobQueueItem *item = createJobQueueItem(wuid);
  8372. queue->enqueue(item);
  8373. }
  8374. extern WORKUNIT_API void abortWorkUnit(const char *wuid)
  8375. {
  8376. StringBuffer xpath("/WorkUnitAborts/");
  8377. xpath.append(wuid);
  8378. Owned<IRemoteConnection> acon = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE, SDS_LOCK_TIMEOUT);
  8379. acon->queryRoot()->setPropInt(NULL, 1);
  8380. }
  8381. extern WORKUNIT_API void secSubmitWorkUnit(const char *wuid, ISecManager &secmgr, ISecUser &secuser)
  8382. {
  8383. if (checkWuSecAccess(wuid, &secmgr, &secuser, SecAccess_Write, "Submit", true, true))
  8384. submitWorkUnit(wuid, secuser.getName(), secuser.credentials().getPassword());
  8385. }
  8386. extern WORKUNIT_API void secAbortWorkUnit(const char *wuid, ISecManager &secmgr, ISecUser &secuser)
  8387. {
  8388. if (!checkWuSecAccess(wuid, &secmgr, &secuser, SecAccess_Write, "Submit", true, true))
  8389. return;
  8390. abortWorkUnit(wuid);
  8391. Owned<IConstWorkUnit> cw = factory->openWorkUnit(wuid);
  8392. if(!cw)
  8393. return;
  8394. WorkunitUpdate wu(&cw->lock());
  8395. const char *abortBy = secuser.getName();
  8396. if (abortBy && *abortBy)
  8397. wu->setTracingValue("AbortBy", abortBy);
  8398. wu->setTracingValueInt64("AbortTimeStamp", getTimeStampNowValue());
  8399. }
  8400. extern WORKUNIT_API void submitWorkUnit(const char *wuid, ISecManager *secmgr, ISecUser *secuser)
  8401. {
  8402. if (secmgr && secuser)
  8403. return secSubmitWorkUnit(wuid, *secmgr, *secuser);
  8404. if (secuser)
  8405. return submitWorkUnit(wuid, secuser->getName(), secuser->credentials().getPassword());
  8406. submitWorkUnit(wuid, "", "");
  8407. }
  8408. extern WORKUNIT_API void abortWorkUnit(const char *wuid, ISecManager *secmgr, ISecUser *secuser)
  8409. {
  8410. if (secmgr && secuser)
  8411. return secAbortWorkUnit(wuid, *secmgr, *secuser);
  8412. abortWorkUnit(wuid);
  8413. }
  8414. bool CLocalWorkUnit::hasWorkflow() const
  8415. {
  8416. return p->hasProp("Workflow");
  8417. }
  8418. unsigned CLocalWorkUnit::queryEventScheduledCount() const
  8419. {
  8420. CriticalBlock block(crit);
  8421. if (p->hasProp("@eventScheduledCount"))
  8422. return p->getPropInt("@eventScheduledCount", 0);
  8423. else
  8424. return p->getPropInt("Workflow/@eventScheduledCount", 0); // Legacy location for this setting
  8425. }
  8426. void CLocalWorkUnit::incEventScheduledCount()
  8427. {
  8428. CriticalBlock block(crit);
  8429. p->setPropInt("@eventScheduledCount", queryEventScheduledCount()+1);
  8430. }
  8431. IPropertyTree * CLocalWorkUnit::queryWorkflowTree() const
  8432. {
  8433. CriticalBlock block(crit);
  8434. return p->queryPropTree("Workflow");
  8435. }
  8436. IConstWorkflowItemIterator* CLocalWorkUnit::getWorkflowItems() const
  8437. {
  8438. // For this to be legally called, we must have the read-able interface. So we are already locked for (at least) read.
  8439. CriticalBlock block(crit);
  8440. if(!workflowIteratorCached)
  8441. {
  8442. assertex(!workflowIterator);
  8443. Owned<IPropertyTree> s = p->getPropTree("Workflow");
  8444. if(s)
  8445. workflowIterator.setown(createWorkflowItemIterator(s));
  8446. workflowIteratorCached = true;
  8447. }
  8448. return workflowIterator.getLink();
  8449. }
  8450. IWorkflowItemArray * CLocalWorkUnit::getWorkflowClone() const
  8451. {
  8452. unsigned count = 0;
  8453. Owned<IConstWorkflowItemIterator> iter = getWorkflowItems();
  8454. for(iter->first(); iter->isValid(); iter->next())
  8455. count++;
  8456. Owned<IWorkflowItemArray> array = createWorkflowItemArray(count);
  8457. for(iter->first(); iter->isValid(); iter->next())
  8458. array->addClone(iter->query());
  8459. return array.getLink();
  8460. }
  8461. IWorkflowItem * CLocalWorkUnit::addWorkflowItem(unsigned wfid, WFType type, WFMode mode, unsigned success, unsigned failure, unsigned recovery, unsigned retriesAllowed, unsigned contingencyFor)
  8462. {
  8463. // For this to be legally called, we must have the write-able interface. So we are already locked for write.
  8464. CriticalBlock block(crit);
  8465. workflowIterator.clear();
  8466. workflowIteratorCached = false;
  8467. IPropertyTree * s = p->queryPropTree("Workflow");
  8468. if(!s)
  8469. s = p->addPropTree("Workflow");
  8470. return createWorkflowItem(s, wfid, type, mode, success, failure, recovery, retriesAllowed, contingencyFor);
  8471. }
  8472. IWorkflowItemIterator * CLocalWorkUnit::updateWorkflowItems()
  8473. {
  8474. // For this to be legally called, we must have the write-able interface. So we are already locked for write.
  8475. CriticalBlock block(crit);
  8476. if(!workflowIterator)
  8477. {
  8478. IPropertyTree * s = p->queryPropTree("Workflow");
  8479. if(!s)
  8480. s = p->addPropTree("Workflow");
  8481. workflowIterator.setown(createWorkflowItemIterator(s));
  8482. workflowIteratorCached = true;
  8483. }
  8484. return workflowIterator.getLink();
  8485. }
  8486. void CLocalWorkUnit::syncRuntimeWorkflow(IWorkflowItemArray * array)
  8487. {
  8488. Owned<IWorkflowItemIterator> iter = updateWorkflowItems();
  8489. Owned<IWorkflowItem> item;
  8490. for(iter->first(); iter->isValid(); iter->next())
  8491. {
  8492. item.setown(iter->get());
  8493. item->syncRuntimeData(array->queryWfid(item->queryWfid()));
  8494. }
  8495. workflowIterator.clear();
  8496. workflowIteratorCached = false;
  8497. }
  8498. void CLocalWorkUnit::resetWorkflow()
  8499. {
  8500. if (hasWorkflow())
  8501. {
  8502. Owned<IWorkflowItemIterator> iter = updateWorkflowItems();
  8503. Owned<IWorkflowItem> wf;
  8504. for(iter->first(); iter->isValid(); iter->next())
  8505. {
  8506. wf.setown(iter->get());
  8507. wf->reset();
  8508. }
  8509. workflowIterator.clear();
  8510. workflowIteratorCached = false;
  8511. }
  8512. }
  8513. void CLocalWorkUnit::schedule()
  8514. {
  8515. CriticalBlock block(crit);
  8516. if(queryEventScheduledCount() == 0) return;
  8517. switch(getState())
  8518. {
  8519. case WUStateCompleted:
  8520. setState(WUStateWait);
  8521. break;
  8522. case WUStateFailed:
  8523. case WUStateArchived:
  8524. case WUStateAborting:
  8525. case WUStateAborted:
  8526. case WUStateScheduled:
  8527. throw MakeStringException(WUERR_CannotSchedule, "Cannot schedule workunit in this state");
  8528. }
  8529. StringBuffer rootPath;
  8530. rootPath.append("/Schedule/").append(queryClusterName());
  8531. Owned<IRemoteConnection> conn = querySDS().connect(rootPath.str(), myProcessSession(), RTM_LOCK_WRITE | RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
  8532. Owned<IPropertyTree> root = conn->getRoot();
  8533. if(!root->hasChildren())
  8534. {
  8535. StringBuffer addPath;
  8536. addPath.append("/Schedulers/").append(queryClusterName());
  8537. Owned<IRemoteConnection> addConn = querySDS().connect(addPath.str(), myProcessSession(), RTM_LOCK_WRITE | RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
  8538. }
  8539. char const * wuid = p->queryName();
  8540. StringBuffer xpath("*/*/");
  8541. ncnameEscape(wuid, xpath);
  8542. bool more;
  8543. do more = root->removeProp(xpath.str()); while(more);
  8544. Owned<IConstWorkflowItemIterator> iter = getWorkflowItems();
  8545. Owned<IWorkflowEvent> event;
  8546. Owned<IPropertyTree> branch1, branch2;
  8547. for(iter->first(); iter->isValid(); iter->next())
  8548. {
  8549. event.setown(iter->query()->getScheduleEvent());
  8550. if(!event) continue;
  8551. ncnameEscape(event->queryName(), xpath.clear());
  8552. ensurePTree(root, xpath.str());
  8553. branch1.setown(root->getPropTree(xpath.str()));
  8554. ncnameEscape(event->queryText(), xpath.clear());
  8555. ensurePTree(branch1, xpath.str());
  8556. branch2.setown(branch1->getPropTree(xpath.str()));
  8557. ncnameEscape(wuid, xpath.clear());
  8558. ensurePTree(branch2, xpath.str());
  8559. }
  8560. }
  8561. void CLocalWorkUnit::deschedule()
  8562. {
  8563. if(queryEventScheduledCount() == 0) return;
  8564. if(getState() == WUStateWait)
  8565. setState(WUStateCompleted);
  8566. doDescheduleWorkkunit(p->queryName());
  8567. }
  8568. mapEnums localFileUploadTypes[] = {
  8569. { UploadTypeFileSpray, "FileSpray" },
  8570. { UploadTypeWUResult, "WUResult" },
  8571. { UploadTypeWUResultCsv, "WUResultCsv" },
  8572. { UploadTypeWUResultXml, "WUResultXml" },
  8573. { UploadTypeSize, NULL }
  8574. };
  8575. class CLocalFileUpload : public CInterface, implements IConstLocalFileUpload
  8576. {
  8577. public:
  8578. CLocalFileUpload(IPropertyTree * _tree) : tree(_tree) {}
  8579. CLocalFileUpload(unsigned id, LocalFileUploadType type, char const * source, char const * destination, char const * eventTag)
  8580. {
  8581. tree.setown(createPTree());
  8582. tree->setPropInt("@id", id);
  8583. setEnum(tree, "@type", type, localFileUploadTypes);
  8584. tree->setProp("@source", source);
  8585. tree->setProp("@destination", destination);
  8586. if (eventTag)
  8587. tree->setProp("@eventTag", eventTag);
  8588. }
  8589. IMPLEMENT_IINTERFACE;
  8590. IPropertyTree * getTree() { return tree.getLink(); }
  8591. virtual unsigned queryID() const { return tree->getPropInt("@id"); }
  8592. virtual LocalFileUploadType queryType() const { return (LocalFileUploadType)getEnum(tree, "@type", localFileUploadTypes); }
  8593. virtual IStringVal & getSource(IStringVal & ret) const { ret.set(tree->queryProp("@source")); return ret; }
  8594. virtual IStringVal & getDestination(IStringVal & ret) const { ret.set(tree->queryProp("@destination")); return ret; }
  8595. virtual IStringVal & getEventTag(IStringVal & ret) const { if(tree->hasProp("@eventTag")) ret.set(tree->queryProp("@eventTag")); else ret.clear(); return ret; }
  8596. private:
  8597. Owned<IPropertyTree> tree;
  8598. };
  8599. class CLocalFileUploadIterator : public CInterface, implements IConstLocalFileUploadIterator
  8600. {
  8601. public:
  8602. CLocalFileUploadIterator(IPropertyTree * _tree) : tree(_tree), iter(tree->getElements("LocalFileUpload")) {}
  8603. IMPLEMENT_IINTERFACE;
  8604. bool first() { return iter->first(); }
  8605. bool isValid() { return iter->isValid(); }
  8606. bool next() { return iter->next(); }
  8607. IConstLocalFileUpload * get() { return new CLocalFileUpload(&iter->get()); }
  8608. private:
  8609. Owned<IPropertyTree> tree;
  8610. Owned<IPropertyTreeIterator> iter;
  8611. };
  8612. IConstLocalFileUploadIterator * CLocalWorkUnit::getLocalFileUploads() const
  8613. {
  8614. // For this to be legally called, we must have the read-able interface. So we are already locked for (at least) read.
  8615. CriticalBlock block(crit);
  8616. Owned<IPropertyTree> s = p->getPropTree("LocalFileUploads");
  8617. if(s)
  8618. return new CLocalFileUploadIterator(s.getClear());
  8619. else
  8620. return NULL;
  8621. }
  8622. bool CLocalWorkUnit::requiresLocalFileUpload() const
  8623. {
  8624. SCMStringBuffer dest;
  8625. Owned<IConstWUResult> result;
  8626. Owned<IConstLocalFileUploadIterator> iter(getLocalFileUploads());
  8627. if(!iter)
  8628. return false;
  8629. for(iter->first(); iter->isValid(); iter->next())
  8630. {
  8631. Owned<IConstLocalFileUpload> upload(iter->get());
  8632. switch(upload->queryType())
  8633. {
  8634. case UploadTypeWUResult:
  8635. case UploadTypeWUResultCsv:
  8636. case UploadTypeWUResultXml:
  8637. upload->getDestination(dest);
  8638. result.setown(getResultByName(dest.str()));
  8639. if(!result)
  8640. return true;
  8641. break;
  8642. default:
  8643. throw MakeStringException(WUERR_InvalidUploadFormat, "Unsupported local file upload type %s", getEnumText(upload->queryType(), localFileUploadTypes));
  8644. }
  8645. }
  8646. return false;
  8647. }
  8648. unsigned CLocalWorkUnit::addLocalFileUpload(LocalFileUploadType type, char const * source, char const * destination, char const * eventTag)
  8649. {
  8650. // For this to be legally called, we must have the write-able interface. So we are already locked for write.
  8651. CriticalBlock block(crit);
  8652. IPropertyTree * s = p->queryPropTree("LocalFileUploads");
  8653. if(!s)
  8654. s = p->addPropTree("LocalFileUploads");
  8655. unsigned id = s->numChildren();
  8656. Owned<CLocalFileUpload> upload = new CLocalFileUpload(id, type, source, destination, eventTag);
  8657. s->addPropTree("LocalFileUpload", upload->getTree());
  8658. return id;
  8659. }
  8660. IStringVal & CLocalWorkUnit::getAbortBy(IStringVal & str) const
  8661. {
  8662. CriticalBlock block(crit);
  8663. str.set(p->queryProp("Tracing/AbortBy"));
  8664. return str;
  8665. }
  8666. unsigned __int64 CLocalWorkUnit::getAbortTimeStamp() const
  8667. {
  8668. CriticalBlock block(crit);
  8669. return p->getPropInt64("Tracing/AbortTimeStamp", 0);
  8670. }
  8671. #if 0
  8672. void testConstWorkflow(IConstWorkflowItem * cwf, bool * okay, bool * dep)
  8673. {
  8674. DBGLOG("Test workflow const iface %u", cwf->queryWfid());
  8675. unsigned deps = 0;
  8676. Owned<IWorkflowDependencyIterator> diter;
  8677. switch(cwf->queryWfid())
  8678. {
  8679. case 1:
  8680. assertex(!cwf->isScheduled());
  8681. assertex(cwf->queryType() == WFTypeNormal);
  8682. assertex(cwf->queryState() == WFStateNull);
  8683. diter.setown(cwf->getDependencies());
  8684. for(diter->first(); diter->isValid(); diter->next())
  8685. deps++;
  8686. assertex(deps==0);
  8687. okay[0] = true;
  8688. break;
  8689. case 2:
  8690. assertex(!cwf->isScheduled());
  8691. assertex(cwf->queryType() == WFTypeRecovery);
  8692. assertex(cwf->queryState() == WFStateSkip);
  8693. okay[1] = true;
  8694. break;
  8695. case 3:
  8696. assertex(cwf->queryContingencyFor() == 4);
  8697. okay[2] = true;
  8698. break;
  8699. case 4:
  8700. assertex(cwf->isScheduled());
  8701. assertex(cwf->queryType() == WFTypeNormal);
  8702. assertex(cwf->queryState() == WFStateReqd);
  8703. assertex(cwf->querySuccess() == 0);
  8704. assertex(cwf->queryFailure() == 3);
  8705. assertex(cwf->queryRecovery() == 2);
  8706. assertex(cwf->queryRetriesAllowed() == 10);
  8707. assertex(cwf->queryRetriesRemaining() == 10);
  8708. diter.setown(cwf->getDependencies());
  8709. for(diter->first(); diter->isValid(); diter->next())
  8710. {
  8711. dep[diter->query()-1] = true;
  8712. deps++;
  8713. }
  8714. assertex(deps==2);
  8715. assertex(dep[0]);
  8716. assertex(dep[1]);
  8717. okay[3] = true;
  8718. break;
  8719. case 5:
  8720. assertex(cwf->isScheduled());
  8721. assertex(!cwf->isScheduledNow());
  8722. assertex(cwf->querySchedulePriority() == 75);
  8723. assertex(cwf->queryScheduleCount() == 5);
  8724. assertex(cwf->queryScheduleCountRemaining() == 5);
  8725. okay[4] = true;
  8726. break;
  8727. case 6:
  8728. assertex(cwf->isScheduled());
  8729. assertex(!cwf->isScheduledNow());
  8730. assertex(cwf->querySchedulePriority() == 25);
  8731. assertex(!cwf->hasScheduleCount());
  8732. okay[5] = true;
  8733. break;
  8734. default:
  8735. assertex(!"unknown wfid in test");
  8736. }
  8737. }
  8738. void testRuntimeWorkflow(IRuntimeWorkflowItem * rwf, bool * okay)
  8739. {
  8740. DBGLOG("Test workflow runtime iface %u", rwf->queryWfid());
  8741. switch(rwf->queryWfid())
  8742. {
  8743. case 1:
  8744. case 2:
  8745. case 3:
  8746. okay[rwf->queryWfid()-1] = true;
  8747. break;
  8748. case 4:
  8749. {
  8750. unsigned tries = 0;
  8751. while(rwf->testAndDecRetries())
  8752. tries++;
  8753. assertex(tries == 10);
  8754. assertex(rwf->queryRetriesRemaining() == 0);
  8755. rwf->setState(WFStateFail);
  8756. assertex(rwf->queryState() == WFStateFail);
  8757. rwf->reset();
  8758. assertex(rwf->queryRetriesRemaining() == 10);
  8759. assertex(rwf->queryState() == WFStateReqd);
  8760. }
  8761. okay[3] = true;
  8762. break;
  8763. case 5:
  8764. {
  8765. assertex(rwf->queryScheduleCountRemaining() == 5);
  8766. unsigned count = 0;
  8767. do count++; while(rwf->decAndTestScheduleCountRemaining());
  8768. assertex(count == 5);
  8769. assertex(rwf->queryScheduleCountRemaining() == 0);
  8770. rwf->reset();
  8771. assertex(rwf->queryScheduleCountRemaining() == 5);
  8772. }
  8773. okay[4] = true;
  8774. break;
  8775. case 6:
  8776. {
  8777. assertex(!rwf->hasScheduleCount());
  8778. unsigned count;
  8779. for(count=0; count<20; count++)
  8780. assertex(rwf->decAndTestScheduleCountRemaining());
  8781. }
  8782. okay[5] = true;
  8783. break;
  8784. default:
  8785. assertex(!"unknown wfid in test");
  8786. }
  8787. }
  8788. void testWorkflow()
  8789. {
  8790. DBGLOG("workunit.cpp : testWorkflow");
  8791. CLocalWorkUnit wu("W-WF-TEST", 0, 0, 0);
  8792. Owned<IWorkflowItem> wf;
  8793. wf.setown(wu.addWorkflowItem(1, WFTypeNormal, 0, 0, 0, 0, 0));
  8794. wf.setown(wu.addWorkflowItem(2, WFTypeRecovery, 0, 0, 0, 0, 0));
  8795. wf.setown(wu.addWorkflowItem(3, WFTypeFailure, 0, 0, 0, 0, 4));
  8796. wf.setown(wu.addWorkflowItem(4, WFTypeNormal, 0, 3, 2, 10, 0));
  8797. wf->setScheduledNow();
  8798. wf->addDependency(1);
  8799. wf.setown(wu.addWorkflowItem(5, WFTypeNormal, 0, 0, 0, 0, 0));
  8800. wf->setScheduledOn("test", "foo*");
  8801. wf->setSchedulePriority(75);
  8802. wf->setScheduleCount(5);
  8803. wf.setown(wu.addWorkflowItem(6, WFTypeNormal, 0, 0, 0, 0, 0));
  8804. wf->setScheduledOn("test", "bar*");
  8805. wf->setSchedulePriority(25);
  8806. unsigned const n = 6;
  8807. bool okay[n];
  8808. bool dep[n];
  8809. unsigned i;
  8810. for(i=0; i<n; i++)
  8811. okay[i] = dep[i] = 0;
  8812. Owned<IConstWorkflowItemIterator> citer(wu.getWorkflowItems());
  8813. for(citer->first(); citer->isValid(); citer->next())
  8814. testConstWorkflow(citer->query(), okay, dep);
  8815. for(i=0; i<n; i++)
  8816. {
  8817. assertex(okay[i]);
  8818. okay[i] = false;
  8819. }
  8820. Owned<IWorkflowItemIterator> miter(wu.updateWorkflowItems());
  8821. for(miter->first(); miter->isValid(); miter->next())
  8822. {
  8823. Owned<IRuntimeWorkflowItem> rwf(miter->get());
  8824. testRuntimeWorkflow(rwf, okay);
  8825. }
  8826. for(i=0; i<n; i++)
  8827. {
  8828. assertex(okay[i]);
  8829. okay[i] = dep[i] = false;
  8830. }
  8831. Owned<IWorkflowItemArray> array(wu.getWorkflowClone());
  8832. unsigned wfid;
  8833. for(wfid = 1; array->isValid(wfid); wfid++)
  8834. testConstWorkflow(&array->queryWfid(wfid), okay, dep);
  8835. for(i=0; i<n; i++)
  8836. {
  8837. assertex(okay[i]);
  8838. okay[i] = false;
  8839. }
  8840. for(wfid = 1; array->isValid(wfid); wfid++)
  8841. testRuntimeWorkflow(&array->queryWfid(wfid), okay);
  8842. for(i=0; i<n; i++)
  8843. {
  8844. assertex(okay[i]);
  8845. okay[i] = false;
  8846. }
  8847. }
  8848. #endif
  8849. //------------------------------------------------------------------------------------------
  8850. extern WUState waitForWorkUnitToComplete(const char * wuid, int timeout, bool returnOnWaitState)
  8851. {
  8852. return factory->waitForWorkUnit(wuid, (unsigned) timeout, false, returnOnWaitState);
  8853. }
  8854. extern WORKUNIT_API WUState secWaitForWorkUnitToComplete(const char * wuid, ISecManager &secmgr, ISecUser &secuser, int timeout, bool returnOnWaitState)
  8855. {
  8856. if (checkWuSecAccess(wuid, &secmgr, &secuser, SecAccess_Read, "Wait for Complete", false, true))
  8857. return waitForWorkUnitToComplete(wuid, timeout, returnOnWaitState);
  8858. return WUStateUnknown;
  8859. }
  8860. extern bool waitForWorkUnitToCompile(const char * wuid, int timeout)
  8861. {
  8862. switch(factory->waitForWorkUnit(wuid, (unsigned) timeout, true, true))
  8863. {
  8864. case WUStateCompiled:
  8865. case WUStateCompleted:
  8866. case WUStateWait:
  8867. case WUStateUploadingFiles:
  8868. return true;
  8869. default:
  8870. return false;
  8871. }
  8872. }
  8873. extern WORKUNIT_API bool secWaitForWorkUnitToCompile(const char * wuid, ISecManager &secmgr, ISecUser &secuser, int timeout)
  8874. {
  8875. if (checkWuSecAccess(wuid, &secmgr, &secuser, SecAccess_Read, "Wait for Compile", false, true))
  8876. return waitForWorkUnitToCompile(wuid, timeout);
  8877. return false;
  8878. }
  8879. extern WORKUNIT_API bool secDebugWorkunit(const char * wuid, ISecManager &secmgr, ISecUser &secuser, const char *command, StringBuffer &response)
  8880. {
  8881. if (strnicmp(command, "<debug:", 7) == 0 && checkWuSecAccess(wuid, &secmgr, &secuser, SecAccess_Read, "Debug", false, true))
  8882. {
  8883. Owned<IConstWorkUnit> wu = factory->openWorkUnit(wuid, &secmgr, &secuser);
  8884. SCMStringBuffer ip;
  8885. unsigned port;
  8886. port = wu->getDebugAgentListenerPort();
  8887. wu->getDebugAgentListenerIP(ip);
  8888. SocketEndpoint debugEP(ip.str(), port);
  8889. Owned<ISocket> socket = ISocket::connect_timeout(debugEP, 1000);
  8890. unsigned len = (size32_t)strlen(command);
  8891. unsigned revlen = len;
  8892. _WINREV(revlen);
  8893. socket->write(&revlen, sizeof(revlen));
  8894. socket->write(command, len);
  8895. for (;;)
  8896. {
  8897. socket->read(&len, sizeof(len));
  8898. _WINREV(len);
  8899. if (len == 0)
  8900. break;
  8901. if (len & 0x80000000)
  8902. {
  8903. throwUnexpected();
  8904. }
  8905. char * mem = (char*) response.reserve(len);
  8906. socket->read(mem, len);
  8907. }
  8908. return true;
  8909. }
  8910. return false;
  8911. }
  8912. void getSimpleResultType(IWUResult *r, Owned<ITypeInfo> &type)
  8913. {
  8914. TypeInfoArray types;
  8915. StringAttrArray names;
  8916. r->getSchema(types, names, NULL);
  8917. if (types.ordinality()==1)
  8918. type.set(&types.item(0));
  8919. }
  8920. bool isSuppliedParamScalar(IWUResult *r, IPropertyTree &curVal, Owned<ITypeInfo> &type)
  8921. {
  8922. if (!r->isResultScalar())
  8923. return false;
  8924. if (!curVal.hasChildren())
  8925. return true;
  8926. getSimpleResultType(r, type);
  8927. return type && type->isScalar();
  8928. }
  8929. void updateSuppliedXmlParams(IWorkUnit * w)
  8930. {
  8931. Owned<const IPropertyTree> params = w->getXmlParams();
  8932. if (!params)
  8933. return;
  8934. Owned<IPropertyTreeIterator> elems = params->getElements("*");
  8935. ForEach(*elems)
  8936. {
  8937. IPropertyTree & curVal = elems->query();
  8938. const char *name = curVal.queryName();
  8939. Owned<IWUResult> r = updateWorkUnitResult(w, name, -1);
  8940. if (r)
  8941. {
  8942. Owned<ITypeInfo> type;
  8943. StringBuffer s;
  8944. if (isSuppliedParamScalar(r, curVal, type))
  8945. {
  8946. curVal.getProp(".", s);
  8947. r->setResultXML(s);
  8948. r->setResultStatus(ResultStatusSupplied);
  8949. }
  8950. else
  8951. {
  8952. toXML(&curVal, s);
  8953. if (!type)
  8954. getSimpleResultType(r, type);
  8955. bool isSet = (type && type->getTypeCode()==type_set);
  8956. r->setResultRaw(s.length(), s.str(), isSet ? ResultFormatXmlSet : ResultFormatXml);
  8957. }
  8958. }
  8959. else
  8960. DBGLOG("WARNING: no matching variable in workunit for input parameter %s", name);
  8961. }
  8962. }
  8963. IWUResult * updateWorkUnitResult(IWorkUnit * w, const char *name, unsigned sequence)
  8964. {
  8965. switch ((int)sequence)
  8966. {
  8967. case ResultSequenceStored:
  8968. return w->updateVariableByName(name);
  8969. case ResultSequencePersist:
  8970. return w->updateGlobalByName(name);
  8971. case ResultSequenceInternal:
  8972. case ResultSequenceOnce:
  8973. return w->updateTemporaryByName(name);
  8974. default:
  8975. return w->updateResultBySequence(sequence);
  8976. }
  8977. }
  8978. IConstWUResult * getWorkUnitResult(IConstWorkUnit * w, const char *name, unsigned sequence)
  8979. {
  8980. switch ((int)sequence)
  8981. {
  8982. case ResultSequenceStored:
  8983. return w->getVariableByName(name);
  8984. case ResultSequencePersist:
  8985. return w->getGlobalByName(name);
  8986. case ResultSequenceInternal:
  8987. case ResultSequenceOnce:
  8988. return w->getTemporaryByName(name);
  8989. default:
  8990. if (name && name[0])
  8991. return w->getResultByName(name);//name takes precedence over sequence
  8992. else
  8993. return w->getResultBySequence(sequence);
  8994. }
  8995. }
  8996. extern WORKUNIT_API bool getWorkUnitCreateTime(const char *wuid,CDateTime &time)
  8997. {
  8998. if (wuid) {
  8999. char prefchar;
  9000. unsigned year,month,day,hour,min,sec;
  9001. if (sscanf(wuid, "%c%4u%2u%2u-%2u%2u%2u", &prefchar, &year, &month, &day, &hour, &min, &sec)==7) {
  9002. time.set(year, month, day, hour, min, sec, 0, true);
  9003. // time.setDate(year, month, day);
  9004. // time.setTime(hour, min, sec, 0, true); // for some reason time is local
  9005. return true;
  9006. }
  9007. }
  9008. return false;
  9009. }
  9010. extern WORKUNIT_API IStringVal& createToken(const char *wuid, const char *user, const char *password, IStringVal &str)
  9011. {
  9012. StringBuffer wu, token("X");
  9013. wu.append(wuid).append(';').append(user).append(';').append(password);
  9014. encrypt(token,wu.str());
  9015. str.set(token.str());
  9016. return str;
  9017. }
  9018. // This will be replaced by something more secure!
  9019. extern WORKUNIT_API void extractToken(const char *token, const char *wuid, IStringVal &user, IStringVal &password)
  9020. {
  9021. if (token && *token)
  9022. {
  9023. StringBuffer wu;
  9024. decrypt(wu, token+1);
  9025. const char *finger = strchr(wu.str(),';');
  9026. if (finger && strnicmp(wuid, wu.str(), finger-wu.str())==0)
  9027. {
  9028. const char *finger1 = strchr(++finger,';');
  9029. if(finger1)
  9030. {
  9031. user.setLen(finger, (size32_t)(finger1-finger));
  9032. finger1++;
  9033. password.setLen(finger1, (size32_t)(wu.str() + wu.length() - finger1));
  9034. return;
  9035. }
  9036. }
  9037. throw MakeStringException(WUERR_InvalidSecurityToken, "Invalid call to extractToken");
  9038. }
  9039. }
  9040. extern WORKUNIT_API WUState getWorkUnitState(const char* state)
  9041. {
  9042. return (WUState) getEnum(state, states);
  9043. }
  9044. const LogMsgCategory MCschedconn = MCprogress(1000); // Category used to inform about schedule synchronization
  9045. class CWorkflowScheduleConnection : implements IWorkflowScheduleConnection, public CInterface
  9046. {
  9047. public:
  9048. CWorkflowScheduleConnection(char const * wuid)
  9049. {
  9050. basexpath.append("/WorkflowSchedule/").append(wuid);
  9051. flagxpath.append(basexpath.str()).append("/Active");
  9052. }
  9053. IMPLEMENT_IINTERFACE;
  9054. virtual void lock()
  9055. {
  9056. LOG(MCschedconn, "Locking base schedule connection");
  9057. baseconn.setown(querySDS().connect(basexpath.str(), myProcessSession(), RTM_CREATE_QUERY | RTM_LOCK_WRITE, INFINITE));
  9058. if(!baseconn)
  9059. throw MakeStringException(WUERR_ScheduleLockFailed, "Could not get base workflow schedule lock");
  9060. }
  9061. virtual void unlock()
  9062. {
  9063. LOG(MCschedconn, "Unlocking base schedule connection");
  9064. baseconn.clear();
  9065. }
  9066. virtual void setActive()
  9067. {
  9068. LOG(MCschedconn, "Setting active flag in schedule connection");
  9069. flagconn.setown(querySDS().connect(flagxpath.str(), myProcessSession(), RTM_CREATE | RTM_LOCK_WRITE | RTM_DELETE_ON_DISCONNECT, INFINITE));
  9070. if(!flagconn)
  9071. throw MakeStringException(WUERR_ScheduleLockFailed, "Could not get active workflow schedule lock");
  9072. }
  9073. virtual void resetActive()
  9074. {
  9075. LOG(MCschedconn, "Resetting active flag in schedule connection");
  9076. flagconn.clear();
  9077. }
  9078. virtual bool queryActive()
  9079. {
  9080. return baseconn->queryRoot()->hasProp("Active");
  9081. }
  9082. virtual bool pull(IWorkflowItemArray * workflow)
  9083. {
  9084. assertex(baseconn);
  9085. Owned<IPropertyTree> root = baseconn->getRoot();
  9086. Owned<IPropertyTree> eventQueue = root->getPropTree("EventQueue");
  9087. if(!eventQueue) return false;
  9088. if(!eventQueue->hasProp("Item")) return false;
  9089. {
  9090. Owned<IPropertyTreeIterator> eventItems = eventQueue->getElements("Item");
  9091. Owned<IPropertyTree> eventItem;
  9092. Owned<IRuntimeWorkflowItemIterator> wfItems = workflow->getSequenceIterator();
  9093. Owned<IRuntimeWorkflowItem> wfItem;
  9094. for(eventItems->first(); eventItems->isValid(); eventItems->next())
  9095. {
  9096. eventItem.setown(&eventItems->get());
  9097. const char * eventName = eventItem->queryProp("@name");
  9098. const char * eventText = eventItem->queryProp("@text");
  9099. for(wfItems->first(); wfItems->isValid(); wfItems->next())
  9100. {
  9101. wfItem.setown(wfItems->get());
  9102. if(wfItem->queryState() != WFStateWait)
  9103. continue;
  9104. Owned<IWorkflowEvent> targetEvent = wfItem->getScheduleEvent();
  9105. if(!targetEvent || !targetEvent->matches(eventName, eventText))
  9106. continue;
  9107. wfItem->setEvent(eventName, eventText);
  9108. wfItem->setState(WFStateReqd);
  9109. resetDependentsState(workflow, *wfItem);
  9110. }
  9111. }
  9112. }
  9113. bool more;
  9114. do
  9115. more = eventQueue->removeProp("Item");
  9116. while(more);
  9117. return true;
  9118. }
  9119. virtual void push(char const * name, char const * text)
  9120. {
  9121. assertex(baseconn);
  9122. Owned<IPropertyTree> root = baseconn->getRoot();
  9123. IPropertyTree *eventQueue = ensurePTree(root, "EventQueue");
  9124. IPropertyTree *eventItem = eventQueue->addPropTree("Item");
  9125. eventItem->setProp("@name", name);
  9126. eventItem->setProp("@text", text);
  9127. }
  9128. virtual void remove()
  9129. {
  9130. if (baseconn)
  9131. {
  9132. baseconn->close(true);
  9133. baseconn.clear();
  9134. }
  9135. }
  9136. private:
  9137. void resetItemStateAndDependents(IWorkflowItemArray * workflow, unsigned wfid) const
  9138. {
  9139. if (wfid)
  9140. resetItemStateAndDependents(workflow, workflow->queryWfid(wfid));
  9141. }
  9142. void resetItemStateAndDependents(IWorkflowItemArray * workflow, IRuntimeWorkflowItem & item) const
  9143. {
  9144. switch(item.queryState())
  9145. {
  9146. case WFStateDone:
  9147. case WFStateFail:
  9148. {
  9149. item.setState(WFStateNull);
  9150. resetItemStateAndDependents(workflow, item.queryPersistWfid());
  9151. resetDependentsState(workflow, item);
  9152. break;
  9153. }
  9154. }
  9155. }
  9156. void resetDependentsState(IWorkflowItemArray * workflow, IRuntimeWorkflowItem & item) const
  9157. {
  9158. Owned<IWorkflowDependencyIterator> iter(item.getDependencies());
  9159. for(iter->first(); iter->isValid(); iter->next())
  9160. {
  9161. IRuntimeWorkflowItem & dep = workflow->queryWfid(iter->query());
  9162. resetItemStateAndDependents(workflow, dep);
  9163. }
  9164. }
  9165. private:
  9166. StringBuffer basexpath;
  9167. StringBuffer flagxpath;
  9168. Owned<IRemoteConnection> baseconn;
  9169. Owned<IRemoteConnection> flagconn;
  9170. };
  9171. extern WORKUNIT_API IWorkflowScheduleConnection * getWorkflowScheduleConnection(char const * wuid)
  9172. {
  9173. return new CWorkflowScheduleConnection(wuid);
  9174. }
  9175. extern WORKUNIT_API IExtendedWUInterface * queryExtendedWU(IConstWorkUnit * wu)
  9176. {
  9177. return QUERYINTERFACE(wu, IExtendedWUInterface);
  9178. }
  9179. extern WORKUNIT_API const IExtendedWUInterface * queryExtendedWU(const IConstWorkUnit * wu)
  9180. {
  9181. return QUERYINTERFACE(wu, const IExtendedWUInterface);
  9182. }
  9183. extern WORKUNIT_API void addExceptionToWorkunit(IWorkUnit * wu, ErrorSeverity severity, const char * source, unsigned code, const char * text, const char * filename, unsigned lineno, unsigned column, unsigned activity)
  9184. {
  9185. Owned<IWUException> we = wu->createException();
  9186. we->setSeverity(severity);
  9187. we->setExceptionMessage(text);
  9188. if (source)
  9189. we->setExceptionSource(source);
  9190. if (code)
  9191. we->setExceptionCode(code);
  9192. if (filename)
  9193. we->setExceptionFileName(filename);
  9194. if (lineno)
  9195. {
  9196. we->setExceptionLineNo(lineno);
  9197. if (column)
  9198. we->setExceptionColumn(lineno);
  9199. }
  9200. if (activity)
  9201. we->setActivityId(activity);
  9202. }
  9203. const char * skipLeadingXml(const char * text)
  9204. {
  9205. if (!text)
  9206. return NULL;
  9207. //skip utf8 BOM, probably excessive
  9208. if (memcmp(text, UTF8_BOM, 3) == 0)
  9209. text += 3;
  9210. for (;;)
  9211. {
  9212. if (isspace(*text))
  9213. text++;
  9214. else if (text[0] == '<' && text[1] == '?')
  9215. {
  9216. text += 2;
  9217. for (;;)
  9218. {
  9219. if (!*text) break;
  9220. if (text[0] == '?' && text[1] == '>')
  9221. {
  9222. text += 2;
  9223. break;
  9224. }
  9225. text++;
  9226. }
  9227. }
  9228. else if (text[0] == '<' && text[1] == '!' && text[2] == '-' && text[3] == '-')
  9229. {
  9230. text += 4;
  9231. for (;;)
  9232. {
  9233. if (!*text) break;
  9234. if (text[0] == '-' && text[1] == '-' && text[2] == '>')
  9235. {
  9236. text += 3;
  9237. break;
  9238. }
  9239. text++;
  9240. }
  9241. }
  9242. else
  9243. break;
  9244. }
  9245. return text;
  9246. }
  9247. extern WORKUNIT_API bool isArchiveQuery(const char * text)
  9248. {
  9249. text = skipLeadingXml(text);
  9250. if (!text)
  9251. return false;
  9252. const char * archivePrefix = "<Archive";
  9253. return memicmp(text, archivePrefix, strlen(archivePrefix)) == 0;
  9254. }
  9255. extern WORKUNIT_API bool isQueryManifest(const char * text)
  9256. {
  9257. text = skipLeadingXml(text);
  9258. if (!text)
  9259. return false;
  9260. const char * manifestPrefix = "<Manifest";
  9261. return memicmp(text, manifestPrefix, strlen(manifestPrefix)) == 0;
  9262. }
  9263. //------------------------------------------------------------------------------
  9264. // Named Alias helper function
  9265. static IPropertyTree * resolveQueryByDll(IPropertyTree * queryRegistry, const char * dll)
  9266. {
  9267. StringBuffer xpath;
  9268. xpath.append("Query[@dll=\"").append(dll).append("\"]");
  9269. return queryRegistry->getPropTree(xpath);
  9270. }
  9271. static IPropertyTree * resolveQueryByWuid(IPropertyTree * queryRegistry, const char * wuid)
  9272. {
  9273. StringBuffer xpath;
  9274. xpath.append("Query[@wuid=\"").append(wuid).append("\"]");
  9275. return queryRegistry->getPropTree(xpath);
  9276. }
  9277. static void clearAliases(IPropertyTree * queryRegistry, const char * id)
  9278. {
  9279. StringBuffer lcId(id);
  9280. lcId.toLowerCase();
  9281. StringBuffer xpath;
  9282. xpath.append("Alias[@id=\"").append(lcId).append("\"]");
  9283. Owned<IPropertyTreeIterator> iter = queryRegistry->getElements(xpath);
  9284. ForEach(*iter)
  9285. {
  9286. queryRegistry->removeProp(xpath.str());
  9287. }
  9288. }
  9289. IPropertyTree * addNamedQuery(IPropertyTree * queryRegistry, const char * name, const char * wuid, const char * dll, bool library, const char *userid, const char *snapshot)
  9290. {
  9291. StringBuffer lcName(name);
  9292. lcName.toLowerCase();
  9293. StringBuffer xpath;
  9294. xpath.append("Query[@name=\"").append(lcName.str()).append("\"]");
  9295. Owned<IPropertyTreeIterator> iter = queryRegistry->getElements(xpath);
  9296. unsigned seq = 1;
  9297. ForEach(*iter)
  9298. {
  9299. IPropertyTree &item = iter->query();
  9300. const char *thisWuid = item.queryProp("@wuid");
  9301. if (strieq(wuid, thisWuid))
  9302. return &item;
  9303. unsigned thisSeq = item.getPropInt("@seq");
  9304. if (thisSeq >= seq)
  9305. seq = thisSeq + 1;
  9306. }
  9307. StringBuffer id;
  9308. id.append(lcName).append(".").append(seq);
  9309. IPropertyTree * newEntry = createPTree("Query", ipt_caseInsensitive|ipt_lowmem);
  9310. newEntry->setProp("@name", lcName);
  9311. newEntry->setProp("@wuid", wuid);
  9312. newEntry->setProp("@dll", dll);
  9313. newEntry->setProp("@id", id);
  9314. newEntry->setPropInt("@seq", seq);
  9315. if (library)
  9316. newEntry->setPropBool("@isLibrary", true);
  9317. if (userid && *userid)
  9318. newEntry->setProp("@publishedBy", userid);
  9319. if (snapshot && *snapshot)
  9320. newEntry->setProp("@snapshot", snapshot);
  9321. return queryRegistry->addPropTree("Query", newEntry);
  9322. }
  9323. void removeNamedQuery(IPropertyTree * queryRegistry, const char * id)
  9324. {
  9325. StringBuffer lcId(id);
  9326. lcId.toLowerCase();
  9327. clearAliases(queryRegistry, lcId);
  9328. StringBuffer xpath;
  9329. xpath.append("Query[@id=\"").append(lcId).append("\"]");
  9330. queryRegistry->removeProp(xpath);
  9331. }
  9332. void removeDllFromNamedQueries(IPropertyTree * queryRegistry, const char * dll)
  9333. {
  9334. Owned<IPropertyTree> match = resolveQueryByDll(queryRegistry, dll);
  9335. if (!match)
  9336. return;
  9337. clearAliases(queryRegistry, match->queryProp("@id"));
  9338. queryRegistry->removeTree(match);
  9339. }
  9340. void removeWuidFromNamedQueries(IPropertyTree * queryRegistry, const char * wuid)
  9341. {
  9342. Owned<IPropertyTree> match = resolveQueryByWuid(queryRegistry, wuid);
  9343. if (!match)
  9344. return;
  9345. clearAliases(queryRegistry, match->queryProp("@id"));
  9346. queryRegistry->removeTree(match);
  9347. }
  9348. void removeAliasesFromNamedQuery(IPropertyTree * queryRegistry, const char * id)
  9349. {
  9350. clearAliases(queryRegistry, id);
  9351. }
  9352. void setQueryAlias(IPropertyTree * queryRegistry, const char * name, const char * value)
  9353. {
  9354. StringBuffer lcName(name);
  9355. lcName.toLowerCase();
  9356. StringBuffer xpath;
  9357. xpath.append("Alias[@name=\"").append(lcName).append("\"]");
  9358. IPropertyTree * match = queryRegistry->queryPropTree(xpath);
  9359. if (!match)
  9360. {
  9361. match = queryRegistry->addPropTree("Alias");
  9362. match->setProp("@name", lcName);
  9363. }
  9364. match->setProp("@id", value);
  9365. }
  9366. extern WORKUNIT_API IPropertyTree * getQueryById(IPropertyTree * queryRegistry, const char *queryid)
  9367. {
  9368. if (!queryRegistry || !queryid)
  9369. return NULL;
  9370. StringBuffer xpath;
  9371. xpath.append("Query[@id=\"").append(queryid).append("\"]");
  9372. return queryRegistry->getPropTree(xpath);
  9373. }
  9374. extern WORKUNIT_API IPropertyTree * getQueryById(const char *queryset, const char *queryid, bool readonly)
  9375. {
  9376. Owned<IPropertyTree> queryRegistry = getQueryRegistry(queryset, readonly);
  9377. return getQueryById(queryRegistry, queryid);
  9378. }
  9379. extern WORKUNIT_API IPropertyTree * resolveQueryAlias(IPropertyTree * queryRegistry, const char * alias)
  9380. {
  9381. if (!queryRegistry || !alias)
  9382. return NULL;
  9383. StringBuffer xpath;
  9384. unsigned cnt = 0;
  9385. StringBuffer lc(alias);
  9386. const char * search = lc.toLowerCase().str();
  9387. for (;;)
  9388. {
  9389. xpath.set("Alias[@name='").append(search).append("']/@id");
  9390. const char * queryId = queryRegistry->queryProp(xpath);
  9391. if (!queryId)
  9392. break;
  9393. //Check for too many alias indirections.
  9394. if (cnt++ > 10)
  9395. return NULL;
  9396. search = lc.set(queryId).toLowerCase().str();
  9397. }
  9398. return getQueryById(queryRegistry, search);
  9399. }
  9400. extern WORKUNIT_API IPropertyTree * resolveQueryAlias(const char *queryset, const char *alias, bool readonly)
  9401. {
  9402. Owned<IPropertyTree> queryRegistry = getQueryRegistry(queryset, readonly);
  9403. return resolveQueryAlias(queryRegistry, alias);
  9404. }
  9405. void setQuerySuspendedState(IPropertyTree * queryRegistry, const char *id, bool suspend, const char *userid)
  9406. {
  9407. StringBuffer lcId(id);
  9408. lcId.toLowerCase();
  9409. StringBuffer xpath;
  9410. xpath.append("Query[@id=\"").append(lcId).append("\"]");
  9411. IPropertyTree *tree = queryRegistry->queryPropTree(xpath);
  9412. if (tree)
  9413. {
  9414. if (tree->getPropBool("@suspended", false) == suspend)
  9415. return;
  9416. if (suspend)
  9417. {
  9418. tree->addPropBool("@suspended", true);
  9419. if (userid && *userid)
  9420. tree->addProp("@suspendedBy", userid);
  9421. }
  9422. else
  9423. {
  9424. tree->removeProp("@suspended");
  9425. tree->removeProp("@suspendedBy");
  9426. }
  9427. }
  9428. else
  9429. throw MakeStringException((suspend)? QUERRREG_SUSPEND : QUERRREG_UNSUSPEND, "Modifying query suspended state failed. Could not find query %s", id);
  9430. }
  9431. void setQueryCommentForNamedQuery(IPropertyTree * queryRegistry, const char *id, const char *queryComment)
  9432. {
  9433. if (queryComment)
  9434. {
  9435. StringBuffer lcId(id);
  9436. lcId.toLowerCase();
  9437. StringBuffer xpath;
  9438. xpath.append("Query[@id=\"").append(lcId).append("\"]");
  9439. IPropertyTree *tree = queryRegistry->queryPropTree(xpath);
  9440. if (tree)
  9441. tree->setProp("@queryComment", queryComment);
  9442. else
  9443. throw MakeStringException(QUERRREG_COMMENT, "Could not find query %s", id);
  9444. }
  9445. }
  9446. extern WORKUNIT_API IPropertyTree * getQueryRegistryRoot()
  9447. {
  9448. Owned<IRemoteConnection> conn = querySDS().connect("/QuerySets", myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT);
  9449. if (conn)
  9450. return conn->getRoot();
  9451. else
  9452. return NULL;
  9453. }
  9454. extern WORKUNIT_API void checkAddLibrariesToQueryEntry(IPropertyTree *queryTree, IConstWULibraryIterator *libraries)
  9455. {
  9456. if (!queryTree || !libraries)
  9457. return;
  9458. if (queryTree->hasProp("@libCount")) //already added
  9459. return;
  9460. unsigned libCount=0;
  9461. ForEach(*libraries)
  9462. {
  9463. IConstWULibrary &library = libraries->query();
  9464. SCMStringBuffer libname;
  9465. if (!library.getName(libname).length())
  9466. continue;
  9467. queryTree->addProp("Library", libname.str());
  9468. libCount++;
  9469. }
  9470. queryTree->setPropInt("@libCount", libCount);
  9471. }
  9472. extern WORKUNIT_API void checkAddLibrariesToQueryEntry(IPropertyTree *queryTree, IConstWorkUnit *cw)
  9473. {
  9474. Owned<IConstWULibraryIterator> libraries = &cw->getLibraries();
  9475. checkAddLibrariesToQueryEntry(queryTree, libraries);
  9476. }
  9477. extern WORKUNIT_API IPropertyTree * getQueryRegistry(const char * wsEclId, bool readonly)
  9478. {
  9479. //Only lock the branch for the target we're interested in.
  9480. StringBuffer xpath;
  9481. xpath.append("/QuerySets/QuerySet[@id=\"").append(wsEclId).append("\"]");
  9482. Owned<IRemoteConnection> conn = querySDS().connect(xpath.str(), myProcessSession(), readonly ? RTM_LOCK_READ : RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
  9483. if (conn)
  9484. return conn->getRoot();
  9485. if (readonly)
  9486. return NULL;
  9487. //Lock the QuerySets in case another thread/client wants to check/add the same QuerySet.
  9488. Owned<IRemoteConnection> globalLock = querySDS().connect("/QuerySets/", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
  9489. //Re-check if the QuerySet has been added between checking the 1st time and gaining the globalLock.
  9490. conn.setown(querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT));
  9491. if (conn)
  9492. return conn->getRoot();
  9493. conn.setown(querySDS().connect("/QuerySets/QuerySet", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_ADD, SDS_LOCK_TIMEOUT));
  9494. if (!conn)
  9495. throwUnexpected();
  9496. IPropertyTree * root = conn->queryRoot();
  9497. root->setProp("@id",wsEclId);
  9498. conn->commit();
  9499. return LINK(root);
  9500. }
  9501. IPropertyTree * addNamedPackageSet(IPropertyTree * packageRegistry, const char * name, IPropertyTree *packageInfo, bool overWrite)
  9502. {
  9503. StringBuffer xpath;
  9504. StringBuffer lcName(name);
  9505. lcName.toLowerCase();
  9506. // see if "name" already exists
  9507. xpath.append("Package[@id='").append(name).append("']");
  9508. IPropertyTree *pkgTree = packageRegistry->queryPropTree(xpath.str());
  9509. if (pkgTree)
  9510. {
  9511. if (overWrite)
  9512. packageRegistry->removeTree(pkgTree);
  9513. else
  9514. throw MakeStringException(WUERR_PackageAlreadyExists, "Package name %s already exists, either delete it or specify overwrite",name);
  9515. }
  9516. IPropertyTree *tree = packageRegistry->addPropTree("Package", packageInfo);
  9517. tree->setProp("@id", lcName);
  9518. return tree;
  9519. }
  9520. void removeNamedPackage(IPropertyTree * packageRegistry, const char * id)
  9521. {
  9522. StringBuffer lcId(id);
  9523. lcId.toLowerCase();
  9524. StringBuffer xpath;
  9525. xpath.append("Package[@id=\"").append(lcId).append("\"]");
  9526. packageRegistry->removeProp(xpath);
  9527. }
  9528. extern WORKUNIT_API IPropertyTree * getPackageSetRegistry(const char * wsEclId, bool readonly)
  9529. {
  9530. //Only lock the branch for the target we're interested in.
  9531. StringBuffer xpath;
  9532. xpath.append("/PackageSets/PackageSet[@id=\"").append(wsEclId).append("\"]");
  9533. Owned<IRemoteConnection> conn = querySDS().connect(xpath.str(), myProcessSession(), readonly ? RTM_LOCK_READ : RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
  9534. if (conn)
  9535. return conn->getRoot();
  9536. if (readonly)
  9537. return NULL;
  9538. //Lock the PackageSets in case another thread/client wants to check/add the same PackageSet.
  9539. Owned<IRemoteConnection> globalLock = querySDS().connect("/PackageSets/", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
  9540. //Re-check if the PackageSet has been added between checking the 1st time and gaining the globalLock.
  9541. conn.setown(querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT));
  9542. if (conn)
  9543. return conn->getRoot();
  9544. conn.setown(querySDS().connect("/PackageSets/PackageSet", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_ADD, SDS_LOCK_TIMEOUT));
  9545. if (!conn)
  9546. throwUnexpected();
  9547. IPropertyTree* root = conn->queryRoot();
  9548. root->setProp("@id",wsEclId);
  9549. conn->commit();
  9550. return LINK(root);
  9551. }
  9552. void addQueryToQuerySet(IWorkUnit *workunit, IPropertyTree *queryRegistry, const char *queryName, WUQueryActivationOptions activateOption, StringBuffer &newQueryId, const char *userid)
  9553. {
  9554. StringBuffer cleanQueryName;
  9555. appendUtf8XmlName(cleanQueryName, strlen(queryName), queryName);
  9556. SCMStringBuffer dllName;
  9557. Owned<IConstWUQuery> q = workunit->getQuery();
  9558. q->getQueryDllName(dllName);
  9559. if (!dllName.length())
  9560. throw MakeStringException(WUERR_InvalidDll, "Cannot deploy query - no associated dll.");
  9561. StringBuffer currentTargetClusterType;
  9562. queryRegistry->getProp("@targetclustertype", currentTargetClusterType);
  9563. SCMStringBuffer targetClusterType;
  9564. workunit->getDebugValue("targetclustertype", targetClusterType);
  9565. SCMStringBuffer snapshot;
  9566. workunit->getSnapshot(snapshot);
  9567. if (currentTargetClusterType.length() < 1)
  9568. {
  9569. queryRegistry->setProp("@targetclustertype", targetClusterType.str());
  9570. }
  9571. else
  9572. {
  9573. if (strcmp(currentTargetClusterType.str(), "roxie") == 0 && strcmp(currentTargetClusterType.str(), targetClusterType.str())!=0)
  9574. {
  9575. throw MakeStringException(WUERR_MismatchClusterType, "TargetClusterTypes of workunit and queryset do not match.");
  9576. }
  9577. }
  9578. IPropertyTree *newEntry = addNamedQuery(queryRegistry, cleanQueryName, workunit->queryWuid(), dllName.str(), isLibrary(workunit), userid, snapshot.str());
  9579. Owned<IConstWULibraryIterator> libraries = &workunit->getLibraries();
  9580. checkAddLibrariesToQueryEntry(newEntry, libraries);
  9581. newQueryId.append(newEntry->queryProp("@id"));
  9582. workunit->setIsQueryService(true); //will check querysets before delete
  9583. workunit->commit();
  9584. activateQuery(queryRegistry, activateOption, queryName, newQueryId, userid);
  9585. }
  9586. void activateQuery(IPropertyTree *queryRegistry, WUQueryActivationOptions activateOption, const char *queryName, const char *queryId, const char *userid)
  9587. {
  9588. StringBuffer cleanQueryName;
  9589. appendUtf8XmlName(cleanQueryName, strlen(queryName), queryName);
  9590. if (activateOption == ACTIVATE_SUSPEND_PREVIOUS|| activateOption == ACTIVATE_DELETE_PREVIOUS)
  9591. {
  9592. Owned<IPropertyTree> prevQuery = resolveQueryAlias(queryRegistry, cleanQueryName);
  9593. setQueryAlias(queryRegistry, cleanQueryName, queryId);
  9594. if (prevQuery && !streq(queryId, prevQuery->queryProp("@id")))
  9595. {
  9596. if (activateOption == ACTIVATE_SUSPEND_PREVIOUS)
  9597. setQuerySuspendedState(queryRegistry, prevQuery->queryProp("@id"), true, userid);
  9598. else
  9599. removeNamedQuery(queryRegistry, prevQuery->queryProp("@id"));
  9600. }
  9601. }
  9602. else if (activateOption == MAKE_ACTIVATE || activateOption == MAKE_ACTIVATE_LOAD_DATA_ONLY)
  9603. setQueryAlias(queryRegistry, cleanQueryName, queryId);
  9604. }
  9605. void addQueryToQuerySet(IWorkUnit *workunit, const char *querySetName, const char *queryName, WUQueryActivationOptions activateOption, StringBuffer &newQueryId, const char *userid)
  9606. {
  9607. Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, false);
  9608. addQueryToQuerySet(workunit, queryRegistry, queryName, activateOption, newQueryId, userid);
  9609. }
  9610. bool removeQuerySetAlias(const char *querySetName, const char *alias)
  9611. {
  9612. Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, true);
  9613. StringBuffer xpath;
  9614. xpath.appendf("Alias[@name='%s']", alias);
  9615. IPropertyTree *t = queryRegistry->queryPropTree(xpath);
  9616. return queryRegistry->removeTree(t);
  9617. }
  9618. void addQuerySetAlias(const char *querySetName, const char *alias, const char *id)
  9619. {
  9620. Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, false);
  9621. setQueryAlias(queryRegistry, alias, id);
  9622. }
  9623. void setSuspendQuerySetQuery(const char *querySetName, const char *id, bool suspend, const char *userid)
  9624. {
  9625. Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, true);
  9626. setQuerySuspendedState(queryRegistry, id, suspend, userid);
  9627. }
  9628. void deleteQuerySetQuery(const char *querySetName, const char *id)
  9629. {
  9630. Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, true);
  9631. removeNamedQuery(queryRegistry, id);
  9632. }
  9633. void removeQuerySetAliasesFromNamedQuery(const char *querySetName, const char * id)
  9634. {
  9635. Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, true);
  9636. clearAliases(queryRegistry, id);
  9637. }
  9638. void setQueryCommentForNamedQuery(const char *querySetName, const char *id, const char *queryComment)
  9639. {
  9640. Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, true);
  9641. setQueryCommentForNamedQuery(queryRegistry, id, queryComment);
  9642. }
  9643. const char *queryIdFromQuerySetWuid(IPropertyTree *queryRegistry, const char *wuid, const char *queryName, IStringVal &id)
  9644. {
  9645. if (!queryRegistry)
  9646. return NULL;
  9647. StringBuffer xpath;
  9648. xpath.appendf("Query[@wuid='%s']", wuid);
  9649. if (queryName && *queryName)
  9650. xpath.appendf("[@name='%s']", queryName);
  9651. IPropertyTree *q = queryRegistry->queryPropTree(xpath.str());
  9652. if (q)
  9653. {
  9654. id.set(q->queryProp("@id"));
  9655. }
  9656. return id.str();
  9657. }
  9658. const char *queryIdFromQuerySetWuid(const char *querySetName, const char *wuid, const char *queryName, IStringVal &id)
  9659. {
  9660. Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, true);
  9661. return queryIdFromQuerySetWuid(queryRegistry, wuid, queryName, id);
  9662. }
  9663. extern WORKUNIT_API void gatherLibraryNames(StringArray &names, StringArray &unresolved, IWorkUnitFactory &workunitFactory, IConstWorkUnit &cw, IPropertyTree *queryset)
  9664. {
  9665. Owned<IConstWULibraryIterator> wulibraries = &cw.getLibraries();
  9666. ForEach(*wulibraries)
  9667. {
  9668. SCMStringBuffer libname;
  9669. IConstWULibrary &wulibrary = wulibraries->query();
  9670. wulibrary.getName(libname);
  9671. if (names.contains(libname.str()) || unresolved.contains(libname.str()))
  9672. continue;
  9673. Owned<IPropertyTree> query = resolveQueryAlias(queryset, libname.str());
  9674. if (query && query->getPropBool("@isLibrary"))
  9675. {
  9676. const char *wuid = query->queryProp("@wuid");
  9677. Owned<IConstWorkUnit> libcw = workunitFactory.openWorkUnit(wuid);
  9678. if (libcw)
  9679. {
  9680. names.appendUniq(libname.str());
  9681. gatherLibraryNames(names, unresolved, workunitFactory, *libcw, queryset);
  9682. continue;
  9683. }
  9684. }
  9685. unresolved.appendUniq(libname.str());
  9686. }
  9687. }
  9688. bool looksLikeAWuid(const char * wuid, const char firstChar)
  9689. {
  9690. if (!wuid)
  9691. return false;
  9692. if (wuid[0] != firstChar)
  9693. return false;
  9694. if (!isdigit(wuid[1]) || !isdigit(wuid[2]) || !isdigit(wuid[3]) || !isdigit(wuid[4]))
  9695. return false;
  9696. if (!isdigit(wuid[5]) || !isdigit(wuid[6]) || !isdigit(wuid[7]) || !isdigit(wuid[8]))
  9697. return false;
  9698. return (wuid[9]=='-');
  9699. }
  9700. IPropertyTree * resolveDefinitionInArchive(IPropertyTree * archive, const char * path)
  9701. {
  9702. IPropertyTree * module = archive;
  9703. const char * dot = strrchr(path, '.');
  9704. StringBuffer xpath;
  9705. if (dot)
  9706. {
  9707. xpath.clear().append("Module[@key='").appendLower(dot-path, path).append("']");
  9708. module = archive->queryPropTree(xpath);
  9709. path = dot+1;
  9710. }
  9711. else
  9712. module = archive->queryPropTree("Module[@key='']");
  9713. if (!module)
  9714. return NULL;
  9715. xpath.clear().append("Attribute[@key='").appendLower(strlen(path), path).append("']");
  9716. return module->queryPropTree(xpath);
  9717. }
  9718. extern WORKUNIT_API void associateLocalFile(IWUQuery * query, WUFileType type, const char * name, const char * description, unsigned crc, unsigned minActivity, unsigned maxActivity)
  9719. {
  9720. StringBuffer hostname;
  9721. queryHostIP().getIpText(hostname);
  9722. StringBuffer fullPathname;
  9723. makeAbsolutePath(name, fullPathname);
  9724. query->addAssociatedFile(type, fullPathname, hostname, description, crc, minActivity, maxActivity);
  9725. }
  9726. extern WORKUNIT_API void descheduleWorkunit(char const * wuid)
  9727. {
  9728. Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
  9729. Owned<IWorkUnit> workunit = factory->updateWorkUnit(wuid);
  9730. if(workunit)
  9731. workunit->deschedule();
  9732. else
  9733. doDescheduleWorkkunit(wuid);
  9734. }
  9735. extern WORKUNIT_API void updateWorkunitTimeStat(IWorkUnit * wu, StatisticScopeType scopeType, const char * scope, StatisticKind kind, const char * description, unsigned __int64 value)
  9736. {
  9737. wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), scopeType, scope, kind, description, value, 1, 0, StatsMergeReplace);
  9738. }
  9739. extern WORKUNIT_API void updateWorkunitTimings(IWorkUnit * wu, ITimeReporter *timer)
  9740. {
  9741. StringBuffer scope;
  9742. for (unsigned i = 0; i < timer->numSections(); i++)
  9743. {
  9744. StatisticScopeType scopeType= timer->getScopeType(i);
  9745. timer->getScope(i, scope.clear());
  9746. StatisticKind kind = timer->getTimerType(i);
  9747. wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), scopeType, scope, kind, NULL, timer->getTime(i), timer->getCount(i), timer->getMaxTime(i), StatsMergeReplace);
  9748. }
  9749. }
  9750. extern WORKUNIT_API void getWorkunitTotalTime(IConstWorkUnit* workunit, const char* creator, unsigned __int64 & totalTimeNs, unsigned __int64 & totalThisTimeNs)
  9751. {
  9752. StatisticsFilter summaryTimeFilter(SCTsummary, creator, SSTglobal, GLOBAL_SCOPE, SMeasureTimeNs, StTimeElapsed);
  9753. Owned<IConstWUStatistic> totalThorTime = getStatistic(workunit, summaryTimeFilter);
  9754. Owned<IConstWUStatistic> totalThisThorTime = workunit->getStatistic(queryStatisticsComponentName(), GLOBAL_SCOPE, StTimeElapsed);
  9755. if (totalThorTime)
  9756. totalTimeNs = totalThorTime->getValue();
  9757. else
  9758. totalTimeNs = 0;
  9759. if (totalThisThorTime)
  9760. totalThisTimeNs = totalThisThorTime->getValue();
  9761. else
  9762. totalThisTimeNs = 0;
  9763. }
  9764. extern WORKUNIT_API void addTimeStamp(IWorkUnit * wu, StatisticScopeType scopeType, const char * scope, StatisticKind kind)
  9765. {
  9766. wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), scopeType, scope, kind, NULL, getTimeStampNowValue(), 1, 0, StatsMergeAppend);
  9767. }
  9768. IConstWUStatistic * getStatistic(IConstWorkUnit * wu, const IStatisticsFilter & filter)
  9769. {
  9770. Owned<IConstWUStatisticIterator> iter = &wu->getStatistics(&filter);
  9771. if (iter->first())
  9772. return &OLINK(iter->query());
  9773. return NULL;
  9774. }
  9775. class GlobalStatisticGatherer : public CInterfaceOf<IStatisticGatherer>
  9776. {
  9777. public:
  9778. GlobalStatisticGatherer(IWorkUnit * _wu) : wu(_wu) {}
  9779. virtual void beginScope(const StatsScopeId & id)
  9780. {
  9781. prevLenStack.append(scope.length());
  9782. if (scope.length())
  9783. scope.append(":");
  9784. id.getScopeText(scope);
  9785. scopeTypeStack.append(id.queryScopeType());
  9786. }
  9787. virtual void beginSubGraphScope(unsigned id)
  9788. {
  9789. StatsScopeId scopeId(SSTsubgraph, id);
  9790. beginScope(scopeId);
  9791. }
  9792. virtual void beginActivityScope(unsigned id)
  9793. {
  9794. StatsScopeId scopeId(SSTactivity, id);
  9795. beginScope(scopeId);
  9796. }
  9797. virtual void beginEdgeScope(unsigned id, unsigned oid)
  9798. {
  9799. StatsScopeId scopeId(SSTedge, id, oid);
  9800. beginScope(scopeId);
  9801. }
  9802. virtual void endScope()
  9803. {
  9804. scope.setLength(prevLenStack.popGet());
  9805. scopeTypeStack.pop();
  9806. }
  9807. virtual void addStatistic(StatisticKind kind, unsigned __int64 value)
  9808. {
  9809. StatisticScopeType scopeType = scopeTypeStack.ordinality() ? (StatisticScopeType)scopeTypeStack.tos() : SSTglobal;
  9810. wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), scopeType, scope, kind, NULL, value, 1, 0, StatsMergeAppend);
  9811. }
  9812. virtual void updateStatistic(StatisticKind kind, unsigned __int64 value, StatsMergeAction mergeAction)
  9813. {
  9814. StatisticScopeType scopeType = scopeTypeStack.ordinality() ? (StatisticScopeType)scopeTypeStack.tos() : SSTglobal;
  9815. wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), scopeType, scope, kind, NULL, value, 1, 0, mergeAction);
  9816. }
  9817. virtual IStatisticCollection * getResult()
  9818. {
  9819. return NULL;
  9820. }
  9821. protected:
  9822. Linked<IWorkUnit> wu;
  9823. StringBuffer scope;
  9824. UnsignedArray prevLenStack;
  9825. UnsignedArray scopeTypeStack;
  9826. };
  9827. IStatisticGatherer * createGlobalStatisticGatherer(IWorkUnit * wu)
  9828. {
  9829. return new GlobalStatisticGatherer(wu);
  9830. }
  9831. extern WORKUNIT_API IPropertyTree * getWUGraphProgress(const char * wuid, bool readonly)
  9832. {
  9833. if (!wuid || !*wuid)
  9834. return NULL;
  9835. VStringBuffer path("/GraphProgress/%s", wuid);
  9836. Owned<IRemoteConnection> conn = querySDS().connect(path.str(),myProcessSession(),readonly ? RTM_LOCK_READ : RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
  9837. if (conn)
  9838. return conn->getRoot();
  9839. else
  9840. return NULL;
  9841. }