workunit.cpp 352 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305730673077308730973107311731273137314731573167317731873197320732173227323732473257326732773287329733073317332733373347335733673377338733973407341734273437344734573467347734873497350735173527353735473557356735773587359736073617362736373647365736673677368736973707371737273737374737573767377737873797380738173827383738473857386738773887389739073917392739373947395739673977398739974007401740274037404740574067407740874097410741174127413741474157416741774187419742074217422742374247425742674277428742974307431743274337434743574367437743874397440744174427443744474457446744774487449745074517452745374547455745674577458745974607461746274637464746574667467746874697470747174727473747474757476747774787479748074817482748374847485748674877488748974907491749274937494749574967497749874997500750175027503750475057506750775087509751075117512751375147515751675177518751975207521752275237524752575267527752875297530753175327533753475357536753775387539754075417542754375447545754675477548754975507551755275537554755575567557755875597560756175627563756475657566756775687569757075717572757375747575757675777578757975807581758275837584758575867587758875897590759175927593759475957596759775987599760076017602760376047605760676077608760976107611761276137614761576167617761876197620762176227623762476257626762776287629763076317632763376347635763676377638763976407641764276437644764576467647764876497650765176527653765476557656765776587659766076617662766376647665766676677668766976707671767276737674767576767677767876797680768176827683768476857686768776887689769076917692769376947695769676977698769977007701770277037704770577067707770877097710771177127713771477157716771777187719772077217722772377247725772677277728772977307731773277337734773577367737773877397740774177427743774477457746774777487749775077517752775377547755775677577758775977607761776277637764776577667767776877697770777177727773777477757776777777787779778077817782778377847785778677877788778977907791779277937794779577967797779877997800780178027803780478057806780778087809781078117812781378147815781678177818781978207821782278237824782578267827782878297830783178327833783478357836783778387839784078417842784378447845784678477848784978507851785278537854785578567857785878597860786178627863786478657866786778687869787078717872787378747875787678777878787978807881788278837884788578867887788878897890789178927893789478957896789778987899790079017902790379047905790679077908790979107911791279137914791579167917791879197920792179227923792479257926792779287929793079317932793379347935793679377938793979407941794279437944794579467947794879497950795179527953795479557956795779587959796079617962796379647965796679677968796979707971797279737974797579767977797879797980798179827983798479857986798779887989799079917992799379947995799679977998799980008001800280038004800580068007800880098010801180128013801480158016801780188019802080218022802380248025802680278028802980308031803280338034803580368037803880398040804180428043804480458046804780488049805080518052805380548055805680578058805980608061806280638064806580668067806880698070807180728073807480758076807780788079808080818082808380848085808680878088808980908091809280938094809580968097809880998100810181028103810481058106810781088109811081118112811381148115811681178118811981208121812281238124812581268127812881298130813181328133813481358136813781388139814081418142814381448145814681478148814981508151815281538154815581568157815881598160816181628163816481658166816781688169817081718172817381748175817681778178817981808181818281838184818581868187818881898190819181928193819481958196819781988199820082018202820382048205820682078208820982108211821282138214821582168217821882198220822182228223822482258226822782288229823082318232823382348235823682378238823982408241824282438244824582468247824882498250825182528253825482558256825782588259826082618262826382648265826682678268826982708271827282738274827582768277827882798280828182828283828482858286828782888289829082918292829382948295829682978298829983008301830283038304830583068307830883098310831183128313831483158316831783188319832083218322832383248325832683278328832983308331833283338334833583368337833883398340834183428343834483458346834783488349835083518352835383548355835683578358835983608361836283638364836583668367836883698370837183728373837483758376837783788379838083818382838383848385838683878388838983908391839283938394839583968397839883998400840184028403840484058406840784088409841084118412841384148415841684178418841984208421842284238424842584268427842884298430843184328433843484358436843784388439844084418442844384448445844684478448844984508451845284538454845584568457845884598460846184628463846484658466846784688469847084718472847384748475847684778478847984808481848284838484848584868487848884898490849184928493849484958496849784988499850085018502850385048505850685078508850985108511851285138514851585168517851885198520852185228523852485258526852785288529853085318532853385348535853685378538853985408541854285438544854585468547854885498550855185528553855485558556855785588559856085618562856385648565856685678568856985708571857285738574857585768577857885798580858185828583858485858586858785888589859085918592859385948595859685978598859986008601860286038604860586068607860886098610861186128613861486158616861786188619862086218622862386248625862686278628862986308631863286338634863586368637863886398640864186428643864486458646864786488649865086518652865386548655865686578658865986608661866286638664866586668667866886698670867186728673867486758676867786788679868086818682868386848685868686878688868986908691869286938694869586968697869886998700870187028703870487058706870787088709871087118712871387148715871687178718871987208721872287238724872587268727872887298730873187328733873487358736873787388739874087418742874387448745874687478748874987508751875287538754875587568757875887598760876187628763876487658766876787688769877087718772877387748775877687778778877987808781878287838784878587868787878887898790879187928793879487958796879787988799880088018802880388048805880688078808880988108811881288138814881588168817881888198820882188228823882488258826882788288829883088318832883388348835883688378838883988408841884288438844884588468847884888498850885188528853885488558856885788588859886088618862886388648865886688678868886988708871887288738874887588768877887888798880888188828883888488858886888788888889889088918892889388948895889688978898889989008901890289038904890589068907890889098910891189128913891489158916891789188919892089218922892389248925892689278928892989308931893289338934893589368937893889398940894189428943894489458946894789488949895089518952895389548955895689578958895989608961896289638964896589668967896889698970897189728973897489758976897789788979898089818982898389848985898689878988898989908991899289938994899589968997899889999000900190029003900490059006900790089009901090119012901390149015901690179018901990209021902290239024902590269027902890299030903190329033903490359036903790389039904090419042904390449045904690479048904990509051905290539054905590569057905890599060906190629063906490659066906790689069907090719072907390749075907690779078907990809081908290839084908590869087908890899090909190929093909490959096909790989099910091019102910391049105910691079108910991109111911291139114911591169117911891199120912191229123912491259126912791289129913091319132913391349135913691379138913991409141914291439144914591469147914891499150915191529153915491559156915791589159916091619162916391649165916691679168916991709171917291739174917591769177917891799180918191829183918491859186918791889189919091919192919391949195919691979198919992009201920292039204920592069207920892099210921192129213921492159216921792189219922092219222922392249225922692279228922992309231923292339234923592369237923892399240924192429243924492459246924792489249925092519252925392549255925692579258925992609261926292639264926592669267926892699270927192729273927492759276927792789279928092819282928392849285928692879288928992909291929292939294929592969297929892999300930193029303930493059306930793089309931093119312931393149315931693179318931993209321932293239324932593269327932893299330933193329333933493359336933793389339934093419342934393449345934693479348934993509351935293539354935593569357935893599360936193629363936493659366936793689369937093719372937393749375937693779378937993809381938293839384938593869387938893899390939193929393939493959396939793989399940094019402940394049405940694079408940994109411941294139414941594169417941894199420942194229423942494259426942794289429943094319432943394349435943694379438943994409441944294439444944594469447944894499450945194529453945494559456945794589459946094619462946394649465946694679468946994709471947294739474947594769477947894799480948194829483948494859486948794889489949094919492949394949495949694979498949995009501950295039504950595069507950895099510951195129513951495159516951795189519952095219522952395249525952695279528952995309531953295339534953595369537953895399540954195429543954495459546954795489549955095519552955395549555955695579558955995609561956295639564956595669567956895699570957195729573957495759576957795789579958095819582958395849585958695879588958995909591959295939594959595969597959895999600960196029603960496059606960796089609961096119612961396149615961696179618961996209621962296239624962596269627962896299630963196329633963496359636963796389639964096419642964396449645964696479648964996509651965296539654965596569657965896599660966196629663966496659666966796689669967096719672967396749675967696779678967996809681968296839684968596869687968896899690969196929693969496959696969796989699970097019702970397049705970697079708970997109711971297139714971597169717971897199720972197229723972497259726972797289729973097319732973397349735973697379738973997409741974297439744974597469747974897499750975197529753975497559756975797589759976097619762976397649765976697679768976997709771977297739774977597769777977897799780978197829783978497859786978797889789979097919792979397949795979697979798979998009801980298039804980598069807980898099810981198129813981498159816981798189819982098219822982398249825982698279828982998309831983298339834983598369837983898399840984198429843984498459846984798489849985098519852985398549855985698579858985998609861986298639864986598669867986898699870987198729873987498759876987798789879988098819882988398849885988698879888988998909891989298939894989598969897989898999900990199029903990499059906990799089909991099119912991399149915991699179918991999209921992299239924992599269927992899299930993199329933993499359936993799389939994099419942994399449945994699479948994999509951995299539954995599569957995899599960996199629963996499659966996799689969997099719972997399749975997699779978997999809981998299839984998599869987998899899990999199929993999499959996999799989999100001000110002100031000410005100061000710008100091001010011100121001310014100151001610017100181001910020100211002210023100241002510026100271002810029100301003110032100331003410035100361003710038100391004010041100421004310044100451004610047100481004910050100511005210053100541005510056100571005810059100601006110062100631006410065100661006710068100691007010071100721007310074100751007610077100781007910080100811008210083100841008510086100871008810089100901009110092100931009410095100961009710098100991010010101101021010310104101051010610107101081010910110101111011210113101141011510116101171011810119101201012110122101231012410125101261012710128101291013010131101321013310134101351013610137101381013910140101411014210143101441014510146101471014810149101501015110152101531015410155101561015710158101591016010161101621016310164101651016610167101681016910170101711017210173101741017510176101771017810179101801018110182101831018410185101861018710188101891019010191101921019310194101951019610197101981019910200102011020210203102041020510206102071020810209102101021110212102131021410215102161021710218102191022010221102221022310224102251022610227102281022910230102311023210233102341023510236102371023810239102401024110242102431024410245102461024710248102491025010251102521025310254102551025610257102581025910260102611026210263102641026510266102671026810269102701027110272102731027410275102761027710278102791028010281102821028310284102851028610287102881028910290102911029210293102941029510296102971029810299103001030110302103031030410305103061030710308103091031010311103121031310314103151031610317103181031910320103211032210323103241032510326103271032810329103301033110332103331033410335103361033710338103391034010341103421034310344103451034610347103481034910350103511035210353103541035510356103571035810359103601036110362103631036410365103661036710368103691037010371103721037310374103751037610377103781037910380103811038210383103841038510386103871038810389103901039110392103931039410395103961039710398103991040010401104021040310404104051040610407104081040910410104111041210413104141041510416104171041810419104201042110422104231042410425104261042710428104291043010431104321043310434104351043610437104381043910440104411044210443104441044510446104471044810449104501045110452104531045410455104561045710458104591046010461104621046310464104651046610467104681046910470104711047210473104741047510476104771047810479104801048110482104831048410485104861048710488104891049010491104921049310494104951049610497104981049910500105011050210503105041050510506105071050810509105101051110512105131051410515105161051710518105191052010521105221052310524105251052610527105281052910530105311053210533105341053510536105371053810539105401054110542105431054410545105461054710548105491055010551105521055310554105551055610557105581055910560105611056210563105641056510566105671056810569105701057110572105731057410575105761057710578105791058010581105821058310584105851058610587105881058910590105911059210593105941059510596105971059810599106001060110602106031060410605106061060710608106091061010611106121061310614106151061610617106181061910620106211062210623106241062510626106271062810629106301063110632106331063410635106361063710638106391064010641106421064310644106451064610647106481064910650106511065210653106541065510656106571065810659106601066110662106631066410665106661066710668106691067010671106721067310674106751067610677106781067910680106811068210683106841068510686106871068810689106901069110692106931069410695106961069710698106991070010701107021070310704107051070610707107081070910710107111071210713107141071510716107171071810719107201072110722107231072410725107261072710728107291073010731107321073310734107351073610737107381073910740107411074210743107441074510746107471074810749107501075110752107531075410755107561075710758107591076010761
  1. /*##############################################################################
  2. HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. ############################################################################## */
  13. #include "jlib.hpp"
  14. #include "workunit.hpp"
  15. #include "jprop.hpp"
  16. #include "jmisc.hpp"
  17. #include "jexcept.hpp"
  18. #include "jiter.ipp"
  19. #include "jptree.hpp"
  20. #include "jtime.ipp"
  21. #include "jencrypt.hpp"
  22. #include "junicode.hpp"
  23. #include "jlzw.hpp"
  24. #include "jregexp.hpp"
  25. #include "eclrtl.hpp"
  26. #include "deftype.hpp"
  27. #include <time.h>
  28. #include "mpbase.hpp"
  29. #include "daclient.hpp"
  30. #include "dadfs.hpp"
  31. #include "dafdesc.hpp"
  32. #include "dasds.hpp"
  33. #include "danqs.hpp"
  34. #include "dautils.hpp"
  35. #include "dllserver.hpp"
  36. #include "thorhelper.hpp"
  37. #include "workflow.hpp"
  38. #include "nbcd.hpp"
  39. #include "seclib.hpp"
  40. #include "wuerror.hpp"
  41. #include "wujobq.hpp"
  42. #include "environment.hpp"
  43. #define GLOBAL_WORKUNIT "global"
  44. #define SDS_LOCK_TIMEOUT (5*60*1000) // 5mins, 30s a bit short
  45. static int workUnitTraceLevel = 1;
  46. static StringBuffer &getXPath(StringBuffer &wuRoot, const char *wuid)
  47. {
  48. // MORE - can fold in the date
  49. return wuRoot.append("/WorkUnits/").append(wuid);
  50. }
  51. //To be called by eclserver, but esp etc. won't know, so we need to store it.
  52. static StringBuffer & appendLibrarySuffix(StringBuffer & suffix)
  53. {
  54. #ifdef _WIN32
  55. suffix.append("W");
  56. #else
  57. suffix.append("L");
  58. #endif
  59. #ifdef __64BIT__
  60. suffix.append("64");
  61. #else
  62. suffix.append("32");
  63. #endif
  64. return suffix;
  65. }
  66. typedef MapStringTo<bool> UniqueScopes;
  67. static void wuAccessError(const char *username, const char *action, const char *wuscope, const char *wuid, bool excpt, bool log)
  68. {
  69. StringBuffer err;
  70. err.append("Workunit Access Denied - action: ").append(action).append(" user:").append(username ? username : "<Unknown>");
  71. if (wuid)
  72. err.append(" workunit:").append(wuid);
  73. if (wuscope)
  74. err.append(" scope:").append(wuscope);
  75. //MORE - we would need more information passed in from outside if we want to make the audit message format the same as from higher level ESP calls
  76. SYSLOG(AUDIT_TYPE_ACCESS_FAILURE, err.str());
  77. if (log)
  78. LOG(MCuserError, "%s", err.str());
  79. if (excpt)
  80. throw MakeStringException(WUERR_AccessError, "%s", err.str());
  81. }
  82. static bool checkWuScopeSecAccess(const char *wuscope, ISecManager &secmgr, ISecUser *secuser, int required, const char *action, bool excpt, bool log)
  83. {
  84. bool ret=(!secuser) ? true : (secmgr.authorizeEx(RT_WORKUNIT_SCOPE, *secuser, wuscope)>=required);
  85. if (!ret && (log || excpt))
  86. wuAccessError(secuser ? secuser->getName() : NULL, action, wuscope, NULL, excpt, log);
  87. return ret;
  88. }
  89. static bool checkWuScopeListSecAccess(const char *wuscope, ISecResourceList *scopes, int required, const char *action, bool excpt, bool log)
  90. {
  91. if (!scopes)
  92. return true;
  93. bool ret=true;
  94. if (wuscope)
  95. {
  96. Owned<ISecResource> res=scopes->getResource(wuscope);
  97. if (!res || res->getAccessFlags()<required)
  98. ret=false;
  99. }
  100. else
  101. {
  102. for (int seq=0; ret && seq<scopes->count(); seq++)
  103. {
  104. ISecResource *res=scopes->queryResource(seq);
  105. if (res && res->getAccessFlags()<required)
  106. return false;
  107. }
  108. }
  109. if (!ret && (log || excpt))
  110. wuAccessError(NULL, action, wuscope, NULL, excpt, log);
  111. return ret;
  112. }
  113. static bool checkWuSecAccess(IConstWorkUnit &cw, ISecManager &secmgr, ISecUser *secuser, int required, const char *action, bool excpt, bool log)
  114. {
  115. SCMStringBuffer wuscope;
  116. bool ret=(!secuser) ? true : (secmgr.authorizeEx(RT_WORKUNIT_SCOPE, *secuser, cw.getWuScope(wuscope).str())>=required);
  117. if (!ret && (log || excpt))
  118. {
  119. wuAccessError(secuser ? secuser->getName() : NULL, action, wuscope.str(), cw.queryWuid(), excpt, log);
  120. }
  121. return ret;
  122. }
  123. static bool checkWuSecAccess(const char *wuid, ISecManager &secmgr, ISecUser *secuser, int required, const char *action, bool excpt, bool log)
  124. {
  125. StringBuffer wuRoot;
  126. Owned<IRemoteConnection> conn = querySDS().connect(getXPath(wuRoot, wuid).str(), myProcessSession(), 0, SDS_LOCK_TIMEOUT);
  127. if (conn)
  128. {
  129. Owned<IPropertyTree> ptree=conn->getRoot();
  130. return checkWuScopeSecAccess(ptree->queryProp("@scope"), secmgr, secuser, required, action, excpt, log);
  131. }
  132. if (log || excpt)
  133. wuAccessError(secuser ? secuser->getName() : NULL, action, "Unknown", NULL, excpt, log);
  134. return false;
  135. }
  136. void doDescheduleWorkkunit(char const * wuid)
  137. {
  138. StringBuffer xpath;
  139. xpath.append("*/*/*/");
  140. ncnameEscape(wuid, xpath);
  141. Owned<IRemoteConnection> conn = querySDS().connect("/Schedule", myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
  142. if(!conn) return;
  143. Owned<IPropertyTree> root = conn->getRoot();
  144. bool more;
  145. do more = root->removeProp(xpath.str()); while(more);
  146. }
  147. #define PROGRESS_FORMAT_V 2
  148. class CConstGraphProgress : public CInterface, implements IConstWUGraphProgress
  149. {
  150. class CWuGraphStats : public CInterfaceOf<IWUGraphStats>
  151. {
  152. public:
  153. CWuGraphStats(CConstGraphProgress &_parent, StatisticCreatorType _creatorType, const char * _creator, const char * _rootScope, unsigned _id)
  154. : parent(_parent), creatorType(_creatorType), creator(_creator), id(_id)
  155. {
  156. StringBuffer subgraphScopeName;
  157. subgraphScopeName.append(_rootScope);
  158. StatisticScopeType scopeType = SSTgraph;
  159. StatsScopeId rootScopeId;
  160. verifyex(rootScopeId.setScopeText(_rootScope));
  161. collector.setown(createStatisticsGatherer(_creatorType, _creator, rootScopeId));
  162. }
  163. virtual void beforeDispose()
  164. {
  165. Owned<IStatisticCollection> stats = collector->getResult();
  166. MemoryBuffer compressed;
  167. {
  168. MemoryBuffer serialized;
  169. serializeStatisticCollection(serialized, stats);
  170. compressToBuffer(compressed, serialized.length(), serialized.toByteArray());
  171. }
  172. unsigned minActivity = 0;
  173. unsigned maxActivity = 0;
  174. stats->getMinMaxActivity(minActivity, maxActivity);
  175. parent.setSubgraphStats(creatorType, creator, id, compressed, minActivity, maxActivity);
  176. }
  177. virtual IStatisticGatherer & queryStatsBuilder()
  178. {
  179. return *collector;
  180. }
  181. protected:
  182. CConstGraphProgress &parent;
  183. Owned<IStatisticGatherer> collector;
  184. StringAttr creator;
  185. StatisticCreatorType creatorType;
  186. unsigned id;
  187. };
  188. class CGraphProgress : public CInterface, implements IWUGraphProgress
  189. {
  190. CConstGraphProgress &parent;
  191. public:
  192. IMPLEMENT_IINTERFACE;
  193. CGraphProgress(CConstGraphProgress &_parent) : parent(_parent)
  194. {
  195. parent.lockWrite();
  196. }
  197. ~CGraphProgress()
  198. {
  199. parent.unlock();
  200. }
  201. virtual IPropertyTree * getProgressTree() { return parent.getProgressTree(); }
  202. virtual WUGraphState queryGraphState() { return parent.queryGraphState(); }
  203. virtual WUGraphState queryNodeState(WUGraphIDType nodeId) { return parent.queryNodeState(nodeId); }
  204. virtual IWUGraphProgress * update() { throwUnexpected(); }
  205. virtual IWUGraphStats * update(StatisticCreatorType creatorType, const char * creator, unsigned subgraph) { throwUnexpected(); }
  206. virtual unsigned queryFormatVersion() { return parent.queryFormatVersion(); }
  207. virtual void setGraphState(WUGraphState state)
  208. {
  209. parent.setGraphState(state);
  210. }
  211. virtual void setNodeState(WUGraphIDType nodeId, WUGraphState state)
  212. {
  213. parent.setNodeState(nodeId, state);
  214. }
  215. };
  216. void clearConnection()
  217. {
  218. conn.clear();
  219. progress.clear();
  220. }
  221. public:
  222. IMPLEMENT_IINTERFACE;
  223. static void deleteWuidProgress(const char *wuid)
  224. {
  225. StringBuffer path("/GraphProgress/");
  226. path.append(wuid);
  227. Owned<IRemoteConnection> conn = querySDS().connect(path.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
  228. if (conn)
  229. conn->close(true);
  230. }
  231. CConstGraphProgress(const char *_wuid, const char *_graphName) : wuid(_wuid), graphName(_graphName)
  232. {
  233. rootPath.append("/GraphProgress/").append(wuid).append('/').append(graphName).append('/');
  234. connected = connectedWrite = false;
  235. formatVersion = 0;
  236. }
  237. CConstGraphProgress(const char *_wuid, const char *_graphName, IPropertyTree *_progress) : wuid(_wuid), graphName(_graphName), progress(_progress)
  238. {
  239. formatVersion = progress->getPropInt("@format");
  240. connectedWrite = false; // should never be
  241. connected = true;
  242. }
  243. void connect()
  244. {
  245. clearConnection();
  246. packProgress(wuid,false);
  247. conn.setown(querySDS().connect(rootPath.str(), myProcessSession(), RTM_LOCK_READ|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT));
  248. progress.set(conn->queryRoot());
  249. formatVersion = progress->getPropInt("@format");
  250. connected = true;
  251. }
  252. void lockWrite()
  253. {
  254. if (connectedWrite) return;
  255. if (!rootPath.length())
  256. throw MakeStringException(WUERR_GraphProgressWriteUnsupported, "Writing to graph progress unsupported in this context");
  257. // JCSMORE - look at using changeMode here.
  258. if (conn)
  259. clearConnection();
  260. else
  261. packProgress(wuid,false);
  262. conn.setown(querySDS().connect(rootPath.str(), myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT));
  263. progress.set(conn->queryRoot());
  264. if (!progress->hasChildren()) // i.e. blank.
  265. {
  266. formatVersion = PROGRESS_FORMAT_V;
  267. progress->setPropInt("@format", PROGRESS_FORMAT_V);
  268. }
  269. else
  270. formatVersion = progress->getPropInt("@format");
  271. connected = connectedWrite = true;
  272. }
  273. void unlock()
  274. {
  275. connected = false;
  276. connectedWrite = false;
  277. clearConnection();
  278. }
  279. static bool getRunningGraph(const char *wuid, IStringVal &graphName, WUGraphIDType &subId)
  280. {
  281. StringBuffer path;
  282. Owned<IRemoteConnection> conn = querySDS().connect(path.append("/GraphProgress/").append(wuid).str(), myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT);
  283. if (!conn) return false;
  284. const char *name = conn->queryRoot()->queryProp("Running/@graph");
  285. if (name)
  286. {
  287. graphName.set(name);
  288. subId = conn->queryRoot()->getPropInt64("Running/@subId");
  289. return true;
  290. }
  291. else
  292. return false;
  293. }
  294. void setGraphState(WUGraphState state)
  295. {
  296. progress->setPropInt("@_state", (unsigned)state);
  297. }
  298. void setNodeState(WUGraphIDType nodeId, WUGraphState state)
  299. {
  300. if (!connectedWrite) lockWrite();
  301. StringBuffer path;
  302. path.append("node[@id=\"").append(nodeId).append("\"]");
  303. IPropertyTree *node = progress->queryPropTree(path.str());
  304. if (!node)
  305. {
  306. node = progress->addPropTree("node", createPTree());
  307. node->setPropInt("@id", (int)nodeId);
  308. }
  309. node->setPropInt("@_state", (unsigned)state);
  310. switch (state)
  311. {
  312. case WUGraphRunning:
  313. {
  314. StringBuffer path;
  315. Owned<IRemoteConnection> conn = querySDS().connect(path.append("/GraphProgress/").append(wuid).str(), myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
  316. IPropertyTree *running = conn->queryRoot()->setPropTree("Running", createPTree());
  317. running->setProp("@graph", graphName);
  318. running->setPropInt64("@subId", nodeId);
  319. break;
  320. }
  321. case WUGraphComplete:
  322. {
  323. StringBuffer path;
  324. Owned<IRemoteConnection> conn = querySDS().connect(path.append("/GraphProgress/").append(wuid).str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
  325. conn->queryRoot()->removeProp("Running"); // only one thing running at any given time and one thing with lockWrite access
  326. break;
  327. }
  328. }
  329. }
  330. virtual IPropertyTree * getProgressTree()
  331. {
  332. if (!connected) connect();
  333. if (progress->getPropBool("@stats"))
  334. return createProcessTreeFromStats();
  335. return LINK(progress);
  336. }
  337. virtual WUGraphState queryGraphState()
  338. {
  339. return (WUGraphState)queryProgressStateTree()->getPropInt("@_state", (unsigned)WUGraphUnknown);
  340. }
  341. virtual WUGraphState queryNodeState(WUGraphIDType nodeId)
  342. {
  343. StringBuffer path;
  344. path.append("node[@id=\"").append(nodeId).append("\"]/@_state");
  345. return (WUGraphState)queryProgressStateTree()->getPropInt(path.str(), (unsigned)WUGraphUnknown);
  346. }
  347. virtual IWUGraphProgress * update()
  348. {
  349. return new CGraphProgress(*this);
  350. }
  351. virtual IWUGraphStats * update(StatisticCreatorType creatorType, const char * creator, unsigned subgraph)
  352. {
  353. return new CWuGraphStats(*this, creatorType, creator, graphName, subgraph);
  354. }
  355. virtual unsigned queryFormatVersion()
  356. {
  357. if (!connected) connect();
  358. return formatVersion;
  359. }
  360. static bool packProgress(const char *wuid,bool pack)
  361. {
  362. //MORE: This function could be deleted, because progress information is never actually packed
  363. StringBuffer path;
  364. path.append("/GraphProgress/").append(wuid);
  365. Owned<IRemoteConnection> conn(querySDS().connect(path.str(), myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT));
  366. if (!conn)
  367. return false;
  368. Owned<IPropertyTree> newt;
  369. MemoryBuffer buf;
  370. IPropertyTree *root = conn->queryRoot();
  371. if (root->getPropBin("Packed",buf)) {
  372. if (pack)
  373. return true;
  374. newt.setown(createPTree(buf));
  375. IPropertyTree *running = root->queryPropTree("Running");
  376. if (running)
  377. newt->setPropTree("Running",createPTreeFromIPT(running));
  378. }
  379. else {
  380. if (!pack)
  381. return true;
  382. newt.setown(createPTree(wuid));
  383. IPropertyTree *running = root->queryPropTree("Running");
  384. if (running) {
  385. newt->setPropTree("Running",createPTreeFromIPT(running));
  386. root->removeTree(running);
  387. }
  388. root->serialize(buf);
  389. newt->setPropBin("Packed",buf.length(),buf.bufferBase());
  390. }
  391. root->setPropTree(NULL,newt.getClear());
  392. return true;
  393. }
  394. private:
  395. IPropertyTree * queryProgressStateTree()
  396. {
  397. if (!connected) connect();
  398. return progress;
  399. }
  400. static void expandStats(IPropertyTree * target, IStatisticCollection & collection)
  401. {
  402. StringBuffer formattedValue;
  403. unsigned numStats = collection.getNumStatistics();
  404. for (unsigned i=0; i < numStats; i++)
  405. {
  406. StatisticKind kind;
  407. unsigned __int64 value;
  408. collection.getStatistic(kind, value, i);
  409. formatStatistic(formattedValue.clear(), value, kind);
  410. target->setProp(queryTreeTag(kind), formattedValue);
  411. }
  412. }
  413. void expandProcessTreeFromStats(IPropertyTree * rootTarget, IPropertyTree * target, IStatisticCollection * collection)
  414. {
  415. expandStats(target, *collection);
  416. StringBuffer scopeName;
  417. Owned<IStatisticCollectionIterator> activityIter = &collection->getScopes(NULL);
  418. ForEach(*activityIter)
  419. {
  420. IStatisticCollection & cur = activityIter->query();
  421. cur.getScope(scopeName.clear());
  422. const char * id = scopeName.str();
  423. const char * tag;
  424. IPropertyTree * curTarget = target;
  425. switch (cur.queryScopeType())
  426. {
  427. case SSTedge:
  428. tag = "edge";
  429. id += strlen(EdgeScopePrefix);
  430. break;
  431. case SSTactivity:
  432. tag = "node";
  433. id += strlen(ActivityScopePrefix);
  434. break;
  435. case SSTsubgraph:
  436. //All subgraphs are added a root elements in the progress tree
  437. curTarget = rootTarget;
  438. tag = "node";
  439. id += strlen(SubGraphScopePrefix);
  440. break;
  441. default:
  442. throwUnexpected();
  443. }
  444. IPropertyTree * next = curTarget->addPropTree(tag, createPTree());
  445. next->setProp("@id", id);
  446. expandProcessTreeFromStats(rootTarget, next, &cur);
  447. }
  448. }
  449. IPropertyTree * createProcessTreeFromStats()
  450. {
  451. MemoryBuffer compressed;
  452. MemoryBuffer serialized;
  453. Owned<IPropertyTree> progressTree = createPTree();
  454. Owned<IPropertyTreeIterator> iter = progress->getElements("sg*");
  455. ForEach(*iter)
  456. {
  457. IPropertyTree & curSubGraph = iter->query();
  458. curSubGraph.getPropBin("Stats", compressed.clear());
  459. //Protect against updates that delete the stats while we are iterating
  460. if (compressed.length())
  461. {
  462. decompressToBuffer(serialized.clear(), compressed);
  463. Owned<IStatisticCollection> collection = createStatisticCollection(serialized);
  464. expandProcessTreeFromStats(progressTree, progressTree, collection);
  465. }
  466. }
  467. return progressTree.getClear();
  468. }
  469. void setSubgraphStats(StatisticCreatorType creatorType, const char * creator, unsigned id, const MemoryBuffer & compressed, unsigned minActivity, unsigned maxActivity)
  470. {
  471. StringBuffer tag;
  472. tag.append("sg").append(id);
  473. IPropertyTree * subgraph = createPTree(tag);
  474. subgraph->setProp("@c", queryCreatorTypeName(creatorType));
  475. subgraph->setProp("@creator", creator);
  476. subgraph->setPropInt("@minActivity", minActivity);
  477. subgraph->setPropInt("@maxActivity", maxActivity);
  478. //Replace the particular subgraph statistics added by this creator
  479. tag.append("[@creator='").append(creator).append("']");
  480. lockWrite();
  481. subgraph = progress->setPropTree(tag, subgraph);
  482. subgraph->setPropBin("Stats", compressed.length(), compressed.toByteArray());
  483. if (!progress->getPropBool("@stats", false))
  484. progress->setPropBool("@stats", true);
  485. unlock();
  486. }
  487. private:
  488. Owned<IRemoteConnection> conn;
  489. Linked<IPropertyTree> progress;
  490. StringAttr wuid, graphName;
  491. StringBuffer rootPath;
  492. bool connected, connectedWrite;
  493. unsigned formatVersion;
  494. };
  495. //--------------------------------------------------------------------------------------------------------------------
  496. class ExtractedStatistic : public CInterfaceOf<IConstWUStatistic>
  497. {
  498. public:
  499. virtual IStringVal & getDescription(IStringVal & str, bool createDefault) const
  500. {
  501. str.set(description);
  502. return str;
  503. }
  504. virtual IStringVal & getCreator(IStringVal & str) const
  505. {
  506. str.set(creator);
  507. return str;
  508. }
  509. virtual IStringVal & getScope(IStringVal & str) const
  510. {
  511. str.set(scope);
  512. return str;
  513. }
  514. virtual IStringVal & getFormattedValue(IStringVal & str) const
  515. {
  516. StringBuffer formatted;
  517. formatStatistic(formatted, value, measure);
  518. str.set(formatted);
  519. return str;
  520. }
  521. virtual StatisticMeasure getMeasure() const
  522. {
  523. return measure;
  524. }
  525. virtual StatisticKind getKind() const
  526. {
  527. return kind;
  528. }
  529. virtual StatisticCreatorType getCreatorType() const
  530. {
  531. return creatorType;
  532. }
  533. virtual StatisticScopeType getScopeType() const
  534. {
  535. return scopeType;
  536. }
  537. virtual unsigned __int64 getValue() const
  538. {
  539. return value;
  540. }
  541. virtual unsigned __int64 getCount() const
  542. {
  543. return count;
  544. }
  545. virtual unsigned __int64 getMax() const
  546. {
  547. return max;
  548. }
  549. virtual unsigned __int64 getTimestamp() const
  550. {
  551. return timeStamp;
  552. }
  553. virtual bool matches(const IStatisticsFilter * filter) const
  554. {
  555. return filter->matches(creatorType, creator, scopeType, scope, measure, kind);
  556. }
  557. public:
  558. StringAttr creator;
  559. StringAttr description;
  560. StringBuffer scope;
  561. StatisticMeasure measure;
  562. StatisticKind kind;
  563. StatisticCreatorType creatorType;
  564. StatisticScopeType scopeType;
  565. unsigned __int64 value;
  566. unsigned __int64 count;
  567. unsigned __int64 max;
  568. unsigned __int64 timeStamp;
  569. };
  570. class CConstGraphProgressStatisticsIterator : public CInterfaceOf<IConstWUStatisticIterator>
  571. {
  572. public:
  573. CConstGraphProgressStatisticsIterator(const char * wuid, const IStatisticsFilter * _filter) : filter(_filter)
  574. {
  575. if (filter)
  576. scopes.appendList(filter->queryScope(), ":");
  577. const char * searchGraph = "*";
  578. if (scopes.ordinality())
  579. searchGraph = scopes.item(0);
  580. rootPath.append("/GraphProgress/").append(wuid).append('/');
  581. bool singleGraph = false;
  582. if (!containsWildcard(searchGraph))
  583. {
  584. rootPath.append(searchGraph).append("/");
  585. singleGraph = true;
  586. }
  587. //Don't lock the statistics while we iterate - any partial updates must not cause problems
  588. conn.setown(querySDS().connect(rootPath.str(), myProcessSession(), RTM_NONE, SDS_LOCK_TIMEOUT));
  589. if (conn && !singleGraph)
  590. graphIter.setown(conn->queryRoot()->getElements("*"));
  591. curStat.setown(new ExtractedStatistic);
  592. //These are currently constant for all graph statistics instances
  593. curStat->count = 1;
  594. curStat->max = 0;
  595. valid = false;
  596. }
  597. virtual IConstWUStatistic & query()
  598. {
  599. return *curStat;
  600. }
  601. virtual bool first()
  602. {
  603. valid = false;
  604. if (!conn)
  605. return false;
  606. if (graphIter && !graphIter->first())
  607. return false;
  608. ensureUniqueStatistic();
  609. if (!firstSubGraph())
  610. return false;
  611. valid = true;
  612. return true;
  613. }
  614. virtual bool next()
  615. {
  616. ensureUniqueStatistic();
  617. if (!nextStatistic())
  618. {
  619. if (!nextSubGraph())
  620. {
  621. if (!nextGraph())
  622. {
  623. valid = false;
  624. return false;
  625. }
  626. }
  627. }
  628. return true;
  629. }
  630. virtual bool isValid()
  631. {
  632. return valid;
  633. }
  634. protected:
  635. bool firstSubGraph()
  636. {
  637. IPropertyTree & graphNode = graphIter ? graphIter->query() : *conn->queryRoot();
  638. const char * xpath = "sg*";
  639. StringBuffer childXpath;
  640. if (scopes.isItem(1))
  641. {
  642. const char * scope1 = scopes.item(1);
  643. if (strnicmp(scope1, "sg", 2) == 0)
  644. {
  645. childXpath.append(scope1);
  646. xpath = childXpath.str();
  647. }
  648. }
  649. subgraphIter.setown(graphNode.getElements(xpath));
  650. if (!subgraphIter)
  651. subgraphIter.setown(graphNode.getElements("sg0"));
  652. if (!subgraphIter->first())
  653. return false;
  654. if (firstStat())
  655. return true;
  656. return nextSubGraph();
  657. }
  658. bool nextSubGraph()
  659. {
  660. loop
  661. {
  662. if (!subgraphIter->next())
  663. return false;
  664. if (firstStat())
  665. return true;
  666. }
  667. }
  668. bool nextGraph()
  669. {
  670. if (!graphIter)
  671. return false;
  672. loop
  673. {
  674. if (!graphIter->next())
  675. return false;
  676. if (firstSubGraph())
  677. return true;
  678. }
  679. }
  680. bool firstStat()
  681. {
  682. IPropertyTree & curSubGraph = subgraphIter->query();
  683. if (!checkSubGraph())
  684. return false;
  685. curSubGraph.getPropBin("Stats", compressed.clear());
  686. //Don't crash on old format progress...
  687. if (compressed.length() == 0)
  688. return false;
  689. decompressToBuffer(serialized.clear(), compressed);
  690. Owned<IStatisticCollection> collection = createStatisticCollection(serialized);
  691. curStat->timeStamp = collection->queryWhenCreated();
  692. return beginCollection(*collection);
  693. }
  694. bool beginCollection(IStatisticCollection & collection)
  695. {
  696. collections.append(OLINK(collection));
  697. numStats = collection.getNumStatistics();
  698. curStatIndex = 0;
  699. if (checkScope())
  700. {
  701. if (curStatIndex < numStats)
  702. {
  703. if (checkStatistic())
  704. return true;
  705. return nextStatistic();
  706. }
  707. }
  708. return nextChildScope();
  709. }
  710. bool nextStatistic()
  711. {
  712. //Finish iterating the statistics at this level.
  713. while (++curStatIndex < numStats)
  714. {
  715. if (checkStatistic())
  716. return true;
  717. }
  718. return nextChildScope();
  719. }
  720. bool nextChildScope()
  721. {
  722. loop
  723. {
  724. if (collections.ordinality() == 0)
  725. return false;
  726. IStatisticCollection * curCollection = &collections.tos();
  727. if (childIterators.ordinality() < collections.ordinality())
  728. {
  729. //Start iterating the children for the current collection
  730. childIterators.append(curCollection->getScopes(NULL));
  731. if (!childIterators.tos().first())
  732. {
  733. finishCollection();
  734. continue;
  735. }
  736. }
  737. else if (!childIterators.tos().next())
  738. {
  739. finishCollection();
  740. continue;
  741. }
  742. if (beginCollection(childIterators.tos().query()))
  743. return true;
  744. }
  745. }
  746. void finishCollection()
  747. {
  748. collections.pop();
  749. childIterators.pop();
  750. }
  751. bool checkSubGraph()
  752. {
  753. if (!filter)
  754. return true;
  755. IPropertyTree & curSubGraph = subgraphIter->query();
  756. curStat->creatorType = queryCreatorType(curSubGraph.queryProp("@c"));
  757. curStat->creator.set(curSubGraph.queryProp("@creator"));
  758. return filter->matches(curStat->creatorType, curStat->creator, SSTall, NULL, SMeasureAll, StKindAll);
  759. }
  760. bool checkScope()
  761. {
  762. if (!filter)
  763. return true;
  764. IStatisticCollection * collection = &collections.tos();
  765. curStat->scopeType = collection->queryScopeType();
  766. collection->getFullScope(curStat->scope.clear());
  767. return filter->matches(SCTall, NULL, curStat->scopeType, curStat->scope, SMeasureAll, StKindAll);
  768. }
  769. bool checkStatistic()
  770. {
  771. IStatisticCollection & collection = collections.tos();
  772. collection.getStatistic(curStat->kind, curStat->value, curStatIndex);
  773. curStat->measure = queryMeasure(curStat->kind);
  774. if (!filter)
  775. return true;
  776. if (!filter->matches(SCTall, NULL, SSTall, NULL, curStat->measure, curStat->kind))
  777. return false;
  778. return true;
  779. }
  780. void ensureUniqueStatistic()
  781. {
  782. //If something else has linked this statistic, clone a unique one.
  783. if (curStat->IsShared())
  784. curStat.setown(new ExtractedStatistic(*curStat));
  785. }
  786. private:
  787. Owned<IRemoteConnection> conn;
  788. Owned<ExtractedStatistic> curStat;
  789. const IStatisticsFilter * filter;
  790. StringArray scopes;
  791. StringBuffer rootPath;
  792. Owned<IPropertyTreeIterator> graphIter;
  793. Owned<IPropertyTreeIterator> subgraphIter;
  794. IArrayOf<IStatisticCollection> collections;
  795. IArrayOf<IStatisticCollectionIterator> childIterators;
  796. MemoryBuffer compressed;
  797. MemoryBuffer serialized;
  798. unsigned numStats;
  799. unsigned curStatIndex;
  800. bool valid;
  801. };
  802. //--------------------------------------------------------------------------------------------------------------------
  803. class CLocalWUAppValue : public CInterface, implements IConstWUAppValue
  804. {
  805. Owned<IPropertyTree> p;
  806. StringAttr prop;
  807. public:
  808. IMPLEMENT_IINTERFACE;
  809. CLocalWUAppValue(IPropertyTree *p,unsigned child);
  810. virtual IStringVal & getApplication(IStringVal & str) const;
  811. virtual IStringVal & getName(IStringVal & str) const;
  812. virtual IStringVal & getValue(IStringVal & dt) const;
  813. };
  814. class CLocalWUStatistic : public CInterface, implements IConstWUStatistic
  815. {
  816. Owned<IPropertyTree> p;
  817. public:
  818. IMPLEMENT_IINTERFACE;
  819. CLocalWUStatistic(IPropertyTree *p);
  820. virtual IStringVal & getCreator(IStringVal & str) const;
  821. virtual IStringVal & getDescription(IStringVal & str, bool createDefault) const;
  822. virtual IStringVal & getFormattedValue(IStringVal & str) const;
  823. virtual IStringVal & getType(IStringVal & str) const;
  824. virtual IStringVal & getScope(IStringVal & str) const;
  825. virtual StatisticMeasure getMeasure() const;
  826. virtual StatisticCreatorType getCreatorType() const;
  827. virtual StatisticScopeType getScopeType() const;
  828. virtual StatisticKind getKind() const;
  829. virtual unsigned __int64 getValue() const;
  830. virtual unsigned __int64 getCount() const;
  831. virtual unsigned __int64 getMax() const;
  832. virtual unsigned __int64 getTimestamp() const;
  833. virtual bool matches(const IStatisticsFilter * filter) const;
  834. };
  835. class CLocalWULegacyTiming : public CInterface, implements IConstWUStatistic
  836. {
  837. Owned<IPropertyTree> p;
  838. public:
  839. IMPLEMENT_IINTERFACE;
  840. CLocalWULegacyTiming(IPropertyTree *p);
  841. virtual IStringVal & getCreator(IStringVal & str) const;
  842. virtual IStringVal & getDescription(IStringVal & str, bool createDefault) const;
  843. virtual IStringVal & getFormattedValue(IStringVal & str) const;
  844. virtual IStringVal & getType(IStringVal & str) const;
  845. virtual IStringVal & getScope(IStringVal & str) const;
  846. virtual StatisticMeasure getMeasure() const;
  847. virtual StatisticCreatorType getCreatorType() const;
  848. virtual StatisticScopeType getScopeType() const;
  849. virtual StatisticKind getKind() const;
  850. virtual unsigned __int64 getValue() const;
  851. virtual unsigned __int64 getCount() const;
  852. virtual unsigned __int64 getMax() const;
  853. virtual unsigned __int64 getTimestamp() const;
  854. virtual bool matches(const IStatisticsFilter * filter) const;
  855. };
  856. template <typename T, typename IT> struct CachedTags
  857. {
  858. CachedTags(): cached(false) {}
  859. void load(IPropertyTree* p,const char* xpath)
  860. {
  861. if (!cached)
  862. {
  863. assertex(tags.length() == 0);
  864. Owned<IPropertyTreeIterator> r = p->getElements(xpath);
  865. for (r->first(); r->isValid(); r->next())
  866. {
  867. IPropertyTree *rp = &r->query();
  868. rp->Link();
  869. tags.append(*new T(rp));
  870. }
  871. cached = true;
  872. }
  873. }
  874. void append(IPropertyTree * p)
  875. {
  876. tags.append(*new T(p));
  877. }
  878. operator IArrayOf<IT>&() { return tags; }
  879. unsigned ordinality() const { return tags.ordinality(); }
  880. void kill()
  881. {
  882. cached = false;
  883. tags.kill();
  884. }
  885. bool cached;
  886. IArrayOf<IT> tags;
  887. };
  888. template <> struct CachedTags<CLocalWUAppValue, IConstWUAppValue>
  889. {
  890. CachedTags(): cached(false) {}
  891. void load(IPropertyTree* p,const char* xpath)
  892. {
  893. if (!cached)
  894. {
  895. assertex(tags.length() == 0);
  896. Owned<IPropertyTreeIterator> r = p->getElements(xpath);
  897. for (r->first(); r->isValid(); r->next())
  898. {
  899. IPropertyTree *rp = &r->query();
  900. Owned<IPropertyTreeIterator> v = rp->getElements("*");
  901. unsigned pos = 1;
  902. for (v->first(); v->isValid(); v->next())
  903. {
  904. rp->Link();
  905. tags.append(*new CLocalWUAppValue(rp,pos++));
  906. }
  907. }
  908. cached = true;
  909. }
  910. }
  911. operator IArrayOf<IConstWUAppValue>&() { return tags; }
  912. void kill()
  913. {
  914. cached = false;
  915. tags.kill();
  916. }
  917. bool cached;
  918. IArrayOf<IConstWUAppValue> tags;
  919. };
  920. class CLocalWorkUnit : public CInterface, implements IWorkUnit , implements IExtendedWUInterface
  921. {
  922. friend StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool decodeGraphs, bool includeProgress);
  923. friend void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool decodeGraphs, bool includeProgress);
  924. protected:
  925. Owned<IPropertyTree> p;
  926. mutable CriticalSection crit;
  927. mutable Owned<IWUQuery> query;
  928. mutable Owned<IWUWebServicesInfo> webServicesInfo;
  929. mutable Owned<IWURoxieQueryInfo> roxieQueryInfo;
  930. mutable Owned<IWorkflowItemIterator> workflowIterator;
  931. mutable bool workflowIteratorCached;
  932. mutable bool resultsCached;
  933. mutable bool temporariesCached;
  934. mutable bool variablesCached;
  935. mutable bool exceptionsCached;
  936. mutable bool pluginsCached;
  937. mutable bool librariesCached;
  938. mutable bool activitiesCached;
  939. mutable bool webServicesInfoCached;
  940. mutable bool roxieQueryInfoCached;
  941. mutable IArrayOf<IWUPlugin> plugins;
  942. mutable IArrayOf<IWULibrary> libraries;
  943. mutable IArrayOf<IWUException> exceptions;
  944. mutable IArrayOf<IWUGraph> graphs;
  945. mutable IArrayOf<IWUResult> results;
  946. mutable IArrayOf<IWUResult> temporaries;
  947. mutable IArrayOf<IWUResult> variables;
  948. mutable CachedTags<CLocalWUAppValue,IConstWUAppValue> appvalues;
  949. mutable CachedTags<CLocalWUStatistic,IConstWUStatistic> statistics;
  950. mutable CachedTags<CLocalWULegacyTiming,IConstWUStatistic> legacyTimings;
  951. mutable Owned<IUserDescriptor> userDesc;
  952. Mutex locked;
  953. Owned<ISecManager> secMgr;
  954. Owned<ISecUser> secUser;
  955. mutable Owned<IPropertyTree> cachedGraphs;
  956. public:
  957. IMPLEMENT_IINTERFACE;
  958. CLocalWorkUnit(IPropertyTree* root, ISecManager *secmgr, ISecUser *secuser);
  959. void beforeDispose();
  960. CLocalWorkUnit(const char *dummyWuid, const char *xml, ISecManager *secmgr, ISecUser *secuser);
  961. IPropertyTree *getUnpackedTree(bool includeProgress) const;
  962. ISecManager *querySecMgr(){return secMgr.get();}
  963. ISecUser *querySecUser(){return secUser.get();}
  964. void setSecIfcs(ISecManager *mgr, ISecUser*usr){secMgr.set(mgr); secUser.set(usr);}
  965. virtual bool aborting() const;
  966. virtual void forceReload() {};
  967. virtual WUAction getAction() const;
  968. virtual IStringVal& getActionEx(IStringVal & str) const;
  969. virtual IStringVal & getApplicationValue(const char * application, const char * propname, IStringVal & str) const;
  970. virtual int getApplicationValueInt(const char * application, const char * propname, int defVal) const;
  971. virtual IConstWUAppValueIterator & getApplicationValues() const;
  972. virtual bool hasWorkflow() const;
  973. virtual unsigned queryEventScheduledCount() const;
  974. virtual IPropertyTree * queryWorkflowTree() const;
  975. virtual IConstWorkflowItemIterator * getWorkflowItems() const;
  976. virtual IWorkflowItemArray * getWorkflowClone() const;
  977. virtual IConstLocalFileUploadIterator * getLocalFileUploads() const;
  978. virtual bool requiresLocalFileUpload() const;
  979. virtual bool getIsQueryService() const;
  980. virtual const char *queryClusterName() const;
  981. virtual bool hasDebugValue(const char * propname) const;
  982. virtual IStringVal & getDebugValue(const char * propname, IStringVal & str) const;
  983. virtual IStringIterator & getDebugValues() const;
  984. virtual IStringIterator & getDebugValues(const char *prop) const;
  985. virtual int getDebugValueInt(const char * propname, int defVal) const;
  986. virtual __int64 getDebugValueInt64(const char * propname, __int64 defVal) const;
  987. virtual bool getDebugValueBool(const char * propname, bool defVal) const;
  988. virtual unsigned getExceptionCount() const;
  989. virtual IConstWUExceptionIterator & getExceptions() const;
  990. virtual IConstWUResult * getGlobalByName(const char * name) const;
  991. virtual unsigned getGraphCount() const;
  992. virtual unsigned getSourceFileCount() const;
  993. virtual unsigned getResultCount() const;
  994. virtual unsigned getVariableCount() const;
  995. virtual unsigned getApplicationValueCount() const;
  996. virtual IConstWUGraphIterator & getGraphs(WUGraphType type) const;
  997. virtual IConstWUGraphMetaIterator & getGraphsMeta(WUGraphType type) const;
  998. virtual IConstWUGraph * getGraph(const char *name) const;
  999. virtual IConstWUGraphProgress * getGraphProgress(const char * name) const;
  1000. virtual const char *queryJobName() const;
  1001. virtual IConstWUPlugin * getPluginByName(const char * name) const;
  1002. virtual IConstWUPluginIterator & getPlugins() const;
  1003. virtual IConstWULibraryIterator & getLibraries() const;
  1004. virtual WUPriorityClass getPriority() const;
  1005. virtual int getPriorityLevel() const;
  1006. virtual int getPriorityValue() const;
  1007. virtual IConstWUQuery * getQuery() const;
  1008. virtual bool getRescheduleFlag() const;
  1009. virtual IConstWUResult * getResultByName(const char * name) const;
  1010. virtual IConstWUResult * getResultBySequence(unsigned seq) const;
  1011. virtual unsigned getResultLimit() const;
  1012. virtual IConstWUResultIterator & getResults() const;
  1013. virtual IStringVal & getScope(IStringVal & str) const;
  1014. virtual IStringVal & getSecurityToken(IStringVal & str) const;
  1015. virtual WUState getState() const;
  1016. virtual IStringVal & getStateEx(IStringVal & str) const;
  1017. virtual __int64 getAgentSession() const;
  1018. virtual unsigned getAgentPID() const;
  1019. virtual const char *queryStateDesc() const;
  1020. virtual IConstWUResult * getTemporaryByName(const char * name) const;
  1021. virtual IConstWUResultIterator & getTemporaries() const;
  1022. virtual IConstWUStatisticIterator & getStatistics(const IStatisticsFilter * filter) const;
  1023. virtual IConstWUStatistic * getStatistic(const char * creator, const char * scope, StatisticKind kind) const;
  1024. virtual IConstWUWebServicesInfo * getWebServicesInfo() const;
  1025. virtual IConstWURoxieQueryInfo * getRoxieQueryInfo() const;
  1026. virtual IStringVal & getXmlParams(IStringVal & params) const;
  1027. virtual const IPropertyTree *getXmlParams() const;
  1028. virtual unsigned __int64 getHash() const;
  1029. virtual IStringIterator *getLogs(const char *type, const char *component) const;
  1030. virtual IStringIterator *getProcesses(const char *type) const;
  1031. virtual IPropertyTreeIterator* getProcesses(const char *type, const char *instance) const;
  1032. virtual bool getWuDate(unsigned & year, unsigned & month, unsigned& day);
  1033. virtual IStringVal & getSnapshot(IStringVal & str) const;
  1034. virtual const char *queryUser() const;
  1035. virtual IStringVal & getWuScope(IStringVal & str) const;
  1036. virtual IConstWUResult * getVariableByName(const char * name) const;
  1037. virtual IConstWUResultIterator & getVariables() const;
  1038. virtual const char *queryWuid() const;
  1039. virtual bool getRunningGraph(IStringVal &graphName, WUGraphIDType &subId) const;
  1040. virtual bool isProtected() const;
  1041. virtual bool isPausing() const;
  1042. virtual IWorkUnit& lock();
  1043. virtual void requestAbort();
  1044. virtual void subscribe(WUSubscribeOptions options);
  1045. virtual unsigned calculateHash(unsigned prevHash);
  1046. virtual void copyWorkUnit(IConstWorkUnit *cached, bool all);
  1047. virtual unsigned queryFileUsage(const char *filename) const;
  1048. virtual bool getCloneable() const;
  1049. virtual IUserDescriptor * queryUserDescriptor() const;
  1050. virtual unsigned getCodeVersion() const;
  1051. virtual unsigned getWuidVersion() const;
  1052. virtual void getBuildVersion(IStringVal & buildVersion, IStringVal & eclVersion) const;
  1053. virtual IPropertyTree * getDiskUsageStats();
  1054. virtual IPropertyTreeIterator & getFileIterator() const;
  1055. virtual bool archiveWorkUnit(const char *base,bool del,bool ignoredllerrors,bool deleteOwned);
  1056. virtual void packWorkUnit(bool pack=true);
  1057. virtual IJlibDateTime & getTimeScheduled(IJlibDateTime &val) const;
  1058. virtual IPropertyTreeIterator & getFilesReadIterator() const;
  1059. virtual void protect(bool protectMode);
  1060. virtual IConstWULibrary * getLibraryByName(const char * name) const;
  1061. virtual unsigned getDebugAgentListenerPort() const;
  1062. virtual IStringVal & getDebugAgentListenerIP(IStringVal &ip) const;
  1063. void clearExceptions();
  1064. void commit();
  1065. IWUException *createException();
  1066. void addProcess(const char *type, const char *instance, unsigned pid, const char *log);
  1067. void setAction(WUAction action);
  1068. void setApplicationValue(const char * application, const char * propname, const char * value, bool overwrite);
  1069. void setApplicationValueInt(const char * application, const char * propname, int value, bool overwrite);
  1070. void incEventScheduledCount();
  1071. void setIsQueryService(bool value);
  1072. void setCloneable(bool value);
  1073. void setIsClone(bool value);
  1074. void setClusterName(const char * value);
  1075. void setCodeVersion(unsigned version, const char * buildVersion, const char * eclVersion);
  1076. void setDebugValue(const char * propname, const char * value, bool overwrite);
  1077. void setDebugValueInt(const char * propname, int value, bool overwrite);
  1078. void setJobName(const char * value);
  1079. void setPriority(WUPriorityClass cls);
  1080. void setPriorityLevel(int level);
  1081. void setRescheduleFlag(bool value);
  1082. void setResultLimit(unsigned value);
  1083. void setState(WUState state);
  1084. void setStateEx(const char * text);
  1085. void setAgentSession(__int64 sessionId);
  1086. void setSecurityToken(const char *value);
  1087. void setStatistic(StatisticCreatorType creatorType, const char * creator, StatisticScopeType scopeType, const char * scope, StatisticKind kind, const char * optDescription, unsigned __int64 value, unsigned __int64 count, unsigned __int64 maxValue, StatsMergeAction mergeAction);
  1088. void setTracingValue(const char * propname, const char * value);
  1089. void setTracingValueInt(const char * propname, int value);
  1090. void setUser(const char * value);
  1091. void setWuScope(const char * value);
  1092. void setSnapshot(const char * value);
  1093. void setDebugAgentListenerPort(unsigned port);
  1094. void setDebugAgentListenerIP(const char * ip);
  1095. void setXmlParams(const char *params);
  1096. void setXmlParams(IPropertyTree *tree);
  1097. void setHash(unsigned __int64 hash);
  1098. IWorkflowItem* addWorkflowItem(unsigned wfid, WFType type, WFMode mode, unsigned success, unsigned failure, unsigned recovery, unsigned retriesAllowed, unsigned contingencyFor);
  1099. IWorkflowItemIterator * updateWorkflowItems();
  1100. void syncRuntimeWorkflow(IWorkflowItemArray * array);
  1101. void resetWorkflow();
  1102. void schedule();
  1103. void deschedule();
  1104. unsigned addLocalFileUpload(LocalFileUploadType type, char const * source, char const * destination, char const * eventTag);
  1105. IWUResult * updateGlobalByName(const char * name);
  1106. IWUGraph * createGraph(const char * name, WUGraphType type, IPropertyTree *xgmml);
  1107. IWUGraph * updateGraph(const char * name);
  1108. IWUQuery * updateQuery();
  1109. IWUWebServicesInfo* updateWebServicesInfo(bool create);
  1110. IWURoxieQueryInfo* updateRoxieQueryInfo(const char *wuid, const char *roxieClusterName);
  1111. IWUPlugin * updatePluginByName(const char * name);
  1112. IWULibrary * updateLibraryByName(const char * name);
  1113. IWUResult * updateResultByName(const char * name);
  1114. IWUResult * updateResultBySequence(unsigned seq);
  1115. IWUResult * updateTemporaryByName(const char * name);
  1116. IWUResult * updateVariableByName(const char * name);
  1117. void addFile(const char *fileName, StringArray *clusters, unsigned usageCount, WUFileKind fileKind, const char *graphOwner);
  1118. void noteFileRead(IDistributedFile *file);
  1119. void releaseFile(const char *fileName);
  1120. void clearGraphProgress();
  1121. void resetBeforeGeneration();
  1122. void deleteTempFiles(const char *graph, bool deleteOwned, bool deleteJobOwned);
  1123. void deleteTemporaries();
  1124. void addDiskUsageStats(__int64 avgNodeUsage, unsigned minNode, __int64 minNodeUsage, unsigned maxNode, __int64 maxNodeUsage, __int64 graphId);
  1125. void setTimeScheduled(const IJlibDateTime &val);
  1126. // ILocalWorkUnit - used for debugging etc
  1127. void loadXML(const char *xml);
  1128. void serialize(MemoryBuffer &tgt);
  1129. void deserialize(MemoryBuffer &src);
  1130. IWorkUnit &lockRemote(bool commit);
  1131. void unlockRemote();
  1132. void abort();
  1133. void cleanupAndDelete(bool deldll,bool deleteOwned, const StringArray *deleteExclusions=NULL);
  1134. bool switchThorQueue(const char *cluster, IQueueSwitcher *qs);
  1135. void setAllowedClusters(const char *value);
  1136. IStringVal & getAllowedClusters(IStringVal & str) const;
  1137. void remoteCheckAccess(IUserDescriptor *user, bool writeaccess) const;
  1138. void setAllowAutoQueueSwitch(bool val);
  1139. bool getAllowAutoQueueSwitch() const;
  1140. void setLibraryInformation(const char * name, unsigned interfaceHash, unsigned definitionHash);
  1141. virtual void setResultInt(const char * name, unsigned sequence, __int64 val)
  1142. {
  1143. Owned<IWUResult> r = updateResult(name, sequence);
  1144. if (r)
  1145. {
  1146. r->setResultInt(val);
  1147. r->setResultStatus(ResultStatusCalculated);
  1148. }
  1149. }
  1150. virtual void setResultUInt(const char * name, unsigned sequence, unsigned __int64 val)
  1151. {
  1152. Owned<IWUResult> r = updateResult(name, sequence);
  1153. if (r)
  1154. {
  1155. r->setResultUInt(val);
  1156. r->setResultStatus(ResultStatusCalculated);
  1157. }
  1158. }
  1159. virtual void setResultReal(const char *name, unsigned sequence, double val)
  1160. {
  1161. Owned<IWUResult> r = updateResult(name, sequence);
  1162. if (r)
  1163. {
  1164. r->setResultReal(val);
  1165. r->setResultStatus(ResultStatusCalculated);
  1166. }
  1167. }
  1168. virtual void setResultVarString(const char * stepname, unsigned sequence, const char *val)
  1169. {
  1170. setResultString(stepname, sequence, strlen(val), val);
  1171. }
  1172. virtual void setResultVarUnicode(const char * stepname, unsigned sequence, UChar const *val)
  1173. {
  1174. setResultUnicode(stepname, sequence, rtlUnicodeStrlen(val), val);
  1175. }
  1176. virtual void setResultString(const char * stepname, unsigned sequence, int len, const char *val)
  1177. {
  1178. doSetResultString(type_string, stepname, sequence, len, val);
  1179. }
  1180. virtual void setResultData(const char * stepname, unsigned sequence, int len, const void *val)
  1181. {
  1182. doSetResultString(type_data, stepname, sequence, len, (const char *)val);
  1183. }
  1184. virtual void setResultRaw(const char * name, unsigned sequence, int len, const void *val)
  1185. {
  1186. Owned<IWUResult> r = updateResult(name, sequence);
  1187. if (r)
  1188. {
  1189. r->setResultRaw(len, val, ResultFormatRaw);
  1190. r->setResultStatus(ResultStatusCalculated);
  1191. }
  1192. }
  1193. virtual void setResultSet(const char * name, unsigned sequence, bool isAll, size32_t len, const void *val, ISetToXmlTransformer *)
  1194. {
  1195. Owned<IWUResult> r = updateResult(name, sequence);
  1196. if (r)
  1197. {
  1198. r->setResultIsAll(isAll);
  1199. r->setResultRaw(len, val, ResultFormatRaw);
  1200. r->setResultStatus(ResultStatusCalculated);
  1201. }
  1202. }
  1203. virtual void setResultUnicode(const char * name, unsigned sequence, int len, UChar const * val)
  1204. {
  1205. Owned<IWUResult> r = updateResult(name, sequence);
  1206. if (r)
  1207. {
  1208. r->setResultUnicode((char const *)val, len);
  1209. r->setResultStatus(ResultStatusCalculated);
  1210. }
  1211. }
  1212. virtual void setResultBool(const char *name, unsigned sequence, bool val)
  1213. {
  1214. Owned<IWUResult> r = updateResult(name, sequence);
  1215. if (r)
  1216. {
  1217. r->setResultBool(val);
  1218. r->setResultStatus(ResultStatusCalculated);
  1219. }
  1220. }
  1221. virtual void setResultDecimal(const char *name, unsigned sequence, int len, int precision, bool isSigned, const void *val)
  1222. {
  1223. Owned<IWUResult> r = updateResult(name, sequence);
  1224. if (r)
  1225. {
  1226. r->setResultDecimal(val, len);
  1227. r->setResultStatus(ResultStatusCalculated);
  1228. }
  1229. }
  1230. virtual void setResultDataset(const char * name, unsigned sequence, size32_t len, const void *val, unsigned numRows, bool extend)
  1231. {
  1232. Owned<IWUResult> r = updateResult(name, sequence);
  1233. if (r)
  1234. {
  1235. __int64 totalRows = numRows;
  1236. if (extend)
  1237. {
  1238. totalRows += r->getResultRowCount();
  1239. r->addResultRaw(len, val, ResultFormatRaw);
  1240. }
  1241. else
  1242. r->setResultRaw(len, val, ResultFormatRaw);
  1243. r->setResultStatus(ResultStatusCalculated);
  1244. r->setResultRowCount(totalRows);
  1245. r->setResultTotalRowCount(totalRows);
  1246. }
  1247. }
  1248. protected:
  1249. IWUResult *updateResult(const char *name, unsigned sequence)
  1250. {
  1251. Owned <IWUResult> result = updateWorkUnitResult(this, name, sequence);
  1252. if (result)
  1253. {
  1254. SCMStringBuffer rname;
  1255. if (!result->getResultName(rname).length())
  1256. result->setResultName(name);
  1257. }
  1258. return result.getClear();
  1259. }
  1260. void doSetResultString(type_t type, const char *name, unsigned sequence, int len, const char *val)
  1261. {
  1262. Owned<IWUResult> r = updateResult(name, sequence);
  1263. if (r)
  1264. {
  1265. r->setResultString(val, len);
  1266. r->setResultStatus(ResultStatusCalculated);
  1267. }
  1268. }
  1269. protected:
  1270. void init();
  1271. IWUGraph *createGraph();
  1272. IWUResult *createResult();
  1273. void loadGraphs() const;
  1274. void loadResults() const;
  1275. void loadTemporaries() const;
  1276. void loadVariables() const;
  1277. void loadExceptions() const;
  1278. void loadPlugins() const;
  1279. void loadLibraries() const;
  1280. void loadClusters() const;
  1281. void unsubscribe();
  1282. void checkAgentRunning(WUState & state);
  1283. void ensureGraphsUnpacked ()
  1284. {
  1285. IPropertyTree *t = p->queryPropTree("PackedGraphs");
  1286. MemoryBuffer buf;
  1287. if (t&&t->getPropBin(NULL,buf)) {
  1288. cachedGraphs.clear();
  1289. IPropertyTree *st = createPTree(buf);
  1290. if (st) {
  1291. p->setPropTree("Graphs",st);
  1292. p->removeTree(t);
  1293. }
  1294. }
  1295. }
  1296. // Implemented by derived classes
  1297. virtual void _lockRemote() {};
  1298. virtual void _unlockRemote() {};
  1299. };
  1300. class CDaliWorkUnit : public CLocalWorkUnit
  1301. {
  1302. public:
  1303. IMPLEMENT_IINTERFACE;
  1304. CDaliWorkUnit(IRemoteConnection *_conn, ISecManager *secmgr, ISecUser *secuser)
  1305. : connection(_conn), CLocalWorkUnit(_conn->getRoot(), secmgr, secuser)
  1306. {
  1307. abortDirty = true;
  1308. abortState = false;
  1309. }
  1310. ~CDaliWorkUnit()
  1311. {
  1312. // NOTE - order is important - we need to construct connection before p and (especially) destroy after p
  1313. // We use the beforeDIspose() in base class to help ensure this
  1314. p.clear();
  1315. }
  1316. virtual void forceReload()
  1317. {
  1318. synchronized sync(locked); // protect locked workunits (uncommited writes) from reload
  1319. StringBuffer wuRoot;
  1320. getXPath(wuRoot, p->queryName());
  1321. IRemoteConnection *newconn = querySDS().connect(wuRoot.str(), myProcessSession(), 0, SDS_LOCK_TIMEOUT);
  1322. if (!newconn)
  1323. throw MakeStringException(WUERR_ConnectFailed, "Could not connect to workunit %s (deleted?)",p->queryName());
  1324. CriticalBlock block(crit);
  1325. connection.setown(newconn);
  1326. init();
  1327. abortDirty = true;
  1328. p.setown(connection->getRoot());
  1329. }
  1330. virtual void cleanupAndDelete(bool deldll, bool deleteOwned, const StringArray *deleteExclusions)
  1331. {
  1332. CLocalWorkUnit::cleanupAndDelete(deldll, deleteOwned, deleteExclusions);
  1333. connection->close(true);
  1334. PROGLOG("WUID %s removed",p->queryName());
  1335. connection.clear();
  1336. }
  1337. virtual void commit()
  1338. {
  1339. CLocalWorkUnit::commit();
  1340. if (connection)
  1341. connection->commit();
  1342. }
  1343. virtual void _lockRemote()
  1344. {
  1345. StringBuffer wuRoot;
  1346. getXPath(wuRoot, p->queryName());
  1347. if (connection)
  1348. connection->changeMode(RTM_LOCK_WRITE,SDS_LOCK_TIMEOUT);
  1349. else
  1350. connection.setown(querySDS().connect(wuRoot.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT));
  1351. if (!connection)
  1352. throw MakeStringException(WUERR_LockFailed, "Failed to get connection for xpath %s", wuRoot.str());
  1353. init();
  1354. abortDirty = true;
  1355. p.setown(connection->getRoot());
  1356. }
  1357. virtual void _unlockRemote()
  1358. {
  1359. try
  1360. {
  1361. //MORE: I'm not convinced this is useful...
  1362. setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTglobal, NULL, StWhenWorkunitModified, NULL, getTimeStampNowValue(), 1, 0, StatsMergeReplace);
  1363. try
  1364. {
  1365. connection->commit();
  1366. }
  1367. catch (IException *e)
  1368. {
  1369. EXCLOG(e, "Error during workunit commit");
  1370. connection->rollback();
  1371. connection->changeMode(0, SDS_LOCK_TIMEOUT);
  1372. throw;
  1373. }
  1374. connection->changeMode(0, SDS_LOCK_TIMEOUT);
  1375. }
  1376. catch (IException *E)
  1377. {
  1378. StringBuffer s;
  1379. PrintLog("Failed to release write lock on workunit: %s", E->errorMessage(s).str());
  1380. throw;
  1381. }
  1382. }
  1383. virtual void subscribe(WUSubscribeOptions options)
  1384. {
  1385. CriticalBlock block(crit);
  1386. assertex(options==SubscribeOptionAbort);
  1387. if (!abortWatcher)
  1388. {
  1389. abortWatcher.setown(new CWorkUnitAbortWatcher(this, p->queryName()));
  1390. abortDirty = true;
  1391. }
  1392. }
  1393. virtual void unsubscribe()
  1394. {
  1395. CriticalBlock block(crit);
  1396. if (abortWatcher)
  1397. {
  1398. abortWatcher->unsubscribe();
  1399. abortWatcher.clear();
  1400. }
  1401. }
  1402. virtual bool aborting() const
  1403. {
  1404. CriticalBlock block(crit);
  1405. if (abortDirty)
  1406. {
  1407. StringBuffer apath;
  1408. apath.append("/WorkUnitAborts/").append(p->queryName());
  1409. Owned<IRemoteConnection> acon = querySDS().connect(apath.str(), myProcessSession(), 0, SDS_LOCK_TIMEOUT);
  1410. if (acon)
  1411. abortState = acon->queryRoot()->getPropInt(NULL) != 0;
  1412. else
  1413. abortState = false;
  1414. abortDirty = false;
  1415. }
  1416. return abortState;
  1417. }
  1418. protected:
  1419. class CWorkUnitAbortWatcher : public CInterface, implements ISDSSubscription
  1420. {
  1421. CDaliWorkUnit *parent; // not linked - it links me
  1422. SubscriptionId abort;
  1423. public:
  1424. IMPLEMENT_IINTERFACE;
  1425. CWorkUnitAbortWatcher(CDaliWorkUnit *_parent, const char *wuid) : parent(_parent)
  1426. {
  1427. StringBuffer wuRoot;
  1428. wuRoot.append("/WorkUnitAborts/").append(wuid);
  1429. abort = querySDS().subscribe(wuRoot.str(), *this);
  1430. }
  1431. ~CWorkUnitAbortWatcher()
  1432. {
  1433. assertex(abort==0);
  1434. }
  1435. void unsubscribe()
  1436. {
  1437. querySDS().unsubscribe(abort);
  1438. abort = 0;
  1439. }
  1440. void notify(SubscriptionId id, const char *xpath, SDSNotifyFlags flags, unsigned valueLen, const void *valueData)
  1441. {
  1442. parent->abortDirty = true;
  1443. }
  1444. };
  1445. Owned<IRemoteConnection> connection;
  1446. Owned<CWorkUnitAbortWatcher> abortWatcher;
  1447. mutable bool abortDirty;
  1448. mutable bool abortState;
  1449. };
  1450. class CLockedWorkUnit : public CInterface, implements ILocalWorkUnit, implements IExtendedWUInterface
  1451. {
  1452. public:
  1453. Owned<CLocalWorkUnit> c;
  1454. IMPLEMENT_IINTERFACE;
  1455. CLockedWorkUnit(CLocalWorkUnit *_c) : c(_c) {}
  1456. ~CLockedWorkUnit()
  1457. {
  1458. if (workUnitTraceLevel > 1)
  1459. PrintLog("Releasing locked workunit %s", queryWuid());
  1460. if (c)
  1461. c->unlockRemote();
  1462. }
  1463. void setSecIfcs(ISecManager *mgr, ISecUser*usr){c->setSecIfcs(mgr, usr);}
  1464. virtual IConstWorkUnit * unlock()
  1465. {
  1466. c->unlockRemote();
  1467. return c.getClear();
  1468. }
  1469. virtual bool aborting() const
  1470. { return c->aborting(); }
  1471. virtual void forceReload()
  1472. { UNIMPLEMENTED; }
  1473. virtual WUAction getAction() const
  1474. { return c->getAction(); }
  1475. virtual IStringVal& getActionEx(IStringVal & str) const
  1476. { return c->getActionEx(str); }
  1477. virtual IStringVal & getApplicationValue(const char * application, const char * propname, IStringVal & str) const
  1478. { return c->getApplicationValue(application, propname, str); }
  1479. virtual int getApplicationValueInt(const char * application, const char * propname, int defVal) const
  1480. { return c->getApplicationValueInt(application, propname, defVal); }
  1481. virtual IConstWUAppValueIterator & getApplicationValues() const
  1482. { return c->getApplicationValues(); }
  1483. virtual bool hasWorkflow() const
  1484. { return c->hasWorkflow(); }
  1485. virtual unsigned queryEventScheduledCount() const
  1486. { return c->queryEventScheduledCount(); }
  1487. virtual IPropertyTree * queryWorkflowTree() const
  1488. { return c->queryWorkflowTree(); }
  1489. virtual IConstWorkflowItemIterator * getWorkflowItems() const
  1490. { return c->getWorkflowItems(); }
  1491. virtual IWorkflowItemArray * getWorkflowClone() const
  1492. { return c->getWorkflowClone(); }
  1493. virtual bool requiresLocalFileUpload() const
  1494. { return c->requiresLocalFileUpload(); }
  1495. virtual IConstLocalFileUploadIterator * getLocalFileUploads() const
  1496. { return c->getLocalFileUploads(); }
  1497. virtual bool getIsQueryService() const
  1498. { return c->getIsQueryService(); }
  1499. virtual bool getCloneable() const
  1500. { return c->getCloneable(); }
  1501. virtual IUserDescriptor * queryUserDescriptor() const
  1502. { return c->queryUserDescriptor(); }
  1503. virtual const char *queryClusterName() const
  1504. { return c->queryClusterName(); }
  1505. virtual unsigned getCodeVersion() const
  1506. { return c->getCodeVersion(); }
  1507. virtual unsigned getWuidVersion() const
  1508. { return c->getWuidVersion(); }
  1509. virtual void getBuildVersion(IStringVal & buildVersion, IStringVal & eclVersion) const
  1510. { c->getBuildVersion(buildVersion, eclVersion); }
  1511. virtual bool hasDebugValue(const char * propname) const
  1512. { return c->hasDebugValue(propname); }
  1513. virtual IStringVal & getDebugValue(const char * propname, IStringVal & str) const
  1514. { return c->getDebugValue(propname, str); }
  1515. virtual int getDebugValueInt(const char * propname, int defVal) const
  1516. { return c->getDebugValueInt(propname, defVal); }
  1517. virtual __int64 getDebugValueInt64(const char * propname, __int64 defVal) const
  1518. { return c->getDebugValueInt64(propname, defVal); }
  1519. virtual bool getDebugValueBool(const char * propname, bool defVal) const
  1520. { return c->getDebugValueBool(propname, defVal); }
  1521. virtual IStringIterator & getDebugValues() const
  1522. { return c->getDebugValues(NULL); }
  1523. virtual IStringIterator & getDebugValues(const char *prop) const
  1524. { return c->getDebugValues(prop); }
  1525. virtual unsigned getExceptionCount() const
  1526. { return c->getExceptionCount(); }
  1527. virtual IConstWUExceptionIterator & getExceptions() const
  1528. { return c->getExceptions(); }
  1529. virtual unsigned getGraphCount() const
  1530. { return c->getGraphCount(); }
  1531. virtual unsigned getSourceFileCount() const
  1532. { return c->getSourceFileCount(); }
  1533. virtual unsigned getResultCount() const
  1534. { return c->getResultCount(); }
  1535. virtual unsigned getVariableCount() const
  1536. { return c->getVariableCount(); }
  1537. virtual unsigned getApplicationValueCount() const
  1538. { return c->getApplicationValueCount(); }
  1539. virtual IConstWUGraphIterator & getGraphs(WUGraphType type) const
  1540. { return c->getGraphs(type); }
  1541. virtual IConstWUGraphMetaIterator & getGraphsMeta(WUGraphType type) const
  1542. { return c->getGraphsMeta(type); }
  1543. virtual IConstWUGraph * getGraph(const char *name) const
  1544. { return c->getGraph(name); }
  1545. virtual IConstWUGraphProgress * getGraphProgress(const char * name) const
  1546. { return c->getGraphProgress(name); }
  1547. virtual const char *queryJobName() const
  1548. { return c->queryJobName(); }
  1549. virtual IConstWUPlugin * getPluginByName(const char * name) const
  1550. { return c->getPluginByName(name); }
  1551. virtual IConstWUPluginIterator & getPlugins() const
  1552. { return c->getPlugins(); }
  1553. virtual IConstWULibrary* getLibraryByName(const char *name) const
  1554. { return c->getLibraryByName(name); }
  1555. virtual IConstWULibraryIterator & getLibraries() const
  1556. { return c->getLibraries(); }
  1557. virtual WUPriorityClass getPriority() const
  1558. { return c->getPriority(); }
  1559. virtual int getPriorityLevel() const
  1560. { return c->getPriorityLevel(); }
  1561. virtual int getPriorityValue() const
  1562. { return c->getPriorityValue(); }
  1563. virtual IConstWUQuery * getQuery() const
  1564. { return c->getQuery(); }
  1565. virtual IConstWUWebServicesInfo * getWebServicesInfo() const
  1566. { return c->getWebServicesInfo(); }
  1567. virtual IConstWURoxieQueryInfo* getRoxieQueryInfo() const
  1568. { return c->getRoxieQueryInfo(); }
  1569. virtual bool getRescheduleFlag() const
  1570. { return c->getRescheduleFlag(); }
  1571. virtual IConstWUResult * getResultByName(const char * name) const
  1572. { return c->getResultByName(name); }
  1573. virtual IConstWUResult * getResultBySequence(unsigned seq) const
  1574. { return c->getResultBySequence(seq); }
  1575. virtual unsigned getResultLimit() const
  1576. { return c->getResultLimit(); }
  1577. virtual IConstWUResultIterator & getResults() const
  1578. { return c->getResults(); }
  1579. virtual IStringVal & getScope(IStringVal & str) const
  1580. { return c->getScope(str); }
  1581. virtual IStringVal & getSecurityToken(IStringVal & str) const
  1582. { return c->getSecurityToken(str); }
  1583. virtual WUState getState() const
  1584. { return c->getState(); }
  1585. virtual IStringVal & getStateEx(IStringVal & str) const
  1586. { return c->getStateEx(str); }
  1587. virtual __int64 getAgentSession() const
  1588. { return c->getAgentSession(); }
  1589. virtual unsigned getAgentPID() const
  1590. { return c->getAgentPID(); }
  1591. virtual const char *queryStateDesc() const
  1592. { return c->queryStateDesc(); }
  1593. virtual bool getRunningGraph(IStringVal & graphName, WUGraphIDType & subId) const
  1594. { return c->getRunningGraph(graphName, subId); }
  1595. virtual IConstWUStatisticIterator & getStatistics(const IStatisticsFilter * filter) const
  1596. { return c->getStatistics(filter); }
  1597. virtual IConstWUStatistic * getStatistic(const char * creator, const char * scope, StatisticKind kind) const
  1598. { return c->getStatistic(creator, scope, kind); }
  1599. virtual bool getWuDate(unsigned & year, unsigned & month, unsigned& day)
  1600. { return c->getWuDate(year,month,day);}
  1601. virtual IStringVal & getSnapshot(IStringVal & str) const
  1602. { return c->getSnapshot(str); }
  1603. virtual const char *queryUser() const
  1604. { return c->queryUser(); }
  1605. virtual IStringVal & getWuScope(IStringVal & str) const
  1606. { return c->getWuScope(str); }
  1607. virtual const char *queryWuid() const
  1608. { return c->queryWuid(); }
  1609. virtual IConstWUResult * getGlobalByName(const char * name) const
  1610. { return c->getGlobalByName(name); }
  1611. virtual IConstWUResult * getTemporaryByName(const char * name) const
  1612. { return c->getTemporaryByName(name); }
  1613. virtual IConstWUResultIterator & getTemporaries() const
  1614. { return c->getTemporaries(); }
  1615. virtual IConstWUResult * getVariableByName(const char * name) const
  1616. { return c->getVariableByName(name); }
  1617. virtual IConstWUResultIterator & getVariables() const
  1618. { return c->getVariables(); }
  1619. virtual bool isProtected() const
  1620. { return c->isProtected(); }
  1621. virtual bool isPausing() const
  1622. { return c->isPausing(); }
  1623. virtual IWorkUnit & lock()
  1624. { ((CInterface *)this)->Link(); return (IWorkUnit &) *this; }
  1625. virtual bool reload()
  1626. { UNIMPLEMENTED; }
  1627. virtual void subscribe(WUSubscribeOptions options)
  1628. { c->subscribe(options); }
  1629. virtual void requestAbort()
  1630. { c->requestAbort(); }
  1631. virtual unsigned calculateHash(unsigned prevHash)
  1632. { return queryExtendedWU(c)->calculateHash(prevHash); }
  1633. virtual void copyWorkUnit(IConstWorkUnit *cached, bool all)
  1634. { queryExtendedWU(c)->copyWorkUnit(cached, all); }
  1635. virtual bool archiveWorkUnit(const char *base,bool del,bool deldll,bool deleteOwned)
  1636. { return queryExtendedWU(c)->archiveWorkUnit(base,del,deldll,deleteOwned); }
  1637. virtual void packWorkUnit(bool pack)
  1638. { queryExtendedWU(c)->packWorkUnit(pack); }
  1639. virtual unsigned queryFileUsage(const char *filename) const
  1640. { return c->queryFileUsage(filename); }
  1641. virtual IJlibDateTime & getTimeScheduled(IJlibDateTime &val) const
  1642. { return c->getTimeScheduled(val); }
  1643. virtual unsigned getDebugAgentListenerPort() const
  1644. { return c->getDebugAgentListenerPort(); }
  1645. virtual IStringVal & getDebugAgentListenerIP(IStringVal &ip) const
  1646. { return c->getDebugAgentListenerIP(ip); }
  1647. virtual IStringVal & getXmlParams(IStringVal & params) const
  1648. { return c->getXmlParams(params); }
  1649. virtual const IPropertyTree *getXmlParams() const
  1650. { return c->getXmlParams(); }
  1651. virtual unsigned __int64 getHash() const
  1652. { return c->getHash(); }
  1653. virtual IStringIterator *getLogs(const char *type, const char *instance) const
  1654. { return c->getLogs(type, instance); }
  1655. virtual IStringIterator *getProcesses(const char *type) const
  1656. { return c->getProcesses(type); }
  1657. virtual IPropertyTreeIterator* getProcesses(const char *type, const char *instance) const
  1658. { return c->getProcesses(type, instance); }
  1659. virtual void clearExceptions()
  1660. { c->clearExceptions(); }
  1661. virtual void commit()
  1662. { c->commit(); }
  1663. virtual IWUException * createException()
  1664. { return c->createException(); }
  1665. virtual void addProcess(const char *type, const char *instance, unsigned pid, const char *log)
  1666. { c->addProcess(type, instance, pid, log); }
  1667. virtual void protect(bool protectMode)
  1668. { c->protect(protectMode); }
  1669. virtual void setAction(WUAction action)
  1670. { c->setAction(action); }
  1671. virtual void setApplicationValue(const char * application, const char * propname, const char * value, bool overwrite)
  1672. { c->setApplicationValue(application, propname, value, overwrite); }
  1673. virtual void setApplicationValueInt(const char * application, const char * propname, int value, bool overwrite)
  1674. { c->setApplicationValueInt(application, propname, value, overwrite); }
  1675. virtual void incEventScheduledCount()
  1676. { c->incEventScheduledCount(); }
  1677. virtual void setIsQueryService(bool value)
  1678. { c->setIsQueryService(value); }
  1679. virtual void setCloneable(bool value)
  1680. { c->setCloneable(value); }
  1681. virtual void setIsClone(bool value)
  1682. { c->setIsClone(value); }
  1683. virtual void setClusterName(const char * value)
  1684. { c->setClusterName(value); }
  1685. virtual void setCodeVersion(unsigned version, const char * buildVersion, const char * eclVersion)
  1686. { c->setCodeVersion(version, buildVersion, eclVersion); }
  1687. virtual void setDebugValue(const char * propname, const char * value, bool overwrite)
  1688. { c->setDebugValue(propname, value, overwrite); }
  1689. virtual void setDebugValueInt(const char * propname, int value, bool overwrite)
  1690. { c->setDebugValueInt(propname, value, overwrite); }
  1691. virtual void setJobName(const char * value)
  1692. { c->setJobName(value); }
  1693. virtual void setPriority(WUPriorityClass cls)
  1694. { c->setPriority(cls); }
  1695. virtual void setPriorityLevel(int level)
  1696. { c->setPriorityLevel(level); }
  1697. virtual void setRescheduleFlag(bool value)
  1698. { c->setRescheduleFlag(value); }
  1699. virtual void setResultLimit(unsigned value)
  1700. { c->setResultLimit(value); }
  1701. virtual void setSecurityToken(const char *value)
  1702. { c->setSecurityToken(value); }
  1703. virtual void setState(WUState state)
  1704. { c->setState(state); }
  1705. virtual void setStateEx(const char * text)
  1706. { c->setStateEx(text); }
  1707. virtual void setAgentSession(__int64 sessionId)
  1708. { c->setAgentSession(sessionId); }
  1709. virtual void setStatistic(StatisticCreatorType creatorType, const char * creator, StatisticScopeType scopeType, const char * scope, StatisticKind kind, const char * optDescription, unsigned __int64 value, unsigned __int64 count, unsigned __int64 maxValue, StatsMergeAction mergeAction)
  1710. { c->setStatistic(creatorType, creator, scopeType, scope, kind, optDescription, value, count, maxValue, mergeAction); }
  1711. virtual void setTracingValue(const char * propname, const char * value)
  1712. { c->setTracingValue(propname, value); }
  1713. virtual void setTracingValueInt(const char * propname, int value)
  1714. { c->setTracingValueInt(propname, value); }
  1715. virtual void setUser(const char * value)
  1716. { c->setUser(value); }
  1717. virtual void setWuScope(const char * value)
  1718. {
  1719. if (value && *value)
  1720. {
  1721. ISecManager *secmgr=c->querySecMgr();
  1722. ISecUser *secusr=c->querySecUser();
  1723. if (!secmgr || !secusr)
  1724. throw MakeStringException(WUERR_SecurityNotAvailable, "Trying to change workunit scope without security interfaces available");
  1725. if (checkWuScopeSecAccess(value, *secmgr, secusr, SecAccess_Write, "Change Scope", true, true))
  1726. c->setWuScope(value);
  1727. }
  1728. }
  1729. virtual IWorkflowItem* addWorkflowItem(unsigned wfid, WFType type, WFMode mode, unsigned success, unsigned failure, unsigned recovery, unsigned retriesAllowed, unsigned contingencyFor)
  1730. { return c->addWorkflowItem(wfid, type, mode, success, failure, recovery, retriesAllowed, contingencyFor); }
  1731. virtual void syncRuntimeWorkflow(IWorkflowItemArray * array)
  1732. { c->syncRuntimeWorkflow(array); }
  1733. virtual IWorkflowItemIterator * updateWorkflowItems()
  1734. { return c->updateWorkflowItems(); }
  1735. virtual void resetWorkflow()
  1736. { c->resetWorkflow(); }
  1737. virtual void schedule()
  1738. { c->schedule(); }
  1739. virtual void deschedule()
  1740. { c->deschedule(); }
  1741. virtual unsigned addLocalFileUpload(LocalFileUploadType type, char const * source, char const * destination, char const * eventTag)
  1742. { return c->addLocalFileUpload(type, source, destination, eventTag); }
  1743. virtual IWUResult * updateGlobalByName(const char * name)
  1744. { return c->updateGlobalByName(name); }
  1745. virtual IWUGraph * createGraph(const char * name, WUGraphType type, IPropertyTree *xgmml)
  1746. { return c->createGraph(name, type, xgmml); }
  1747. virtual IWUGraph * updateGraph(const char * name)
  1748. { return c->updateGraph(name); }
  1749. virtual IWUQuery * updateQuery()
  1750. { return c->updateQuery(); }
  1751. virtual IWUWebServicesInfo * updateWebServicesInfo(bool create)
  1752. { return c->updateWebServicesInfo(create); }
  1753. virtual IWURoxieQueryInfo * updateRoxieQueryInfo(const char *wuid, const char *roxieClusterName)
  1754. { return c->updateRoxieQueryInfo(wuid, roxieClusterName); }
  1755. virtual IWUPlugin * updatePluginByName(const char * name)
  1756. { return c->updatePluginByName(name); }
  1757. virtual IWULibrary * updateLibraryByName(const char * name)
  1758. { return c->updateLibraryByName(name); }
  1759. virtual IWUResult * updateResultByName(const char * name)
  1760. { return c->updateResultByName(name); }
  1761. virtual IWUResult * updateResultBySequence(unsigned seq)
  1762. { return c->updateResultBySequence(seq); }
  1763. virtual IWUResult * updateTemporaryByName(const char * name)
  1764. { return c->updateTemporaryByName(name); }
  1765. virtual IWUResult * updateVariableByName(const char * name)
  1766. { return c->updateVariableByName(name); }
  1767. virtual void addFile(const char *fileName, StringArray *clusters, unsigned usageCount, WUFileKind fileKind, const char *graphOwner)
  1768. { c->addFile(fileName, clusters, usageCount, fileKind, graphOwner); }
  1769. virtual void noteFileRead(IDistributedFile *file)
  1770. { c->noteFileRead(file); }
  1771. virtual void releaseFile(const char *fileName)
  1772. { c->releaseFile(fileName); }
  1773. virtual void clearGraphProgress()
  1774. { c->clearGraphProgress(); }
  1775. virtual void resetBeforeGeneration()
  1776. { c->resetBeforeGeneration(); }
  1777. virtual void deleteTempFiles(const char *graph, bool deleteOwned, bool deleteJobOwned)
  1778. { c->deleteTempFiles(graph, deleteOwned, deleteJobOwned); }
  1779. virtual void deleteTemporaries()
  1780. { c->deleteTemporaries(); }
  1781. virtual void addDiskUsageStats(__int64 avgNodeUsage, unsigned minNode, __int64 minNodeUsage, unsigned maxNode, __int64 maxNodeUsage, __int64 graphId)
  1782. { c->addDiskUsageStats(avgNodeUsage, minNode, minNodeUsage, maxNode, maxNodeUsage, graphId); }
  1783. virtual IPropertyTree * getDiskUsageStats()
  1784. { return c->getDiskUsageStats(); }
  1785. virtual IPropertyTreeIterator & getFileIterator() const
  1786. { return c->getFileIterator(); }
  1787. virtual IPropertyTreeIterator & getFilesReadIterator() const
  1788. { return c->getFilesReadIterator(); }
  1789. virtual void setSnapshot(const char * value)
  1790. { c->setSnapshot(value); }
  1791. virtual void setTimeScheduled(const IJlibDateTime &val)
  1792. { c->setTimeScheduled(val); }
  1793. virtual void setDebugAgentListenerPort(unsigned port)
  1794. { c->setDebugAgentListenerPort(port); }
  1795. virtual void setDebugAgentListenerIP(const char * ip)
  1796. { c->setDebugAgentListenerIP(ip); }
  1797. virtual void setXmlParams(const char *params)
  1798. { c->setXmlParams(params); }
  1799. virtual void setXmlParams(IPropertyTree *tree)
  1800. { c->setXmlParams(tree); }
  1801. virtual void setHash(unsigned __int64 hash)
  1802. { c->setHash(hash); }
  1803. // ILocalWorkUnit - used for debugging etc
  1804. virtual void serialize(MemoryBuffer &tgt)
  1805. { c->serialize(tgt); }
  1806. virtual void deserialize(MemoryBuffer &src)
  1807. { c->deserialize(src); }
  1808. virtual bool switchThorQueue(const char *cluster, IQueueSwitcher *qs)
  1809. { return c->switchThorQueue(cluster,qs); }
  1810. virtual void setAllowedClusters(const char *value)
  1811. { c->setAllowedClusters(value); }
  1812. virtual IStringVal& getAllowedClusters(IStringVal &str) const
  1813. { return c->getAllowedClusters(str); }
  1814. virtual void remoteCheckAccess(IUserDescriptor *user, bool writeaccess) const
  1815. { c->remoteCheckAccess(user,writeaccess); }
  1816. virtual void setAllowAutoQueueSwitch(bool val)
  1817. { c->setAllowAutoQueueSwitch(val); }
  1818. virtual bool getAllowAutoQueueSwitch() const
  1819. { return c->getAllowAutoQueueSwitch(); }
  1820. virtual void setLibraryInformation(const char * name, unsigned interfaceHash, unsigned definitionHash)
  1821. { c->setLibraryInformation(name, interfaceHash, definitionHash); }
  1822. virtual void setResultInt(const char * name, unsigned sequence, __int64 val)
  1823. { c->setResultInt(name, sequence, val); }
  1824. virtual void setResultUInt(const char * name, unsigned sequence, unsigned __int64 val)
  1825. { c->setResultUInt(name, sequence, val); }
  1826. virtual void setResultReal(const char *name, unsigned sequence, double val)
  1827. { c->setResultReal(name, sequence, val); }
  1828. virtual void setResultVarString(const char * stepname, unsigned sequence, const char *val)
  1829. { c->setResultVarString(stepname, sequence, val); }
  1830. virtual void setResultVarUnicode(const char * stepname, unsigned sequence, UChar const *val)
  1831. { c->setResultVarUnicode(stepname, sequence, val); }
  1832. virtual void setResultString(const char * stepname, unsigned sequence, int len, const char *val)
  1833. { c->setResultString(stepname, sequence, len, val); }
  1834. virtual void setResultData(const char * stepname, unsigned sequence, int len, const void *val)
  1835. { c->setResultData(stepname, sequence, len, val); }
  1836. virtual void setResultRaw(const char * name, unsigned sequence, int len, const void *val)
  1837. { c->setResultRaw(name, sequence, len, val); }
  1838. virtual void setResultSet(const char * name, unsigned sequence, bool isAll, size32_t len, const void *val, ISetToXmlTransformer *xform)
  1839. { c->setResultSet(name, sequence, isAll, len, val, xform); }
  1840. virtual void setResultUnicode(const char * name, unsigned sequence, int len, UChar const * val)
  1841. { c->setResultUnicode(name, sequence, len, val); }
  1842. virtual void setResultBool(const char *name, unsigned sequence, bool val)
  1843. { c->setResultBool(name, sequence, val); }
  1844. virtual void setResultDecimal(const char *name, unsigned sequence, int len, int precision, bool isSigned, const void *val)
  1845. { c->setResultDecimal(name, sequence, len, precision, isSigned, val); }
  1846. virtual void setResultDataset(const char * name, unsigned sequence, size32_t len, const void *val, unsigned numRows, bool extend)
  1847. { c->setResultDataset(name, sequence, len, val, numRows, extend); }
  1848. };
  1849. class CLocalWUAssociated : public CInterface, implements IConstWUAssociatedFile
  1850. {
  1851. Owned<IPropertyTree> p;
  1852. public:
  1853. IMPLEMENT_IINTERFACE;
  1854. CLocalWUAssociated(IPropertyTree *p);
  1855. virtual WUFileType getType() const;
  1856. virtual IStringVal & getDescription(IStringVal & ret) const;
  1857. virtual IStringVal & getIp(IStringVal & ret) const;
  1858. virtual IStringVal & getName(IStringVal & ret) const;
  1859. virtual IStringVal & getNameTail(IStringVal & ret) const;
  1860. virtual unsigned getCrc() const;
  1861. };
  1862. class CLocalWUQuery : public CInterface, implements IWUQuery
  1863. {
  1864. Owned<IPropertyTree> p;
  1865. mutable IArrayOf<IConstWUAssociatedFile> associated;
  1866. mutable CriticalSection crit;
  1867. mutable bool associatedCached;
  1868. private:
  1869. void addSpecialCaseAssociated(WUFileType type, const char * propname, unsigned crc) const;
  1870. void loadAssociated() const;
  1871. public:
  1872. IMPLEMENT_IINTERFACE;
  1873. CLocalWUQuery(IPropertyTree *p);
  1874. virtual WUQueryType getQueryType() const;
  1875. virtual IStringVal& getQueryText(IStringVal &str) const;
  1876. virtual IStringVal& getQueryShortText(IStringVal &str) const;
  1877. virtual IStringVal& getQueryName(IStringVal &str) const;
  1878. virtual IStringVal & getQueryMainDefinition(IStringVal & str) const;
  1879. virtual IStringVal& getQueryDllName(IStringVal &str) const;
  1880. virtual unsigned getQueryDllCrc() const;
  1881. virtual IStringVal& getQueryCppName(IStringVal &str) const;
  1882. virtual IStringVal& getQueryResTxtName(IStringVal &str) const;
  1883. virtual IConstWUAssociatedFile * getAssociatedFile(WUFileType type, unsigned index) const;
  1884. virtual IConstWUAssociatedFileIterator& getAssociatedFiles() const;
  1885. virtual void setQueryType(WUQueryType qt);
  1886. virtual void setQueryText(const char *pstr);
  1887. virtual void setQueryName(const char *);
  1888. virtual void setQueryMainDefinition(const char * str);
  1889. virtual void addAssociatedFile(WUFileType type, const char * name, const char * ip, const char * desc, unsigned crc);
  1890. virtual void removeAssociatedFiles();
  1891. };
  1892. class CLocalWUWebServicesInfo : public CInterface, implements IWUWebServicesInfo
  1893. {
  1894. Owned<IPropertyTree> p;
  1895. mutable CriticalSection crit;
  1896. private:
  1897. public:
  1898. IMPLEMENT_IINTERFACE;
  1899. CLocalWUWebServicesInfo(IPropertyTree *p);
  1900. virtual IStringVal& getModuleName(IStringVal &str) const;
  1901. virtual IStringVal& getAttributeName(IStringVal &str) const;
  1902. virtual IStringVal& getDefaultName(IStringVal &str) const;
  1903. virtual IStringVal& getInfo(const char *name, IStringVal &str) const;
  1904. virtual unsigned getWebServicesCRC() const;
  1905. virtual void setModuleName(const char *);
  1906. virtual void setAttributeName(const char *);
  1907. virtual void setDefaultName(const char *);
  1908. virtual void setInfo(const char *name, const char *info);
  1909. virtual void setWebServicesCRC(unsigned);
  1910. };
  1911. class CLocalWURoxieQueryInfo : public CInterface, implements IWURoxieQueryInfo
  1912. {
  1913. Owned<IPropertyTree> p;
  1914. mutable CriticalSection crit;
  1915. private:
  1916. public:
  1917. IMPLEMENT_IINTERFACE;
  1918. CLocalWURoxieQueryInfo(IPropertyTree *p);
  1919. virtual IStringVal& getQueryInfo(IStringVal &str) const;
  1920. virtual IStringVal& getDefaultPackageInfo(IStringVal &str) const;
  1921. virtual IStringVal& getRoxieClusterName(IStringVal &str) const;
  1922. virtual IStringVal& getWuid(IStringVal &str) const;
  1923. virtual void setQueryInfo(const char *info);
  1924. virtual void setDefaultPackageInfo(const char *, int len);
  1925. virtual void setRoxieClusterName(const char *str);
  1926. virtual void setWuid(const char *str);
  1927. };
  1928. class CLocalWUResult : public CInterface, implements IWUResult
  1929. {
  1930. friend class CLocalWorkUnit;
  1931. mutable CriticalSection crit;
  1932. Owned<IPropertyTree> p;
  1933. Owned<IProperties> xmlns;
  1934. void getSchema(TypeInfoArray &types, StringAttrArray &names, IStringVal * ecl=NULL) const;
  1935. public:
  1936. IMPLEMENT_IINTERFACE;
  1937. CLocalWUResult(IPropertyTree *props);
  1938. ~CLocalWUResult() { try { p.clear(); } catch (IException *E) {E->Release();}}
  1939. virtual WUResultStatus getResultStatus() const;
  1940. virtual IStringVal& getResultName(IStringVal &str) const;
  1941. virtual int getResultSequence() const;
  1942. virtual bool isResultScalar() const;
  1943. virtual IStringVal& getResultXml(IStringVal &str) const;
  1944. virtual unsigned getResultFetchSize() const;
  1945. virtual __int64 getResultTotalRowCount() const;
  1946. virtual __int64 getResultRowCount() const;
  1947. virtual void getResultDataset(IStringVal & ecl, IStringVal & defs) const;
  1948. virtual IStringVal& getResultLogicalName(IStringVal &ecl) const;
  1949. virtual IStringVal& getResultKeyField(IStringVal& ecl) const;
  1950. virtual unsigned getResultRequestedRows() const;
  1951. virtual __int64 getResultInt() const;
  1952. virtual bool getResultBool() const;
  1953. virtual double getResultReal() const;
  1954. virtual IStringVal& getResultString(IStringVal & str) const;
  1955. virtual IDataVal& getResultRaw(IDataVal & data, IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const;
  1956. virtual IDataVal& getResultUnicode(IDataVal & data) const;
  1957. virtual void getResultDecimal(void * val, unsigned length, unsigned precision, bool isSigned) const;
  1958. virtual IStringVal& getResultEclSchema(IStringVal & str) const;
  1959. virtual __int64 getResultRawSize(IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const;
  1960. virtual IDataVal& getResultRaw(IDataVal & data, __int64 from, __int64 length, IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const;
  1961. virtual IStringVal& getResultRecordSizeEntry(IStringVal & str) const;
  1962. virtual IStringVal& getResultTransformerEntry(IStringVal & str) const;
  1963. virtual __int64 getResultRowLimit() const;
  1964. virtual IStringVal& getResultFilename(IStringVal & str) const;
  1965. virtual WUResultFormat getResultFormat() const;
  1966. virtual unsigned getResultHash() const;
  1967. virtual bool getResultIsAll() const;
  1968. virtual IProperties *queryResultXmlns();
  1969. // interface IWUResult
  1970. virtual void setResultStatus(WUResultStatus status);
  1971. virtual void setResultName(const char *name);
  1972. virtual void setResultSequence(unsigned seq);
  1973. virtual void setResultSchemaRaw(unsigned len, const void *schema);
  1974. virtual void setResultScalar(bool isScalar);
  1975. virtual void setResultRaw(unsigned len, const void *xml, WUResultFormat format);
  1976. virtual void setResultFetchSize(unsigned rows); // 0 means file-loaded
  1977. virtual void setResultTotalRowCount(__int64 rows); // -1 means unknown
  1978. virtual void setResultRowCount(__int64 rows);
  1979. virtual void setResultDataset(const char *ecl, const char *defs);
  1980. virtual void setResultLogicalName(const char *logicalName);
  1981. virtual void setResultKeyField(const char * name);
  1982. virtual void setResultRequestedRows(unsigned req);
  1983. virtual void setResultRecordSizeEntry(const char * val);
  1984. virtual void setResultTransformerEntry(const char * val);
  1985. virtual void setResultInt(__int64 val);
  1986. virtual void setResultReal(double val);
  1987. virtual void setResultBool(bool val);
  1988. virtual void setResultString(const char * val, unsigned length);
  1989. virtual void setResultUnicode(const void * val, unsigned length);
  1990. virtual void setResultData(const void * val, unsigned length);
  1991. virtual void setResultDecimal(const void * val, unsigned length);
  1992. virtual void addResultRaw(unsigned len, const void * data, WUResultFormat format);
  1993. virtual void setResultRowLimit(__int64 value);
  1994. virtual void setResultFilename(const char * name);
  1995. virtual void setResultUInt(unsigned __int64 val);
  1996. virtual void setResultIsAll(bool value);
  1997. virtual void setResultFormat(WUResultFormat format);
  1998. virtual void setResultXML(const char *val);
  1999. virtual void setResultRow(unsigned len, const void * data);
  2000. virtual void setResultXmlns(const char *prefix, const char *uri);
  2001. };
  2002. class CLocalWUPlugin : public CInterface, implements IWUPlugin
  2003. {
  2004. Owned<IPropertyTree> p;
  2005. public:
  2006. IMPLEMENT_IINTERFACE;
  2007. CLocalWUPlugin(IPropertyTree *p);
  2008. virtual IStringVal& getPluginName(IStringVal &str) const;
  2009. virtual IStringVal& getPluginVersion(IStringVal &str) const;
  2010. virtual void setPluginName(const char *str);
  2011. virtual void setPluginVersion(const char *str);
  2012. };
  2013. class CLocalWULibrary : public CInterface, implements IWULibrary
  2014. {
  2015. Owned<IPropertyTree> p;
  2016. public:
  2017. IMPLEMENT_IINTERFACE;
  2018. CLocalWULibrary(IPropertyTree *p);
  2019. virtual IStringVal & getName(IStringVal & str) const;
  2020. virtual void setName(const char * str);
  2021. };
  2022. class CLocalWUGraph : public CInterface, implements IWUGraph
  2023. {
  2024. const CLocalWorkUnit &owner;
  2025. Owned<IPropertyTree> p;
  2026. mutable Owned<IPropertyTree> graph; // cached copy of graph xgmml
  2027. mutable Linked<IConstWUGraphProgress> progress;
  2028. unsigned wuidVersion;
  2029. void mergeProgress(IPropertyTree &tree, IPropertyTree &progressTree, const unsigned &progressV) const;
  2030. public:
  2031. IMPLEMENT_IINTERFACE;
  2032. CLocalWUGraph(const CLocalWorkUnit &owner, IPropertyTree *p);
  2033. virtual IStringVal & getXGMML(IStringVal & ret, bool mergeProgress) const;
  2034. virtual IStringVal & getName(IStringVal & ret) const;
  2035. virtual IStringVal & getLabel(IStringVal & ret) const;
  2036. virtual IStringVal & getTypeName(IStringVal & ret) const;
  2037. virtual WUGraphType getType() const;
  2038. virtual WUGraphState getState() const;
  2039. virtual IPropertyTree * getXGMMLTree(bool mergeProgress) const;
  2040. virtual IPropertyTree * getXGMMLTreeRaw() const;
  2041. virtual void setName(const char *str);
  2042. virtual void setLabel(const char *str);
  2043. virtual void setType(WUGraphType type);
  2044. virtual void setXGMML(const char *str);
  2045. virtual void setXGMMLTree(IPropertyTree * tree, bool compress=true);
  2046. };
  2047. class CLocalWUException : public CInterface, implements IWUException
  2048. {
  2049. Owned<IPropertyTree> p;
  2050. public:
  2051. IMPLEMENT_IINTERFACE;
  2052. CLocalWUException(IPropertyTree *p);
  2053. virtual IStringVal& getExceptionSource(IStringVal &str) const;
  2054. virtual IStringVal& getExceptionMessage(IStringVal &str) const;
  2055. virtual unsigned getExceptionCode() const;
  2056. virtual WUExceptionSeverity getSeverity() const;
  2057. virtual IStringVal & getTimeStamp(IStringVal & dt) const;
  2058. virtual IStringVal & getExceptionFileName(IStringVal & str) const;
  2059. virtual unsigned getExceptionLineNo() const;
  2060. virtual unsigned getExceptionColumn() const;
  2061. virtual void setExceptionSource(const char *str);
  2062. virtual void setExceptionMessage(const char *str);
  2063. virtual void setExceptionCode(unsigned code);
  2064. virtual void setSeverity(WUExceptionSeverity level);
  2065. virtual void setTimeStamp(const char * dt);
  2066. virtual void setExceptionFileName(const char *str);
  2067. virtual void setExceptionLineNo(unsigned r);
  2068. virtual void setExceptionColumn(unsigned c);
  2069. };
  2070. //==========================================================================================
  2071. extern WORKUNIT_API bool isSpecialResultSequence(unsigned sequence)
  2072. {
  2073. switch (sequence)
  2074. {
  2075. case ResultSequenceInternal:
  2076. case ResultSequenceOnce:
  2077. case ResultSequencePersist:
  2078. case ResultSequenceStored:
  2079. return true;
  2080. default:
  2081. assertex(sequence <= INT_MAX);
  2082. return false;
  2083. }
  2084. }
  2085. struct mapEnums { int val; const char *str; };
  2086. const char *getEnumText(int value, mapEnums *map)
  2087. {
  2088. const char *defval = map->str;
  2089. while (map->str)
  2090. {
  2091. if (value==map->val)
  2092. return map->str;
  2093. map++;
  2094. }
  2095. assertex(!"Unexpected value in setEnum");
  2096. return defval;
  2097. }
  2098. void setEnum(IPropertyTree *p, const char *propname, int value, mapEnums *map)
  2099. {
  2100. const char *defval = map->str;
  2101. while (map->str)
  2102. {
  2103. if (value==map->val)
  2104. {
  2105. p->setProp(propname, map->str);
  2106. return;
  2107. }
  2108. map++;
  2109. }
  2110. assertex(!"Unexpected value in setEnum");
  2111. p->setProp(propname, defval);
  2112. }
  2113. static int getEnum(const char *v, mapEnums *map)
  2114. {
  2115. if (v)
  2116. {
  2117. while (map->str)
  2118. {
  2119. if (stricmp(v, map->str)==0)
  2120. return map->val;
  2121. map++;
  2122. }
  2123. assertex(!"Unexpected value in getEnum");
  2124. }
  2125. return 0;
  2126. }
  2127. static int getEnum(const IPropertyTree *p, const char *propname, mapEnums *map)
  2128. {
  2129. return getEnum(p->queryProp(propname),map);
  2130. }
  2131. //==========================================================================================
  2132. class CConstWUArrayIterator : public CInterface, implements IConstWorkUnitIterator
  2133. {
  2134. IArrayOf<IPropertyTree> trees;
  2135. Owned<IConstWorkUnit> cur;
  2136. unsigned curTreeNum;
  2137. Linked<IWorkUnitFactory> factory;
  2138. void setCurrent()
  2139. {
  2140. cur.setown(factory->openWorkUnit(trees.item(curTreeNum).queryName(), false));
  2141. }
  2142. public:
  2143. IMPLEMENT_IINTERFACE;
  2144. CConstWUArrayIterator(IWorkUnitFactory *_factory, IArrayOf<IPropertyTree> &_trees)
  2145. : factory(_factory)
  2146. {
  2147. ForEachItemIn(t, _trees)
  2148. trees.append(*LINK(&_trees.item(t)));
  2149. curTreeNum = 0;
  2150. }
  2151. bool first()
  2152. {
  2153. curTreeNum = 0;
  2154. return next();
  2155. }
  2156. bool isValid()
  2157. {
  2158. return (NULL != cur.get());
  2159. }
  2160. bool next()
  2161. {
  2162. if (curTreeNum >= trees.ordinality())
  2163. {
  2164. cur.clear();
  2165. return false;
  2166. }
  2167. setCurrent();
  2168. ++curTreeNum;
  2169. return true;
  2170. }
  2171. IConstWorkUnit & query() { return *cur; }
  2172. };
  2173. //==========================================================================================
  2174. class CStringArrayIterator : public CInterface, implements IStringIterator
  2175. {
  2176. StringArray strings;
  2177. unsigned idx;
  2178. public:
  2179. IMPLEMENT_IINTERFACE;
  2180. CStringArrayIterator() { idx = 0; };
  2181. void append(const char *str) { strings.append(str); }
  2182. virtual bool first() { idx = 0; return strings.isItem(idx); }
  2183. virtual bool next() { idx ++; return strings.isItem(idx); }
  2184. virtual bool isValid() { return strings.isItem(idx); }
  2185. virtual IStringVal & str(IStringVal &s) { s.set(strings.item(idx)); return s; }
  2186. };
  2187. class CCachedJobNameIterator : public CInterface, implements IStringIterator
  2188. {
  2189. Owned<IPropertyTreeIterator> it;
  2190. public:
  2191. IMPLEMENT_IINTERFACE;
  2192. CCachedJobNameIterator(IPropertyTreeIterator *p) : it(p) {};
  2193. virtual bool first() { return it->first(); }
  2194. virtual bool next() { return it->next(); }
  2195. virtual bool isValid() { return it->isValid(); }
  2196. virtual IStringVal & str(IStringVal &s) { s.set(it->query().queryName()+1); return s; }
  2197. };
  2198. class CEmptyStringIterator : public CInterface, implements IStringIterator
  2199. {
  2200. public:
  2201. IMPLEMENT_IINTERFACE;
  2202. virtual bool first() { return false; }
  2203. virtual bool next() { return false; }
  2204. virtual bool isValid() { return false; }
  2205. virtual IStringVal & str(IStringVal &s) { s.clear(); return s; }
  2206. };
  2207. mapEnums workunitSortFields[] =
  2208. {
  2209. { WUSFuser, "@submitID" },
  2210. { WUSFcluster, "@clusterName" },
  2211. { WUSFjob, "@jobName" },
  2212. { WUSFstate, "@state" },
  2213. { WUSFpriority, "@priorityClass" },
  2214. { WUSFprotected, "@protected" },
  2215. { WUSFwuid, "@" },
  2216. { WUSFecl, "Query/Text" },
  2217. { WUSFfileread, "FilesRead/File/@name" },
  2218. { WUSFroxiecluster, "RoxieQueryInfo/@roxieClusterName" },
  2219. { WUSFtotalthortime, "Statistics/Statistic[@c='summary'][@creator='thor'][@kind='TimeElapsed']/@value|"
  2220. "Statistics/Statistic[@c='summary'][@creator='hthor'][@kind='TimeElapsed']/@value|"
  2221. "Statistics/Statistic[@c='summary'][@creator='roxie'][@kind='TimeElapsed']/@value|"
  2222. "Statistics/Statistic[@desc='Total thor time']/@value|"
  2223. "Timings/Timing[@name='Total thor time']/@duration" //Use Statistics first. If not found, use Timings
  2224. },
  2225. { WUSFterm, NULL }
  2226. };
  2227. mapEnums querySortFields[] =
  2228. {
  2229. { WUQSFId, "@id" },
  2230. { WUQSFwuid, "@wuid" },
  2231. { WUQSFname, "@name" },
  2232. { WUQSFdll, "@dll" },
  2233. { WUQSFmemoryLimit, "@memoryLimit" },
  2234. { WUQSFmemoryLimitHi, "@memoryLimit" },
  2235. { WUQSFtimeLimit, "@timeLimit" },
  2236. { WUQSFtimeLimitHi, "@timeLimit" },
  2237. { WUQSFwarnTimeLimit, "@warnTimeLimit" },
  2238. { WUQSFwarnTimeLimitHi, "@warnTimeLimit" },
  2239. { WUQSFpriority, "@priority" },
  2240. { WUQSFpriorityHi, "@priority" },
  2241. { WUQSFQuerySet, "../@id" },
  2242. { WUQSFActivited, "@activated" },
  2243. { WUQSFSuspendedByUser, "@suspended" },
  2244. { WUQSFLibrary, "Library"},
  2245. { WUQSFterm, NULL }
  2246. };
  2247. class asyncRemoveDllWorkItem: public CInterface, implements IWorkQueueItem // class only used in asyncRemoveDll
  2248. {
  2249. StringAttr name;
  2250. public:
  2251. IMPLEMENT_IINTERFACE;
  2252. asyncRemoveDllWorkItem(const char * _name) : name(_name)
  2253. {
  2254. }
  2255. void execute()
  2256. {
  2257. PROGLOG("WU removeDll %s", name.get());
  2258. queryDllServer().removeDll(name, true, true); // <name>, removeDlls=true, removeDirectory=true
  2259. }
  2260. };
  2261. class asyncRemoveRemoteFileWorkItem: public CInterface, implements IWorkQueueItem // class only used in asyncRemoveFile
  2262. {
  2263. RemoteFilename name;
  2264. public:
  2265. IMPLEMENT_IINTERFACE;
  2266. asyncRemoveRemoteFileWorkItem(const char * _ip, const char * _name)
  2267. {
  2268. SocketEndpoint ep(_ip);
  2269. name.setPath(ep, _name);
  2270. }
  2271. void execute()
  2272. {
  2273. Owned<IFile> file = createIFile(name);
  2274. PROGLOG("WU removeDll %s",file->queryFilename());
  2275. file->remove();
  2276. }
  2277. };
  2278. //==========================================================================================
  2279. class CConstQuerySetQueryIterator : public CInterface, implements IConstQuerySetQueryIterator
  2280. {
  2281. IArrayOf<IPropertyTree> trees;
  2282. unsigned index;
  2283. public:
  2284. IMPLEMENT_IINTERFACE;
  2285. CConstQuerySetQueryIterator(IArrayOf<IPropertyTree> &_trees)
  2286. {
  2287. ForEachItemIn(t, _trees)
  2288. trees.append(*LINK(&_trees.item(t)));
  2289. index = 0;
  2290. }
  2291. ~CConstQuerySetQueryIterator()
  2292. {
  2293. trees.kill();
  2294. }
  2295. bool first()
  2296. {
  2297. index = 0;
  2298. return (trees.ordinality()!=0);
  2299. }
  2300. bool next()
  2301. {
  2302. index++;
  2303. return (index<trees.ordinality());
  2304. }
  2305. bool isValid()
  2306. {
  2307. return (index<trees.ordinality());
  2308. }
  2309. IPropertyTree &query()
  2310. {
  2311. return trees.item(index);
  2312. }
  2313. };
  2314. #define WUID_VERSION 2 // recorded in each wuid created, useful for bkwd compat. checks
  2315. class CWorkUnitFactory : public CInterface, implements IWorkUnitFactory, implements IDaliClientShutdown
  2316. {
  2317. Owned<IWorkQueueThread> deletedllworkq;
  2318. public:
  2319. IMPLEMENT_IINTERFACE;
  2320. CWorkUnitFactory()
  2321. {
  2322. // Assumes dali client configuration has already been done
  2323. sdsManager = &querySDS();
  2324. session = myProcessSession();
  2325. deletedllworkq.setown(createWorkQueueThread());
  2326. addShutdownHook(*this);
  2327. }
  2328. ~CWorkUnitFactory()
  2329. {
  2330. removeShutdownHook(*this);
  2331. // deletepool->joinAll();
  2332. }
  2333. void clientShutdown();
  2334. SessionId startSession()
  2335. {
  2336. // Temporary placeholder until startSession is implemented
  2337. #ifdef _WIN32
  2338. return GetTickCount();
  2339. #else
  2340. struct timeval tm;
  2341. gettimeofday(&tm,NULL);
  2342. return tm.tv_usec;
  2343. #endif
  2344. }
  2345. IWorkUnit* ensureNamedWorkUnit(const char *name)
  2346. {
  2347. if (workUnitTraceLevel > 1)
  2348. PrintLog("ensureNamedWorkUnit created %s", name);
  2349. StringBuffer wuRoot;
  2350. getXPath(wuRoot, name);
  2351. IRemoteConnection* conn = sdsManager->connect(wuRoot.str(), session, RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
  2352. conn->queryRoot()->setProp("@xmlns:xsi", "http://www.w3.org/1999/XMLSchema-instance");
  2353. Owned<CLocalWorkUnit> cw = new CDaliWorkUnit(conn, (ISecManager *)NULL, NULL);
  2354. return &cw->lockRemote(false);
  2355. }
  2356. virtual IWorkUnit* createNamedWorkUnit(const char *wuid, const char *app, const char *user)
  2357. {
  2358. StringBuffer wuRoot;
  2359. getXPath(wuRoot, wuid);
  2360. IRemoteConnection *conn;
  2361. if (queryDaliServerVersion().compare("2.0") >= 0)
  2362. conn = sdsManager->connect(wuRoot.str(), session, RTM_LOCK_WRITE|RTM_CREATE_UNIQUE, SDS_LOCK_TIMEOUT);
  2363. else
  2364. conn = sdsManager->connect(wuRoot.str(), session, RTM_LOCK_WRITE|RTM_CREATE, SDS_LOCK_TIMEOUT);
  2365. conn->queryRoot()->setProp("@xmlns:xsi", "http://www.w3.org/1999/XMLSchema-instance");
  2366. conn->queryRoot()->setPropInt("@wuidVersion", WUID_VERSION);
  2367. Owned<CLocalWorkUnit> cw = new CDaliWorkUnit(conn, (ISecManager*)NULL, NULL);
  2368. IWorkUnit* ret = &cw->lockRemote(false);
  2369. ret->setDebugValue("CREATED_BY", app, true);
  2370. ret->setDebugValue("CREATED_FOR", user, true);
  2371. if (user)
  2372. cw->setWuScope(user);
  2373. return ret;
  2374. }
  2375. virtual IWorkUnit* createWorkUnit(const char *app, const char *user)
  2376. {
  2377. StringBuffer wuid("W");
  2378. char result[32];
  2379. time_t ltime;
  2380. time( &ltime );
  2381. tm *today = localtime( &ltime ); // MORE - this is not threadsafe. But I probably don't care that much!
  2382. strftime(result, sizeof(result), "%Y%m%d-%H%M%S", today);
  2383. wuid.append(result);
  2384. if (queryDaliServerVersion().compare("2.0") < 0)
  2385. wuid.append('-').append(startSession());
  2386. if (workUnitTraceLevel > 1)
  2387. PrintLog("createWorkUnit created %s", wuid.str());
  2388. IWorkUnit* ret = createNamedWorkUnit(wuid.str(), app, user);
  2389. if (workUnitTraceLevel > 1)
  2390. PrintLog("createWorkUnit created %s", ret->queryWuid());
  2391. addTimeStamp(ret, SSTglobal, NULL, StWhenCreated);
  2392. return ret;
  2393. }
  2394. bool secDeleteWorkUnit(const char * wuid, ISecManager *secmgr, ISecUser *secuser, bool raiseexceptions)
  2395. {
  2396. if (workUnitTraceLevel > 1)
  2397. PrintLog("deleteWorkUnit %s", wuid);
  2398. StringBuffer wuRoot;
  2399. getXPath(wuRoot, wuid);
  2400. IRemoteConnection *conn = sdsManager->connect(wuRoot.str(), session, RTM_LOCK_WRITE|RTM_LOCK_SUB, SDS_LOCK_TIMEOUT);
  2401. if (!conn)
  2402. {
  2403. if (workUnitTraceLevel > 0)
  2404. PrintLog("deleteWorkUnit %s not found", wuid);
  2405. return false;
  2406. }
  2407. Owned<CLocalWorkUnit> cw = new CDaliWorkUnit(conn, secmgr, secuser); // takes ownership of conn
  2408. if (secmgr && !checkWuSecAccess(*cw.get(), *secmgr, secuser, SecAccess_Full, "delete", true, true)) {
  2409. if (raiseexceptions) {
  2410. // perhaps raise exception here?
  2411. }
  2412. return false;
  2413. }
  2414. if (raiseexceptions) {
  2415. try
  2416. {
  2417. cw->cleanupAndDelete(true,true);
  2418. }
  2419. catch (IException *E)
  2420. {
  2421. StringBuffer s;
  2422. LOG(MCexception(E, MSGCLS_warning), E, s.append("Exception during deleteWorkUnit: ").append(wuid).str());
  2423. E->Release();
  2424. return false;
  2425. }
  2426. }
  2427. else
  2428. cw->cleanupAndDelete(true,true);
  2429. removeWorkUnitFromAllQueues(wuid); //known active workunits wouldn't make it this far
  2430. return true;
  2431. }
  2432. virtual bool deleteWorkUnitEx(const char * wuid)
  2433. {
  2434. return secDeleteWorkUnit(wuid,NULL,NULL,true);
  2435. }
  2436. virtual bool deleteWorkUnit(const char * wuid)
  2437. {
  2438. return secDeleteWorkUnit(wuid,NULL,NULL,false);
  2439. }
  2440. virtual IConstWorkUnitIterator * getWorkUnitsByOwner(const char * owner)
  2441. {
  2442. StringBuffer path("*");
  2443. if (owner && *owner)
  2444. path.append("[@submitID=\"").append(owner).append("\"]");
  2445. return getWorkUnitsByXPath(path.str());
  2446. }
  2447. virtual IConstWorkUnitIterator * getWorkUnitsByState(WUState state);
  2448. virtual IConstWorkUnitIterator * getWorkUnitsByECL(const char* ecl)
  2449. {
  2450. StringBuffer path("*");
  2451. if (ecl && *ecl)
  2452. path.append("[Query/Text=~\"*").append(ecl).append("*\"]");
  2453. return getWorkUnitsByXPath(path.str());
  2454. }
  2455. virtual IConstWorkUnitIterator * getWorkUnitsByCluster(const char* cluster)
  2456. {
  2457. StringBuffer path("*");
  2458. if (cluster && *cluster)
  2459. path.append("[@clusterName=\"").append(cluster).append("\"]");
  2460. return getWorkUnitsByXPath(path.str());
  2461. }
  2462. virtual IConstWorkUnitIterator * getChildWorkUnits(const char *parent)
  2463. {
  2464. StringBuffer path("*[@parent=\"");
  2465. path.append(parent).append("\"]");
  2466. return getWorkUnitsByXPath(path.str());
  2467. }
  2468. virtual IConstWorkUnit* secOpenWorkUnit(const char *wuid, bool lock, ISecManager *secmgr=NULL, ISecUser *secuser=NULL)
  2469. {
  2470. StringBuffer wuidStr(wuid);
  2471. wuidStr.trim();
  2472. if (wuidStr.length() && ('w' == wuidStr.charAt(0)))
  2473. wuidStr.setCharAt(0, 'W');
  2474. if (!wuidStr.length() || ('W' != wuidStr.charAt(0)))
  2475. {
  2476. if (workUnitTraceLevel > 1)
  2477. PrintLog("openWorkUnit %s invalid WUID", nullText(wuidStr.str()));
  2478. return NULL;
  2479. }
  2480. if (workUnitTraceLevel > 1)
  2481. PrintLog("openWorkUnit %s", wuidStr.str());
  2482. StringBuffer wuRoot;
  2483. getXPath(wuRoot, wuidStr.str());
  2484. IRemoteConnection* conn = sdsManager->connect(wuRoot.str(), session, lock ? RTM_LOCK_READ|RTM_LOCK_SUB : 0, SDS_LOCK_TIMEOUT);
  2485. if (conn)
  2486. {
  2487. CLocalWorkUnit *wu = new CDaliWorkUnit(conn, secmgr, secuser);
  2488. if (secmgr && wu)
  2489. {
  2490. if (!checkWuSecAccess(*wu, *secmgr, secuser, SecAccess_Read, "opening", true, true))
  2491. {
  2492. delete wu;
  2493. return NULL;
  2494. }
  2495. }
  2496. return wu;
  2497. }
  2498. else
  2499. {
  2500. if (workUnitTraceLevel > 0)
  2501. PrintLog("openWorkUnit %s not found", wuidStr.str());
  2502. return NULL;
  2503. }
  2504. }
  2505. virtual IConstWorkUnit* openWorkUnit(const char *wuid, bool lock)
  2506. {
  2507. return secOpenWorkUnit(wuid, lock);
  2508. }
  2509. virtual IWorkUnit* secUpdateWorkUnit(const char *wuid, ISecManager *secmgr=NULL, ISecUser *secuser=NULL)
  2510. {
  2511. if (workUnitTraceLevel > 1)
  2512. PrintLog("updateWorkUnit %s", wuid);
  2513. StringBuffer wuRoot;
  2514. getXPath(wuRoot, wuid);
  2515. IRemoteConnection* conn = sdsManager->connect(wuRoot.str(), session, RTM_LOCK_WRITE|RTM_LOCK_SUB, SDS_LOCK_TIMEOUT);
  2516. if (conn)
  2517. {
  2518. Owned<CLocalWorkUnit> cw = new CDaliWorkUnit(conn, secmgr, secuser);
  2519. if (secmgr && cw)
  2520. {
  2521. if (!checkWuSecAccess(*cw.get(), *secmgr, secuser, SecAccess_Write, "updating", true, true))
  2522. return NULL;
  2523. }
  2524. return &cw->lockRemote(false);
  2525. }
  2526. else
  2527. {
  2528. if (workUnitTraceLevel > 0)
  2529. PrintLog("updateWorkUnit %s not found", wuid);
  2530. return NULL;
  2531. }
  2532. }
  2533. virtual IWorkUnit* updateWorkUnit(const char *wuid)
  2534. {
  2535. return secUpdateWorkUnit(wuid);
  2536. }
  2537. virtual int setTracingLevel(int newLevel)
  2538. {
  2539. if (newLevel)
  2540. PrintLog("Setting workunit trace level to %d", newLevel);
  2541. int level = workUnitTraceLevel;
  2542. workUnitTraceLevel = newLevel;
  2543. return level;
  2544. }
  2545. IConstWorkUnitIterator * getWorkUnitsByXPath(const char *xpath)
  2546. {
  2547. return getWorkUnitsByXPath(xpath,NULL,NULL);
  2548. }
  2549. void descheduleAllWorkUnits()
  2550. {
  2551. Owned<IRemoteConnection> conn = querySDS().connect("/Schedule", myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
  2552. if(!conn) return;
  2553. Owned<IPropertyTree> root(conn->queryRoot()->getBranch("."));
  2554. KeptAtomTable entries;
  2555. Owned<IPropertyTreeIterator> iter(root->getElements("*/*/*/*"));
  2556. StringBuffer wuid;
  2557. for(iter->first(); iter->isValid(); iter->next())
  2558. {
  2559. char const * entry = iter->query().queryName();
  2560. if(!entries.find(entry))
  2561. {
  2562. entries.addAtom(entry);
  2563. ncnameUnescape(entry, wuid.clear());
  2564. Owned<IWorkUnit> wu = updateWorkUnit(wuid);
  2565. if(wu && (wu->getState() == WUStateWait))
  2566. wu->setState(WUStateCompleted);
  2567. }
  2568. }
  2569. bool more;
  2570. do more = root->removeProp("*"); while(more);
  2571. }
  2572. IConstWorkUnitIterator * getWorkUnitsByXPath(const char *xpath, ISecManager *secmgr, ISecUser *secuser)
  2573. {
  2574. Owned<IRemoteConnection> conn = sdsManager->connect("/WorkUnits", session, 0, SDS_LOCK_TIMEOUT);
  2575. if (conn)
  2576. {
  2577. CDaliVersion serverVersionNeeded("3.2");
  2578. Owned<IPropertyTreeIterator> iter(queryDaliServerVersion().compare(serverVersionNeeded) < 0 ?
  2579. conn->queryRoot()->getElements(xpath) :
  2580. conn->getElements(xpath));
  2581. return new CConstWUIterator(this, iter, secmgr, secuser);
  2582. }
  2583. else
  2584. return NULL;
  2585. }
  2586. IConstWorkUnitIterator* getWorkUnitsSorted( WUSortField *sortorder, // list of fields to sort by (terminated by WUSFterm)
  2587. WUSortField *filters, // NULL or list of fields to folteron (terminated by WUSFterm)
  2588. const void *filterbuf, // (appended) string values for filters
  2589. unsigned startoffset,
  2590. unsigned maxnum,
  2591. const char *queryowner,
  2592. __int64 *cachehint,
  2593. ISecManager *secmgr,
  2594. ISecUser *secuser,
  2595. unsigned *total)
  2596. {
  2597. class CWorkUnitsPager : public CSimpleInterface, implements IElementsPager
  2598. {
  2599. StringAttr xPath;
  2600. StringAttr sortOrder;
  2601. StringAttr nameFilterLo;
  2602. StringAttr nameFilterHi;
  2603. StringArray unknownAttributes;
  2604. public:
  2605. IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
  2606. CWorkUnitsPager(const char* _xPath, const char *_sortOrder, const char* _nameFilterLo, const char* _nameFilterHi, StringArray& _unknownAttributes)
  2607. : xPath(_xPath), sortOrder(_sortOrder), nameFilterLo(_nameFilterLo), nameFilterHi(_nameFilterHi)
  2608. {
  2609. ForEachItemIn(x, _unknownAttributes)
  2610. unknownAttributes.append(_unknownAttributes.item(x));
  2611. }
  2612. virtual IRemoteConnection* getElements(IArrayOf<IPropertyTree> &elements)
  2613. {
  2614. Owned<IRemoteConnection> conn = querySDS().connect("WorkUnits", myProcessSession(), 0, SDS_LOCK_TIMEOUT);
  2615. if (!conn)
  2616. return NULL;
  2617. Owned<IPropertyTreeIterator> iter = conn->getElements(xPath);
  2618. if (!iter)
  2619. return NULL;
  2620. sortElements(iter, sortOrder.get(), nameFilterLo.get(), nameFilterHi.get(), unknownAttributes, elements);
  2621. return conn.getClear();
  2622. }
  2623. };
  2624. class CScopeChecker : public CSimpleInterface, implements ISortedElementsTreeFilter
  2625. {
  2626. UniqueScopes done;
  2627. ISecManager *secmgr;
  2628. ISecUser *secuser;
  2629. CriticalSection crit;
  2630. public:
  2631. IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
  2632. CScopeChecker(ISecManager *_secmgr,ISecUser *_secuser)
  2633. {
  2634. secmgr = _secmgr;
  2635. secuser = _secuser;
  2636. }
  2637. bool isOK(IPropertyTree &tree)
  2638. {
  2639. const char *scopename = tree.queryProp("@scope");
  2640. if (!scopename||!*scopename)
  2641. return true;
  2642. {
  2643. CriticalBlock block(crit);
  2644. const bool *b = done.getValue(scopename);
  2645. if (b)
  2646. return *b;
  2647. }
  2648. bool ret = checkWuScopeSecAccess(scopename,*secmgr,secuser,SecAccess_Read,"iterating",false,false);
  2649. {
  2650. // conceivably could have already been checked and added, but ok.
  2651. CriticalBlock block(crit);
  2652. done.setValue(scopename,ret);
  2653. }
  2654. return ret;
  2655. }
  2656. };
  2657. Owned<ISortedElementsTreeFilter> sc = new CScopeChecker(secmgr,secuser);
  2658. StringBuffer query;
  2659. StringBuffer so;
  2660. StringAttr namefilter("*");
  2661. StringAttr namefilterlo;
  2662. StringAttr namefilterhi;
  2663. StringArray unknownAttributes;
  2664. if (filters) {
  2665. const char *fv = (const char *)filterbuf;
  2666. for (unsigned i=0;filters[i]!=WUSFterm;i++) {
  2667. int fmt = filters[i];
  2668. int subfmt = (fmt&0xff);
  2669. if (subfmt==WUSFwuid)
  2670. namefilterlo.set(fv);
  2671. else if (subfmt==WUSFwuidhigh)
  2672. namefilterhi.set(fv);
  2673. else if (subfmt==WUSFwildwuid)
  2674. namefilter.set(fv);
  2675. else if (subfmt==WUSFcustom)
  2676. query.append("[").append(fv).append("]");
  2677. else if (!fv || !*fv)
  2678. unknownAttributes.append(getEnumText(subfmt,workunitSortFields));
  2679. else {
  2680. query.append('[').append(getEnumText(subfmt,workunitSortFields)).append('=');
  2681. if (fmt&WUSFnocase)
  2682. query.append('?');
  2683. if (fmt&WUSFwild)
  2684. query.append('~');
  2685. query.append('"').append(fv).append("\"]");
  2686. }
  2687. fv = fv + strlen(fv)+1;
  2688. }
  2689. }
  2690. query.insert(0, namefilter.get());
  2691. if (sortorder) {
  2692. for (unsigned i=0;sortorder[i]!=WUSFterm;i++) {
  2693. if (so.length())
  2694. so.append(',');
  2695. int fmt = sortorder[i];
  2696. if (fmt&WUSFreverse)
  2697. so.append('-');
  2698. if (fmt&WUSFnocase)
  2699. so.append('?');
  2700. if (fmt&WUSFnumeric)
  2701. so.append('#');
  2702. so.append(getEnumText(fmt&0xff,workunitSortFields));
  2703. }
  2704. }
  2705. IArrayOf<IPropertyTree> results;
  2706. Owned<IElementsPager> elementsPager = new CWorkUnitsPager(query.str(), so.length()?so.str():NULL, namefilterlo.get(), namefilterhi.get(), unknownAttributes);
  2707. Owned<IRemoteConnection> conn=getElementsPaged(elementsPager,startoffset,maxnum,secmgr?sc:NULL,queryowner,cachehint,results,total);
  2708. return new CConstWUArrayIterator(this, results);
  2709. }
  2710. IConstWorkUnitIterator* getWorkUnitsSorted( WUSortField *sortorder, // list of fields to sort by (terminated by WUSFterm)
  2711. WUSortField *filters, // NULL or list of fields to filter on (terminated by WUSFterm)
  2712. const void *filterbuf, // (appended) string values for filters
  2713. unsigned startoffset,
  2714. unsigned maxnum,
  2715. const char *queryowner,
  2716. __int64 *cachehint,
  2717. unsigned *total)
  2718. {
  2719. return getWorkUnitsSorted(sortorder,filters,filterbuf,startoffset,maxnum,queryowner,cachehint, NULL, NULL, total);
  2720. }
  2721. IConstQuerySetQueryIterator* getQuerySetQueriesSorted( WUQuerySortField *sortorder, // list of fields to sort by (terminated by WUSFterm)
  2722. WUQuerySortField *filters, // NULL or list of fields to filter on (terminated by WUSFterm)
  2723. const void *filterbuf, // (appended) string values for filters
  2724. unsigned startoffset,
  2725. unsigned maxnum,
  2726. __int64 *cachehint,
  2727. unsigned *total,
  2728. const MapStringTo<bool> *_subset)
  2729. {
  2730. struct PostFilters
  2731. {
  2732. WUQueryFilterBoolean activatedFilter;
  2733. WUQueryFilterBoolean suspendedByUserFilter;
  2734. PostFilters()
  2735. {
  2736. activatedFilter = WUQFSAll;
  2737. suspendedByUserFilter = WUQFSAll;
  2738. };
  2739. } postFilters;
  2740. class CQuerySetQueriesPager : public CSimpleInterface, implements IElementsPager
  2741. {
  2742. StringAttr querySet;
  2743. StringAttr xPath;
  2744. StringAttr sortOrder;
  2745. PostFilters postFilters;
  2746. StringArray unknownAttributes;
  2747. const MapStringTo<bool> *subset;
  2748. void populateQueryTree(const IPropertyTree* querySetTree, IPropertyTree* queryTree)
  2749. {
  2750. const char* querySetId = querySetTree->queryProp("@id");
  2751. VStringBuffer path("Query%s", xPath.get());
  2752. Owned<IPropertyTreeIterator> iter = querySetTree->getElements(path.str());
  2753. ForEach(*iter)
  2754. {
  2755. IPropertyTree &query = iter->query();
  2756. bool activated = false;
  2757. const char* queryId = query.queryProp("@id");
  2758. if (queryId && *queryId)
  2759. {
  2760. if (subset)
  2761. {
  2762. VStringBuffer match("%s/%s", querySetId, queryId);
  2763. if (!subset->getValue(match))
  2764. continue;
  2765. }
  2766. VStringBuffer aliasXPath("Alias[@id='%s']", queryId);
  2767. IPropertyTree *alias = querySetTree->queryPropTree(aliasXPath.str());
  2768. if (alias)
  2769. activated = true;
  2770. }
  2771. if (activated && (postFilters.activatedFilter == WUQFSNo))
  2772. continue;
  2773. if (!activated && (postFilters.activatedFilter == WUQFSYes))
  2774. continue;
  2775. if ((postFilters.suspendedByUserFilter == WUQFSNo) && query.hasProp(getEnumText(WUQSFSuspendedByUser,querySortFields)))
  2776. continue;
  2777. if ((postFilters.suspendedByUserFilter == WUQFSYes) && !query.hasProp(getEnumText(WUQSFSuspendedByUser,querySortFields)))
  2778. continue;
  2779. IPropertyTree *queryWithSetId = queryTree->addPropTree("Query", createPTreeFromIPT(&query));
  2780. queryWithSetId->setProp("@querySetId", querySetId);
  2781. queryWithSetId->setPropBool("@activated", activated);
  2782. }
  2783. }
  2784. IRemoteConnection* populateQueryTree(IPropertyTree* queryTree)
  2785. {
  2786. StringBuffer querySetXPath("QuerySets");
  2787. if (!querySet.isEmpty())
  2788. querySetXPath.appendf("/QuerySet[@id=\"%s\"]", querySet.get());
  2789. Owned<IRemoteConnection> conn = querySDS().connect(querySetXPath.str(), myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT);
  2790. if (!conn)
  2791. return NULL;
  2792. if (querySet.isEmpty())
  2793. {
  2794. Owned<IPropertyTreeIterator> querySetIter = conn->queryRoot()->getElements("*");
  2795. ForEach(*querySetIter)
  2796. populateQueryTree(&querySetIter->query(), queryTree);
  2797. }
  2798. else
  2799. populateQueryTree(conn->queryRoot(), queryTree);
  2800. return conn.getClear();
  2801. }
  2802. public:
  2803. IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
  2804. CQuerySetQueriesPager(const char* _querySet, const char* _xPath, const char *_sortOrder, PostFilters& _postFilters, StringArray& _unknownAttributes, const MapStringTo<bool> *_subset)
  2805. : querySet(_querySet), xPath(_xPath), sortOrder(_sortOrder), subset(_subset)
  2806. {
  2807. postFilters.activatedFilter = _postFilters.activatedFilter;
  2808. postFilters.suspendedByUserFilter = _postFilters.suspendedByUserFilter;
  2809. ForEachItemIn(x, _unknownAttributes)
  2810. unknownAttributes.append(_unknownAttributes.item(x));
  2811. }
  2812. virtual IRemoteConnection* getElements(IArrayOf<IPropertyTree> &elements)
  2813. {
  2814. Owned<IPropertyTree> elementTree = createPTree("Queries");
  2815. Owned<IRemoteConnection> conn = populateQueryTree(elementTree);
  2816. if (!conn)
  2817. return NULL;
  2818. Owned<IPropertyTreeIterator> iter = elementTree->getElements("*");
  2819. if (!iter)
  2820. return NULL;
  2821. sortElements(iter, sortOrder.get(), NULL, NULL, unknownAttributes, elements);
  2822. return conn.getClear();
  2823. }
  2824. };
  2825. StringAttr querySet;
  2826. StringBuffer xPath;
  2827. StringBuffer so;
  2828. StringArray unknownAttributes;
  2829. if (filters)
  2830. {
  2831. const char *fv = (const char *)filterbuf;
  2832. for (unsigned i=0;filters[i]!=WUQSFterm;i++) {
  2833. int fmt = filters[i];
  2834. int subfmt = (fmt&0xff);
  2835. if (subfmt==WUQSFQuerySet)
  2836. querySet.set(fv);
  2837. else if ((subfmt==WUQSFmemoryLimit) || (subfmt==WUQSFtimeLimit) || (subfmt==WUQSFwarnTimeLimit) || (subfmt==WUQSFpriority))
  2838. xPath.append('[').append(getEnumText(subfmt,querySortFields)).append(">=").append(fv).append("]");
  2839. else if ((subfmt==WUQSFmemoryLimitHi) || (subfmt==WUQSFtimeLimitHi) || (subfmt==WUQSFwarnTimeLimitHi) || (subfmt==WUQSFpriorityHi))
  2840. xPath.append('[').append(getEnumText(subfmt,querySortFields)).append("<=").append(fv).append("]");
  2841. else if (subfmt==WUQSFActivited)
  2842. postFilters.activatedFilter = (WUQueryFilterBoolean) atoi(fv);
  2843. else if (subfmt==WUQSFSuspendedByUser)
  2844. postFilters.suspendedByUserFilter = (WUQueryFilterBoolean) atoi(fv);
  2845. else if (!fv || !*fv)
  2846. unknownAttributes.append(getEnumText(subfmt,querySortFields));
  2847. else {
  2848. xPath.append('[').append(getEnumText(subfmt,querySortFields)).append('=');
  2849. if (fmt&WUQSFnocase)
  2850. xPath.append('?');
  2851. if (fmt&WUQSFnumeric)
  2852. xPath.append('#');
  2853. if (fmt&WUQSFwild)
  2854. xPath.append('~');
  2855. xPath.append('"').append(fv).append("\"]");
  2856. }
  2857. fv = fv + strlen(fv)+1;
  2858. }
  2859. }
  2860. if (sortorder) {
  2861. for (unsigned i=0;sortorder[i]!=WUQSFterm;i++) {
  2862. if (so.length())
  2863. so.append(',');
  2864. int fmt = sortorder[i];
  2865. if (fmt&WUQSFreverse)
  2866. so.append('-');
  2867. if (fmt&WUQSFnocase)
  2868. so.append('?');
  2869. if (fmt&WUQSFnumeric)
  2870. so.append('#');
  2871. so.append(getEnumText(fmt&0xff,querySortFields));
  2872. }
  2873. }
  2874. IArrayOf<IPropertyTree> results;
  2875. Owned<IElementsPager> elementsPager = new CQuerySetQueriesPager(querySet.get(), xPath.str(), so.length()?so.str():NULL, postFilters, unknownAttributes, _subset);
  2876. Owned<IRemoteConnection> conn=getElementsPaged(elementsPager,startoffset,maxnum,NULL,"",cachehint,results,total);
  2877. return new CConstQuerySetQueryIterator(results);
  2878. }
  2879. virtual unsigned numWorkUnits()
  2880. {
  2881. Owned<IRemoteConnection> conn = sdsManager->connect("/WorkUnits", session, 0, SDS_LOCK_TIMEOUT);
  2882. if (!conn)
  2883. return 0;
  2884. IPropertyTree *root = conn->queryRoot();
  2885. return root->numChildren();
  2886. }
  2887. virtual unsigned numWorkUnitsFiltered(WUSortField *filters,
  2888. const void *filterbuf,
  2889. ISecManager *secmgr,
  2890. ISecUser *secuser)
  2891. {
  2892. unsigned total;
  2893. Owned<IConstWorkUnitIterator> iter = getWorkUnitsSorted( NULL,filters,filterbuf,0,0x7fffffff,NULL,NULL,secmgr,secuser,&total);
  2894. return total;
  2895. }
  2896. virtual unsigned numWorkUnitsFiltered(WUSortField *filters,const void *filterbuf)
  2897. {
  2898. if (!filters)
  2899. return numWorkUnits();
  2900. return numWorkUnitsFiltered(filters,filterbuf,NULL,NULL);
  2901. }
  2902. void asyncRemoveDll(const char * name)
  2903. {
  2904. deletedllworkq->post(new asyncRemoveDllWorkItem(name));
  2905. }
  2906. void asyncRemoveFile(const char * ip, const char * name)
  2907. {
  2908. deletedllworkq->post(new asyncRemoveRemoteFileWorkItem(ip, name));
  2909. }
  2910. ISDSManager *sdsManager;
  2911. SessionId session;
  2912. ISecManager *secMgr;
  2913. private:
  2914. void deleteChildren(IPropertyTree *root, const char *wuid)
  2915. {
  2916. StringBuffer kids("*[@parent=\"");
  2917. kids.append(wuid).append("\"]");
  2918. Owned<IPropertyTreeIterator> it = root->getElements(kids.str());
  2919. ForEach (*it)
  2920. {
  2921. deleteChildren(root, it->query().queryName());
  2922. }
  2923. root->removeProp(wuid);
  2924. }
  2925. class CConstWUIterator : public CInterface, implements IConstWorkUnitIterator
  2926. {
  2927. Owned<IConstWorkUnit> cur;
  2928. Linked<IWorkUnitFactory> factory;
  2929. Linked<IPropertyTreeIterator> ptreeIter;
  2930. Owned<ISecResourceList> scopes;
  2931. void setCurrent()
  2932. {
  2933. cur.setown(factory->openWorkUnit(ptreeIter->query().queryName(), false));
  2934. }
  2935. bool getNext() // scan for a workunit with permissions
  2936. {
  2937. if (!scopes)
  2938. {
  2939. setCurrent();
  2940. return true;
  2941. }
  2942. do
  2943. {
  2944. const char *scopeName = ptreeIter->query().queryProp("@scope");
  2945. if (!scopeName || !*scopeName || checkWuScopeListSecAccess(scopeName, scopes, SecAccess_Read, "iterating", false, false))
  2946. {
  2947. setCurrent();
  2948. return true;
  2949. }
  2950. }
  2951. while (ptreeIter->next());
  2952. cur.clear();
  2953. return false;
  2954. }
  2955. public:
  2956. IMPLEMENT_IINTERFACE;
  2957. CConstWUIterator(IWorkUnitFactory *_factory, IPropertyTreeIterator *_ptreeIter, ISecManager *secmgr=NULL, ISecUser *secuser=NULL)
  2958. : factory(_factory), ptreeIter(_ptreeIter)
  2959. {
  2960. UniqueScopes us;
  2961. if (secmgr /* && secmgr->authTypeRequired(RT_WORKUNIT_SCOPE) tbd */)
  2962. {
  2963. scopes.setown(secmgr->createResourceList("wuscopes"));
  2964. ForEach(*ptreeIter)
  2965. {
  2966. const char *scopeName = ptreeIter->query().queryProp("@scope");
  2967. if (scopeName && *scopeName && !us.getValue(scopeName))
  2968. {
  2969. scopes->addResource(scopeName);
  2970. us.setValue(scopeName, true);
  2971. }
  2972. }
  2973. if (scopes->count())
  2974. secmgr->authorizeEx(RT_WORKUNIT_SCOPE, *secuser, scopes);
  2975. else
  2976. scopes.clear();
  2977. }
  2978. }
  2979. bool first()
  2980. {
  2981. if (!ptreeIter->first())
  2982. {
  2983. cur.clear();
  2984. return false;
  2985. }
  2986. return getNext();
  2987. }
  2988. bool isValid()
  2989. {
  2990. return (NULL != cur.get());
  2991. }
  2992. bool next()
  2993. {
  2994. if (!ptreeIter->next())
  2995. {
  2996. cur.clear();
  2997. return false;
  2998. }
  2999. return getNext();
  3000. }
  3001. IConstWorkUnit & query() { return *cur; }
  3002. };
  3003. IRemoteConnection* connect(const char *xpath, unsigned flags)
  3004. {
  3005. return sdsManager->connect(xpath, session, flags, SDS_LOCK_TIMEOUT);
  3006. }
  3007. };
  3008. static Owned<CWorkUnitFactory> factory;
  3009. void CWorkUnitFactory::clientShutdown()
  3010. {
  3011. factory.clear();
  3012. }
  3013. void clientShutdownWorkUnit()
  3014. {
  3015. factory.clear();
  3016. }
  3017. extern WORKUNIT_API IWorkUnitFactory * getWorkUnitFactory()
  3018. {
  3019. if (!factory)
  3020. factory.setown(new CWorkUnitFactory());
  3021. return factory.getLink();
  3022. }
  3023. class CSecureWorkUnitFactory : public CInterface, implements IWorkUnitFactory
  3024. {
  3025. public:
  3026. IMPLEMENT_IINTERFACE;
  3027. CSecureWorkUnitFactory(ISecManager &secmgr, ISecUser &secuser)
  3028. {
  3029. if (!factory)
  3030. factory.setown(new CWorkUnitFactory());
  3031. secMgr.set(&secmgr);
  3032. secUser.set(&secuser);
  3033. }
  3034. virtual IWorkUnit* createNamedWorkUnit(const char *wuid, const char *app, const char *user)
  3035. {
  3036. checkWuScopeSecAccess(user, *secMgr.get(), secUser.get(), SecAccess_Write, "Create", true, true);
  3037. IWorkUnit *wu=factory->createNamedWorkUnit(wuid, app, user);
  3038. if (wu)
  3039. {
  3040. CLockedWorkUnit* lw = dynamic_cast<CLockedWorkUnit*>(wu);
  3041. if (lw)
  3042. lw->setSecIfcs(secMgr.get(), secUser.get());
  3043. }
  3044. return wu;
  3045. }
  3046. virtual IWorkUnit* createWorkUnit(const char *app, const char *user)
  3047. {
  3048. checkWuScopeSecAccess(user, *secMgr.get(), secUser.get(), SecAccess_Write, "Create", true, true);
  3049. IWorkUnit *wu=factory->createWorkUnit(app, user);
  3050. if (wu)
  3051. {
  3052. CLockedWorkUnit* lw = dynamic_cast<CLockedWorkUnit*>(wu);
  3053. if (lw)
  3054. lw->setSecIfcs(secMgr.get(), secUser.get());
  3055. }
  3056. return wu;
  3057. }
  3058. virtual bool deleteWorkUnitEx(const char * wuid)
  3059. {
  3060. return factory->secDeleteWorkUnit(wuid, secMgr.get(), secUser.get(), true);
  3061. }
  3062. virtual bool deleteWorkUnit(const char * wuid)
  3063. {
  3064. return factory->secDeleteWorkUnit(wuid, secMgr.get(), secUser.get(), false);
  3065. }
  3066. virtual IConstWorkUnit* openWorkUnit(const char *wuid, bool lock)
  3067. {
  3068. return factory->secOpenWorkUnit(wuid, lock, secMgr.get(), secUser.get());
  3069. }
  3070. virtual IWorkUnit* updateWorkUnit(const char *wuid)
  3071. {
  3072. return factory->secUpdateWorkUnit(wuid, secMgr.get(), secUser.get());
  3073. }
  3074. //make cached workunits a non secure pass through for now.
  3075. virtual IConstWorkUnitIterator * getWorkUnitsByOwner(const char * owner)
  3076. {
  3077. StringBuffer path("*");
  3078. if (owner && *owner)
  3079. path.append("[@submitID=\"").append(owner).append("\"]");
  3080. return factory->getWorkUnitsByXPath(path.str(), secMgr.get(), secUser.get());
  3081. }
  3082. virtual IConstWorkUnitIterator * getWorkUnitsByState(WUState state);
  3083. virtual IConstWorkUnitIterator * getWorkUnitsByECL(const char* ecl)
  3084. {
  3085. return factory->getWorkUnitsByECL(ecl);
  3086. }
  3087. virtual IConstWorkUnitIterator * getWorkUnitsByCluster(const char* cluster)
  3088. {
  3089. return factory->getWorkUnitsByCluster(cluster);
  3090. }
  3091. virtual IConstWorkUnitIterator * getWorkUnitsByXPath(const char * xpath)
  3092. {
  3093. return factory->getWorkUnitsByXPath(xpath, secMgr.get(), secUser.get());
  3094. }
  3095. virtual void descheduleAllWorkUnits()
  3096. {
  3097. factory->descheduleAllWorkUnits();
  3098. }
  3099. virtual IConstWorkUnitIterator * getChildWorkUnits(const char *parent)
  3100. {
  3101. StringBuffer path("*[@parent=\"");
  3102. path.append(parent).append("\"]");
  3103. return factory->getWorkUnitsByXPath(path.str(), secMgr.get(), secUser.get());
  3104. }
  3105. virtual int setTracingLevel(int newLevel)
  3106. {
  3107. return factory->setTracingLevel(newLevel);
  3108. }
  3109. virtual IConstWorkUnitIterator* getWorkUnitsSorted( WUSortField *sortorder, // list of fields to sort by (terminated by WUSFterm)
  3110. WUSortField *filters, // NULL or list of fields to filter on (terminated by WUSFterm)
  3111. const void *filterbuf, // (appended) string values for filters
  3112. unsigned startoffset,
  3113. unsigned maxnum,
  3114. const char *queryowner,
  3115. __int64 *cachehint,
  3116. unsigned *total)
  3117. {
  3118. return factory->getWorkUnitsSorted(sortorder,filters,filterbuf,startoffset,maxnum,queryowner,cachehint, secMgr.get(), secUser.get(), total);
  3119. }
  3120. virtual IConstQuerySetQueryIterator* getQuerySetQueriesSorted( WUQuerySortField *sortorder,
  3121. WUQuerySortField *filters,
  3122. const void *filterbuf,
  3123. unsigned startoffset,
  3124. unsigned maxnum,
  3125. __int64 *cachehint,
  3126. unsigned *total,
  3127. const MapStringTo<bool> *subset)
  3128. {
  3129. return factory->getQuerySetQueriesSorted(sortorder,filters,filterbuf,startoffset,maxnum,cachehint,total, subset);
  3130. }
  3131. virtual unsigned numWorkUnits()
  3132. {
  3133. return factory->numWorkUnits();
  3134. }
  3135. virtual unsigned numWorkUnitsFiltered(WUSortField *filters,
  3136. const void *filterbuf)
  3137. {
  3138. return factory->numWorkUnitsFiltered(filters,filterbuf,secMgr.get(), secUser.get());
  3139. }
  3140. private:
  3141. Owned<CWorkUnitFactory> base_factory;
  3142. Owned<ISecManager> secMgr;
  3143. Owned<ISecUser> secUser;
  3144. };
  3145. extern WORKUNIT_API IWorkUnitFactory * getSecWorkUnitFactory(ISecManager &secmgr, ISecUser &secuser)
  3146. {
  3147. return new CSecureWorkUnitFactory(secmgr, secuser);
  3148. }
  3149. extern WORKUNIT_API IWorkUnitFactory * getWorkUnitFactory(ISecManager *secmgr, ISecUser *secuser)
  3150. {
  3151. if (secmgr && secuser)
  3152. return getSecWorkUnitFactory(*secmgr, *secuser);
  3153. else
  3154. return getWorkUnitFactory();
  3155. }
  3156. //==========================================================================================
  3157. class CStringPTreeIterator : public CInterface, implements IStringIterator
  3158. {
  3159. Owned<IPropertyTreeIterator> it;
  3160. public:
  3161. IMPLEMENT_IINTERFACE;
  3162. CStringPTreeIterator(IPropertyTreeIterator *p) : it(p) {};
  3163. virtual bool first() { return it->first(); }
  3164. virtual bool next() { return it->next(); }
  3165. virtual bool isValid() { return it->isValid(); }
  3166. virtual IStringVal & str(IStringVal &s) { s.set(it->query().queryProp(NULL)); return s; }
  3167. };
  3168. class CStringPTreeTagIterator : public CInterface, implements IStringIterator
  3169. {
  3170. Owned<IPropertyTreeIterator> it;
  3171. public:
  3172. IMPLEMENT_IINTERFACE;
  3173. CStringPTreeTagIterator(IPropertyTreeIterator *p) : it(p) {};
  3174. virtual bool first() { return it->first(); }
  3175. virtual bool next() { return it->next(); }
  3176. virtual bool isValid() { return it->isValid(); }
  3177. virtual IStringVal & str(IStringVal &s) { s.set(it->query().queryName()); return s; }
  3178. };
  3179. class CStringPTreeAttrIterator : public CInterface, implements IStringIterator
  3180. {
  3181. Owned<IPropertyTreeIterator> it;
  3182. StringAttr name;
  3183. public:
  3184. IMPLEMENT_IINTERFACE;
  3185. CStringPTreeAttrIterator(IPropertyTreeIterator *p, const char *_name) : it(p), name(_name) {};
  3186. virtual bool first() { return it->first(); }
  3187. virtual bool next() { return it->next(); }
  3188. virtual bool isValid() { return it->isValid(); }
  3189. virtual IStringVal & str(IStringVal &s) { s.set(it->query().queryProp(name)); return s; }
  3190. };
  3191. //==========================================================================================
  3192. CLocalWorkUnit::CLocalWorkUnit(IPropertyTree* root, ISecManager *secmgr, ISecUser *secuser)
  3193. {
  3194. p.setown(root);
  3195. secMgr.set(secmgr);
  3196. secUser.set(secuser);
  3197. }
  3198. void CLocalWorkUnit::init()
  3199. {
  3200. p.clear();
  3201. cachedGraphs.clear();
  3202. workflowIterator.clear();
  3203. query.clear();
  3204. graphs.kill();
  3205. results.kill();
  3206. variables.kill();
  3207. plugins.kill();
  3208. libraries.kill();
  3209. exceptions.kill();
  3210. temporaries.kill();
  3211. roxieQueryInfo.clear();
  3212. webServicesInfo.clear();
  3213. workflowIteratorCached = false;
  3214. resultsCached = false;
  3215. temporariesCached = false;
  3216. variablesCached = false;
  3217. exceptionsCached = false;
  3218. pluginsCached = false;
  3219. librariesCached = false;
  3220. activitiesCached = false;
  3221. webServicesInfoCached = false;
  3222. roxieQueryInfoCached = false;
  3223. }
  3224. // Dummy workunit support
  3225. CLocalWorkUnit::CLocalWorkUnit(const char *_wuid, const char *xml, ISecManager *secmgr, ISecUser *secuser)
  3226. {
  3227. init();
  3228. if (xml)
  3229. p.setown(createPTreeFromXMLString(xml));
  3230. else
  3231. {
  3232. p.setown(createPTree(_wuid));
  3233. p->setProp("@xmlns:xsi", "http://www.w3.org/1999/XMLSchema-instance");
  3234. }
  3235. secMgr.set(secmgr);
  3236. secUser.set(secuser);
  3237. }
  3238. void CLocalWorkUnit::beforeDispose()
  3239. {
  3240. try
  3241. {
  3242. unsubscribe();
  3243. query.clear();
  3244. webServicesInfo.clear();
  3245. roxieQueryInfo.clear();
  3246. workflowIterator.clear();
  3247. plugins.kill();
  3248. libraries.kill();
  3249. exceptions.kill();
  3250. graphs.kill();
  3251. results.kill();
  3252. temporaries.kill();
  3253. variables.kill();
  3254. appvalues.kill();
  3255. statistics.kill();
  3256. userDesc.clear();
  3257. secMgr.clear();
  3258. secUser.clear();
  3259. cachedGraphs.clear();
  3260. p.clear();
  3261. }
  3262. catch (IException *E) { LOG(MCexception(E, MSGCLS_warning), E, "Exception during ~CLocalWorkUnit"); E->Release(); }
  3263. }
  3264. void CLocalWorkUnit::cleanupAndDelete(bool deldll, bool deleteOwned, const StringArray *deleteExclusions)
  3265. {
  3266. TIME_SECTION("WUDELETE cleanupAndDelete total");
  3267. // Delete any related things in SDS etc that might otherwise be forgotten
  3268. if (p->getPropBool("@protected", false))
  3269. throw MakeStringException(WUERR_WorkunitProtected, "%s: Workunit is protected",p->queryName());
  3270. switch (getState())
  3271. {
  3272. case WUStateAborted:
  3273. case WUStateCompleted:
  3274. case WUStateFailed:
  3275. case WUStateArchived:
  3276. break;
  3277. case WUStateCompiled:
  3278. if (getAction()==WUActionRun || getAction()==WUActionUnknown)
  3279. throw MakeStringException(WUERR_WorkunitActive, "%s: Workunit is active",p->queryName());
  3280. break;
  3281. case WUStateWait:
  3282. throw MakeStringException(WUERR_WorkunitScheduled, "%s: Workunit is scheduled",p->queryName());
  3283. default:
  3284. throw MakeStringException(WUERR_WorkunitActive, "%s: Workunit is active",p->queryName());
  3285. break;
  3286. }
  3287. if (getIsQueryService())
  3288. {
  3289. Owned<IPropertyTree> registry = getQueryRegistryRoot();
  3290. if (registry)
  3291. {
  3292. VStringBuffer xpath("QuerySet/Query[@wuid='%s']", p->queryName());
  3293. if (registry->hasProp(xpath.str()))
  3294. throw MakeStringException(WUERR_WorkunitPublished, "%s: Workunit is published",p->queryName());
  3295. }
  3296. }
  3297. try
  3298. {
  3299. if (deldll && !p->getPropBool("@isClone", false))
  3300. {
  3301. Owned<IConstWUQuery> q = getQuery();
  3302. if (q)
  3303. {
  3304. Owned<IConstWUAssociatedFileIterator> iter = &q->getAssociatedFiles();
  3305. SCMStringBuffer name;
  3306. ForEach(*iter)
  3307. {
  3308. IConstWUAssociatedFile & cur = iter->query();
  3309. cur.getNameTail(name);
  3310. if (!deleteExclusions || (NotFound == deleteExclusions->find(name.str())))
  3311. {
  3312. Owned<IDllEntry> entry = queryDllServer().getEntry(name.str());
  3313. if (entry.get())
  3314. factory->asyncRemoveDll(name.str());
  3315. else
  3316. {
  3317. SCMStringBuffer ip, localPath;
  3318. cur.getName(localPath);
  3319. cur.getIp(ip);
  3320. factory->asyncRemoveFile(ip.str(), localPath.str());
  3321. }
  3322. }
  3323. }
  3324. }
  3325. }
  3326. StringBuffer apath;
  3327. // PROGLOG("wuid dll files removed");
  3328. {
  3329. apath.append("/WorkUnitAborts/").append(p->queryName());
  3330. Owned<IRemoteConnection> acon = factory->sdsManager->connect(apath.str(), factory->session, RTM_LOCK_WRITE | RTM_DELETE_ON_DISCONNECT, SDS_LOCK_TIMEOUT);
  3331. acon.clear();
  3332. }
  3333. // PROGLOG("wuid WorkUnitAborts entry removed");
  3334. deleteTempFiles(NULL, deleteOwned, true); // all, any remaining.
  3335. }
  3336. catch(IException *E)
  3337. {
  3338. StringBuffer s;
  3339. LOG(MCexception(E, MSGCLS_warning), E, s.append("Exception during cleanupAndDelete: ").append(p->queryName()).str());
  3340. E->Release();
  3341. }
  3342. catch (...)
  3343. {
  3344. WARNLOG("Unknown exception during cleanupAndDelete: %s", p->queryName());
  3345. }
  3346. CConstGraphProgress::deleteWuidProgress(p->queryName());
  3347. }
  3348. void CLocalWorkUnit::setTimeScheduled(const IJlibDateTime &val)
  3349. {
  3350. SCMStringBuffer strval;
  3351. val.getGmtString(strval);
  3352. p->setProp("@timescheduled",strval.str());
  3353. }
  3354. IJlibDateTime & CLocalWorkUnit::getTimeScheduled(IJlibDateTime &val) const
  3355. {
  3356. StringBuffer str;
  3357. p->getProp("@timescheduled",str);
  3358. if(str.length())
  3359. val.setGmtString(str.str());
  3360. return val;
  3361. }
  3362. bool modifyAndWriteWorkUnitXML(char const * wuid, StringBuffer & buf, StringBuffer & extra, IFileIO * fileio)
  3363. {
  3364. // kludge in extra chunks of XML such as GraphProgress and GeneratedDlls
  3365. if(extra.length())
  3366. {
  3367. size32_t l = (size32_t)strlen(wuid);
  3368. size32_t p = buf.length()-l-4; // bit of a kludge
  3369. assertex(memcmp(buf.str()+p+2,wuid,l)==0);
  3370. StringAttr tail(buf.str()+p);
  3371. buf.setLength(p);
  3372. buf.append(extra);
  3373. buf.append(tail);
  3374. }
  3375. return (fileio->write(0,buf.length(),buf.str()) == buf.length());
  3376. }
  3377. bool CLocalWorkUnit::archiveWorkUnit(const char *base,bool del,bool ignoredllerrors,bool deleteOwned)
  3378. {
  3379. CriticalBlock block(crit);
  3380. StringBuffer path(base);
  3381. if (!p)
  3382. return false;
  3383. const char *wuid = p->queryName();
  3384. if (!wuid||!*wuid)
  3385. return false;
  3386. addPathSepChar(path).append(wuid).append(".xml");
  3387. Owned<IFile> file = createIFile(path.str());
  3388. if (!file)
  3389. return false;
  3390. Owned<IFileIO> fileio = file->open(IFOcreate);
  3391. if (!fileio)
  3392. return false;
  3393. StringBuffer buf;
  3394. exportWorkUnitToXML(this, buf, false, false);
  3395. StringBuffer extraWorkUnitXML;
  3396. StringBuffer xpath("/GraphProgress/");
  3397. xpath.append(wuid);
  3398. Owned<IRemoteConnection> conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
  3399. if (conn)
  3400. {
  3401. Owned<IPropertyTree> tmp = createPTree("GraphProgress");
  3402. mergePTree(tmp,conn->queryRoot());
  3403. toXML(tmp,extraWorkUnitXML,1,XML_Format);
  3404. conn->close();
  3405. }
  3406. Owned<IConstWUQuery> q = getQuery();
  3407. if (!q)
  3408. {
  3409. if (!modifyAndWriteWorkUnitXML(wuid, buf, extraWorkUnitXML, fileio))
  3410. return false;
  3411. if (del)
  3412. {
  3413. if (getState()==WUStateUnknown)
  3414. setState(WUStateArchived); // to allow delete
  3415. cleanupAndDelete(false,deleteOwned); // no query, may as well delete
  3416. }
  3417. return false;
  3418. }
  3419. StringArray deleteExclusions; // associated files not to delete, added if failure to copy
  3420. Owned<IConstWUAssociatedFileIterator> iter = &q->getAssociatedFiles();
  3421. Owned<IPropertyTree> generatedDlls = createPTree("GeneratedDlls");
  3422. ForEach(*iter)
  3423. {
  3424. IConstWUAssociatedFile & cur = iter->query();
  3425. SCMStringBuffer name;
  3426. cur.getNameTail(name);
  3427. if (name.length())
  3428. {
  3429. Owned<IDllEntry> entry = queryDllServer().getEntry(name.str());
  3430. SCMStringBuffer curPath, curIp;
  3431. cur.getName(curPath);
  3432. cur.getIp(curIp);
  3433. SocketEndpoint curEp(curIp.str());
  3434. RemoteFilename curRfn;
  3435. curRfn.setPath(curEp, curPath.str());
  3436. StringBuffer dst(base);
  3437. addPathSepChar(dst);
  3438. curRfn.getTail(dst);
  3439. Owned<IFile> dstFile = createIFile(dst.str());
  3440. if (entry.get())
  3441. {
  3442. Owned<IException> exception;
  3443. Owned<IDllLocation> loc;
  3444. Owned<IPropertyTree> generatedDllBranch = createPTree();
  3445. generatedDllBranch->setProp("@name", entry->queryName());
  3446. generatedDllBranch->setProp("@kind", entry->queryKind());
  3447. try
  3448. {
  3449. loc.setown(entry->getBestLocation()); //throws exception if no readable locations
  3450. }
  3451. catch(IException * e)
  3452. {
  3453. exception.setown(e);
  3454. loc.setown(entry->getBestLocationCandidate()); //this will be closest of the unreadable locations
  3455. }
  3456. RemoteFilename filename;
  3457. loc->getDllFilename(filename);
  3458. if (!exception)
  3459. {
  3460. Owned<IFile> srcfile = createIFile(filename);
  3461. try
  3462. {
  3463. if (dstFile->exists())
  3464. {
  3465. if (streq(srcfile->queryFilename(), dstFile->queryFilename()))
  3466. deleteExclusions.append(name.str()); // restored workunit, referencing archive location for query dll (no longer true post HPCC-11191 fix)
  3467. // still want to delete if already archived but there are source file copies
  3468. }
  3469. else
  3470. copyFile(dstFile, srcfile);
  3471. }
  3472. catch(IException * e)
  3473. {
  3474. exception.setown(e);
  3475. }
  3476. }
  3477. if (exception)
  3478. {
  3479. if (ignoredllerrors)
  3480. {
  3481. EXCLOG(exception.get(), "archiveWorkUnit (copying associated file)");
  3482. //copy failed, so don't delete the registred dll files
  3483. deleteExclusions.append(name.str());
  3484. }
  3485. else
  3486. throw exception.getClear();
  3487. }
  3488. // Record Associated path to restore back to
  3489. StringBuffer restorePath;
  3490. curRfn.getRemotePath(restorePath);
  3491. generatedDllBranch->setProp("@location", restorePath.str());
  3492. generatedDlls->addPropTree("GeneratedDll", generatedDllBranch.getClear());
  3493. }
  3494. else // no generated dll entry
  3495. {
  3496. Owned<IFile> srcFile = createIFile(curRfn);
  3497. try
  3498. {
  3499. copyFile(dstFile, srcFile);
  3500. }
  3501. catch (IException *e)
  3502. {
  3503. VStringBuffer msg("Failed to archive associated file '%s' to destination '%s'", srcFile->queryFilename(), dstFile->queryFilename());
  3504. EXCLOG(e, msg.str());
  3505. e->Release();
  3506. deleteExclusions.append(name.str());
  3507. }
  3508. }
  3509. }
  3510. }
  3511. iter.clear();
  3512. if (generatedDlls->numChildren())
  3513. toXML(generatedDlls, extraWorkUnitXML, 1, XML_Format);
  3514. if (!modifyAndWriteWorkUnitXML(wuid, buf, extraWorkUnitXML, fileio))
  3515. return false;
  3516. if (del)
  3517. {
  3518. //setState(WUStateArchived); // this isn't useful as about to delete it!
  3519. q.clear();
  3520. cleanupAndDelete(true, deleteOwned, &deleteExclusions);
  3521. }
  3522. return true;
  3523. }
  3524. void CLocalWorkUnit::packWorkUnit(bool pack)
  3525. {
  3526. // only packs Graph info currently
  3527. CriticalBlock block(crit);
  3528. if (!p)
  3529. return;
  3530. const char *wuid = p->queryName();
  3531. if (!wuid||!*wuid)
  3532. return;
  3533. if (pack) {
  3534. if (!p->hasProp("PackedGraphs")) {
  3535. cachedGraphs.clear();
  3536. IPropertyTree *t = p->queryPropTree("Graphs");
  3537. if (t) {
  3538. MemoryBuffer buf;
  3539. t->serialize(buf);
  3540. p->setPropBin("PackedGraphs",buf.length(),buf.bufferBase());
  3541. p->removeTree(t);
  3542. }
  3543. }
  3544. }
  3545. else {
  3546. ensureGraphsUnpacked();
  3547. }
  3548. CConstGraphProgress::packProgress(wuid,pack);
  3549. }
  3550. IPropertyTree * pruneBranch(IPropertyTree * from, char const * xpath)
  3551. {
  3552. Owned<IPropertyTree> ret;
  3553. IPropertyTree * branch = from->queryPropTree(xpath);
  3554. if(branch) {
  3555. ret.setown(createPTreeFromIPT(branch));
  3556. from->removeTree(branch);
  3557. }
  3558. return ret.getClear();
  3559. }
  3560. bool restoreWorkUnit(const char *base,const char *wuid)
  3561. {
  3562. StringBuffer path(base);
  3563. if (!wuid||!*wuid)
  3564. return false;
  3565. addPathSepChar(path).append(wuid).append(".xml");
  3566. Owned<IFile> file = createIFile(path.str());
  3567. if (!file)
  3568. return false;
  3569. Owned<IFileIO> fileio = file->open(IFOread);
  3570. if (!fileio)
  3571. return false;
  3572. Owned<IPropertyTree> pt = createPTree(*fileio);
  3573. if (!pt)
  3574. return false;
  3575. CDateTime dt;
  3576. dt.setNow();
  3577. StringBuffer dts;
  3578. dt.getString(dts);
  3579. pt->setProp("@restoredDate", dts.str());
  3580. VStringBuffer xpath("/WorkUnits/%s", wuid);
  3581. Owned<IRemoteConnection> conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
  3582. if (!conn)
  3583. {
  3584. ERRLOG("restoreWorkUnit could not create to %s", xpath.str());
  3585. return false;
  3586. }
  3587. IPropertyTree *root = conn->queryRoot();
  3588. if (root->hasChildren())
  3589. {
  3590. ERRLOG("restoreWorkUnit WUID %s already exists", wuid);
  3591. return false;
  3592. }
  3593. Owned<IPropertyTree> gprogress = pruneBranch(pt, "GraphProgress[1]");
  3594. Owned<IPropertyTree> generatedDlls = pruneBranch(pt, "GeneratedDlls[1]");
  3595. Owned<IPropertyTree> associatedFiles;
  3596. IPropertyTree *srcAssociated = pt->queryPropTree("Query/Associated");
  3597. if (srcAssociated)
  3598. associatedFiles.setown(createPTreeFromIPT(srcAssociated));
  3599. root->setPropTree(NULL, pt.getClear());
  3600. conn.clear();
  3601. // now kludge back GraphProgress and GeneratedDlls
  3602. if (gprogress)
  3603. {
  3604. VStringBuffer xpath("/GraphProgress/%s", wuid);
  3605. conn.setown(querySDS().connect(xpath, myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT));
  3606. if (conn)
  3607. {
  3608. IPropertyTree *groot = conn->queryRoot();
  3609. if (groot->hasChildren())
  3610. WARNLOG("restoreWorkUnit WUID %s graphprogress already exists, replacing",wuid);
  3611. groot->setPropTree(NULL, gprogress.getClear());
  3612. }
  3613. }
  3614. if (generatedDlls)
  3615. {
  3616. Owned<IPropertyTreeIterator> dlls = generatedDlls->getElements("GeneratedDll");
  3617. for(dlls->first(); dlls->isValid(); dlls->next())
  3618. {
  3619. IPropertyTree & dll = dlls->query();
  3620. char const * name = dll.queryProp("@name");
  3621. char const * kind = dll.queryProp("@kind");
  3622. char const * location = dll.queryProp("@location");
  3623. Owned<IDllEntry> got = queryDllServer().getEntry(name);
  3624. if (!got)
  3625. {
  3626. RemoteFilename dstRfn;
  3627. dstRfn.setRemotePath(location);
  3628. StringBuffer srcPath(base);
  3629. addPathSepChar(srcPath);
  3630. dstRfn.getTail(srcPath);
  3631. OwnedIFile srcFile = createIFile(srcPath);
  3632. OwnedIFile dstFile = createIFile(dstRfn);
  3633. copyFile(dstFile, srcFile);
  3634. queryDllServer().registerDll(name, kind, location);
  3635. }
  3636. }
  3637. }
  3638. if (associatedFiles)
  3639. {
  3640. Owned<IPropertyTreeIterator> associated = associatedFiles->getElements("*");
  3641. ForEach(*associated)
  3642. {
  3643. IPropertyTree &file = associated->query();
  3644. const char *filename = file.queryProp("@filename");
  3645. SocketEndpoint ep(file.queryProp("@ip"));
  3646. RemoteFilename rfn;
  3647. rfn.setPath(ep, filename);
  3648. OwnedIFile dstFile = createIFile(rfn);
  3649. StringBuffer srcPath(base), name;
  3650. addPathSepChar(srcPath);
  3651. rfn.getTail(name);
  3652. srcPath.append(name);
  3653. if (generatedDlls)
  3654. {
  3655. VStringBuffer gDllPath("GeneratedDll[@name=\"%s\"]", name.str());
  3656. if (generatedDlls->hasProp(gDllPath))
  3657. continue; // generated dlls handled separately - see above
  3658. }
  3659. OwnedIFile srcFile = createIFile(srcPath);
  3660. if (srcFile->exists())
  3661. {
  3662. try
  3663. {
  3664. copyFile(dstFile, srcFile);
  3665. }
  3666. catch (IException *e)
  3667. {
  3668. VStringBuffer msg("Failed to restore associated file '%s' to destination '%s'", srcFile->queryFilename(), dstFile->queryFilename());
  3669. EXCLOG(e, msg.str());
  3670. e->Release();
  3671. }
  3672. }
  3673. }
  3674. }
  3675. return true;
  3676. }
  3677. void CLocalWorkUnit::loadXML(const char *xml)
  3678. {
  3679. CriticalBlock block(crit);
  3680. init();
  3681. assertex(xml);
  3682. p.setown(createPTreeFromXMLString(xml));
  3683. }
  3684. void CLocalWorkUnit::serialize(MemoryBuffer &tgt)
  3685. {
  3686. CriticalBlock block(crit);
  3687. StringBuffer x;
  3688. tgt.append(exportWorkUnitToXML(this, x, false, false).str());
  3689. }
  3690. void CLocalWorkUnit::deserialize(MemoryBuffer &src)
  3691. {
  3692. CriticalBlock block(crit);
  3693. StringAttr value;
  3694. src.read(value);
  3695. loadXML(value);
  3696. }
  3697. void CLocalWorkUnit::requestAbort()
  3698. {
  3699. CriticalBlock block(crit);
  3700. abortWorkUnit(p->queryName());
  3701. }
  3702. void CLocalWorkUnit::subscribe(WUSubscribeOptions options)
  3703. {
  3704. }
  3705. void CLocalWorkUnit::unsubscribe()
  3706. {
  3707. }
  3708. void CLocalWorkUnit::unlockRemote()
  3709. {
  3710. CriticalBlock block(crit);
  3711. locked.unlock();
  3712. if (IsShared()) // Is this right? Doesn't feel right! Commit on last unlock would seem smarter
  3713. {
  3714. _unlockRemote();
  3715. }
  3716. }
  3717. IWorkUnit &CLocalWorkUnit::lockRemote(bool commit)
  3718. {
  3719. if (secMgr)
  3720. checkWuSecAccess(*this, *secMgr.get(), secUser.get(), SecAccess_Write, "write lock", true, true);
  3721. locked.lock();
  3722. CriticalBlock block(crit);
  3723. if (commit)
  3724. {
  3725. try
  3726. {
  3727. _lockRemote();
  3728. }
  3729. catch (IException *E)
  3730. {
  3731. StringBuffer s;
  3732. PrintLog("Failed to get write lock on workunit: %s", E->errorMessage(s).str());
  3733. locked.unlock();
  3734. throw;
  3735. }
  3736. }
  3737. return *new CLockedWorkUnit(LINK(this));
  3738. }
  3739. void CLocalWorkUnit::commit()
  3740. {
  3741. // Nothing to do if not backed by a persistent store
  3742. }
  3743. IWorkUnit& CLocalWorkUnit::lock()
  3744. {
  3745. return lockRemote(true);
  3746. }
  3747. const char *CLocalWorkUnit::queryWuid() const
  3748. {
  3749. CriticalBlock block(crit);
  3750. return p->queryName();
  3751. }
  3752. unsigned CLocalWorkUnit::getDebugAgentListenerPort() const
  3753. {
  3754. CriticalBlock block(crit);
  3755. return p->getPropInt("@DebugListenerPort", 0);
  3756. }
  3757. void CLocalWorkUnit::setDebugAgentListenerPort(unsigned port)
  3758. {
  3759. CriticalBlock block(crit);
  3760. p->setPropInt("@DebugListenerPort", port);
  3761. }
  3762. IStringVal& CLocalWorkUnit::getDebugAgentListenerIP(IStringVal &ip) const
  3763. {
  3764. CriticalBlock block(crit);
  3765. ip.set(p->queryProp("@DebugListenerIP"));
  3766. return ip;
  3767. }
  3768. void CLocalWorkUnit::setDebugAgentListenerIP(const char * ip)
  3769. {
  3770. CriticalBlock block(crit);
  3771. p->setProp("@DebugListenerIP", ip);
  3772. }
  3773. IStringVal& CLocalWorkUnit::getSecurityToken(IStringVal &str) const
  3774. {
  3775. CriticalBlock block(crit);
  3776. str.set(p->queryProp("@token"));
  3777. return str;
  3778. }
  3779. void CLocalWorkUnit::setSecurityToken(const char *value)
  3780. {
  3781. CriticalBlock block(crit);
  3782. p->setProp("@token", value);
  3783. }
  3784. bool CLocalWorkUnit::getRunningGraph(IStringVal &graphName, WUGraphIDType &subId) const
  3785. {
  3786. return CConstGraphProgress::getRunningGraph(p->queryName(), graphName, subId);
  3787. }
  3788. void CLocalWorkUnit::setJobName(const char *value)
  3789. {
  3790. CriticalBlock block(crit);
  3791. p->setProp("@jobName", value);
  3792. }
  3793. const char *CLocalWorkUnit::queryJobName() const
  3794. {
  3795. CriticalBlock block(crit);
  3796. const char *ret = p->queryProp("@jobName");
  3797. if (!ret)
  3798. ret = "";
  3799. return ret;
  3800. }
  3801. void CLocalWorkUnit::setClusterName(const char *value)
  3802. {
  3803. CriticalBlock block(crit);
  3804. p->setProp("@clusterName", value);
  3805. }
  3806. const char *CLocalWorkUnit::queryClusterName() const
  3807. {
  3808. CriticalBlock block(crit);
  3809. const char *ret = p->queryProp("@clusterName");
  3810. if (!ret)
  3811. ret = "";
  3812. return ret;
  3813. }
  3814. void CLocalWorkUnit::setAllowedClusters(const char *value)
  3815. {
  3816. setDebugValue("allowedclusters",value, true);
  3817. }
  3818. IStringVal& CLocalWorkUnit::getAllowedClusters(IStringVal &str) const
  3819. {
  3820. CriticalBlock block(crit);
  3821. getDebugValue("allowedclusters",str);
  3822. if (str.length()!=0)
  3823. return str;
  3824. str.set(p->queryProp("@clusterName"));
  3825. return str;
  3826. }
  3827. void CLocalWorkUnit::setAllowAutoQueueSwitch(bool val)
  3828. {
  3829. setDebugValueInt("allowautoqueueswitch",val?1:0,true);
  3830. }
  3831. bool CLocalWorkUnit::getAllowAutoQueueSwitch() const
  3832. {
  3833. CriticalBlock block(crit);
  3834. return getDebugValueBool("allowautoqueueswitch",false);
  3835. }
  3836. void CLocalWorkUnit::setLibraryInformation(const char * name, unsigned interfaceHash, unsigned definitionHash)
  3837. {
  3838. StringBuffer suffix;
  3839. if (name && *name)
  3840. setApplicationValue("LibraryModule", "name", name, true);
  3841. setApplicationValueInt("LibraryModule", "interfaceHash", interfaceHash, true);
  3842. setApplicationValueInt("LibraryModule", "definitionHash", definitionHash, true);
  3843. setApplicationValue("LibraryModule", "platform", appendLibrarySuffix(suffix).str(), true);
  3844. }
  3845. void CLocalWorkUnit::remoteCheckAccess(IUserDescriptor *user, bool writeaccess) const
  3846. {
  3847. unsigned auditflags = DALI_LDAP_AUDIT_REPORT|DALI_LDAP_READ_WANTED;
  3848. if (writeaccess)
  3849. auditflags |= DALI_LDAP_WRITE_WANTED;
  3850. int perm = 255;
  3851. const char *scopename = p->queryProp("@scope");
  3852. if (scopename&&*scopename) {
  3853. if (!user)
  3854. user = queryUserDescriptor();
  3855. perm = querySessionManager().getPermissionsLDAP("workunit",scopename,user,auditflags);
  3856. if (perm<0) {
  3857. if (perm==-1)
  3858. perm = 255;
  3859. else
  3860. perm = 0;
  3861. }
  3862. }
  3863. if (!HASREADPERMISSION(perm))
  3864. throw MakeStringException(WUERR_WorkunitAccessDenied, "Read access denied for workunit %s", queryWuid());
  3865. if (writeaccess && !HASWRITEPERMISSION(perm))
  3866. throw MakeStringException(WUERR_WorkunitAccessDenied, "Write access denied for workunit %s", queryWuid());
  3867. }
  3868. void CLocalWorkUnit::setUser(const char * value)
  3869. {
  3870. CriticalBlock block(crit);
  3871. p->setProp("@submitID", value);
  3872. }
  3873. const char *CLocalWorkUnit::queryUser() const
  3874. {
  3875. CriticalBlock block(crit);
  3876. const char *ret = p->queryProp("@submitID");
  3877. if (!ret)
  3878. ret = "";
  3879. return ret;
  3880. }
  3881. void CLocalWorkUnit::setWuScope(const char * value)
  3882. {
  3883. CriticalBlock block(crit);
  3884. p->setProp("@scope", value);
  3885. }
  3886. IStringVal& CLocalWorkUnit::getWuScope(IStringVal &str) const
  3887. {
  3888. CriticalBlock block(crit);
  3889. str.set(p->queryProp("@scope"));
  3890. return str;
  3891. }
  3892. mapEnums priorityClasses[] = {
  3893. { PriorityClassUnknown, "unknown" },
  3894. { PriorityClassLow, "low" },
  3895. { PriorityClassNormal, "normal" },
  3896. { PriorityClassHigh, "high" },
  3897. { PriorityClassSize, NULL },
  3898. };
  3899. void CLocalWorkUnit::setPriority(WUPriorityClass cls)
  3900. {
  3901. CriticalBlock block(crit);
  3902. setEnum(p, "@priorityClass", cls, priorityClasses);
  3903. }
  3904. WUPriorityClass CLocalWorkUnit::getPriority() const
  3905. {
  3906. CriticalBlock block(crit);
  3907. return (WUPriorityClass) getEnum(p, "@priorityClass", priorityClasses);
  3908. }
  3909. mapEnums states[] = {
  3910. { WUStateUnknown, "unknown" },
  3911. { WUStateCompiled, "compiled" },
  3912. { WUStateRunning, "running" },
  3913. { WUStateCompleted, "completed" },
  3914. { WUStateFailed, "failed" },
  3915. { WUStateArchived, "archived" },
  3916. { WUStateAborting, "aborting" },
  3917. { WUStateAborted, "aborted" },
  3918. { WUStateBlocked, "blocked" },
  3919. { WUStateSubmitted, "submitted" },
  3920. { WUStateScheduled, "scheduled" },
  3921. { WUStateCompiling, "compiling" },
  3922. { WUStateWait, "wait" },
  3923. { WUStateUploadingFiles, "uploading_files" },
  3924. { WUStateDebugPaused, "debugging" },
  3925. { WUStateDebugRunning, "debug_running" },
  3926. { WUStatePaused, "paused" },
  3927. { WUStateSize, NULL }
  3928. };
  3929. const char * getWorkunitStateStr(WUState state)
  3930. {
  3931. return states[state].str;
  3932. }
  3933. IConstWorkUnitIterator * CWorkUnitFactory::getWorkUnitsByState(WUState state)
  3934. {
  3935. StringBuffer path("*");
  3936. path.append("[@state=\"").append(getEnumText(state, states)).append("\"]");
  3937. return getWorkUnitsByXPath(path.str());
  3938. }
  3939. IConstWorkUnitIterator * CSecureWorkUnitFactory::getWorkUnitsByState(WUState state)
  3940. {
  3941. StringBuffer path("*");
  3942. path.append("[@state=\"").append(getEnumText(state, states)).append("\"]");
  3943. return factory->getWorkUnitsByXPath(path.str(), secMgr.get(), secUser.get());
  3944. }
  3945. void CLocalWorkUnit::setState(WUState value)
  3946. {
  3947. CriticalBlock block(crit);
  3948. if (value==WUStateAborted || value==WUStatePaused || value==WUStateCompleted || value==WUStateFailed || value==WUStateSubmitted || value==WUStateWait)
  3949. {
  3950. unsubscribe();
  3951. StringBuffer apath;
  3952. apath.append("/WorkUnitAborts/").append(p->queryName());
  3953. if(factory)
  3954. {
  3955. Owned<IRemoteConnection> acon = factory->sdsManager->connect(apath.str(), factory->session, RTM_LOCK_WRITE|RTM_LOCK_SUB, SDS_LOCK_TIMEOUT);
  3956. if (acon)
  3957. acon->close(true);
  3958. }
  3959. }
  3960. setEnum(p, "@state", value, states);
  3961. if (getDebugValueBool("monitorWorkunit", false))
  3962. {
  3963. switch(value)
  3964. {
  3965. case WUStateAborted:
  3966. FLLOG(MCoperatorWarning, "Workunit %s aborted", p->queryName());
  3967. break;
  3968. case WUStateCompleted:
  3969. FLLOG(MCoperatorProgress, "Workunit %s completed", p->queryName());
  3970. break;
  3971. case WUStateFailed:
  3972. FLLOG(MCoperatorProgress, "Workunit %s failed", p->queryName());
  3973. break;
  3974. }
  3975. }
  3976. p->removeProp("@stateEx");
  3977. }
  3978. void CLocalWorkUnit::setStateEx(const char * text)
  3979. {
  3980. CriticalBlock block(crit);
  3981. p->setProp("@stateEx", text);
  3982. }
  3983. void CLocalWorkUnit::setAgentSession(__int64 sessionId)
  3984. {
  3985. CriticalBlock block(crit);
  3986. p->setPropInt64("@agentSession", sessionId);
  3987. }
  3988. bool CLocalWorkUnit::aborting() const
  3989. {
  3990. return false;
  3991. }
  3992. bool CLocalWorkUnit::getIsQueryService() const
  3993. {
  3994. CriticalBlock block(crit);
  3995. return p->getPropBool("@isQueryService", false);
  3996. }
  3997. void CLocalWorkUnit::setIsQueryService(bool value)
  3998. {
  3999. CriticalBlock block(crit);
  4000. p->setPropBool("@isQueryService", value);
  4001. }
  4002. void CLocalWorkUnit::checkAgentRunning(WUState & state)
  4003. {
  4004. if (queryDaliServerVersion().compare("2.1")<0)
  4005. return;
  4006. switch(state)
  4007. {
  4008. case WUStateRunning:
  4009. case WUStateDebugPaused:
  4010. case WUStateDebugRunning:
  4011. case WUStateBlocked:
  4012. case WUStateAborting:
  4013. case WUStateCompiling:
  4014. case WUStatePaused:
  4015. {
  4016. SessionId agent = getAgentSession();
  4017. if((agent>0) && querySessionManager().sessionStopped(agent, 0))
  4018. {
  4019. forceReload();
  4020. state = (WUState) getEnum(p, "@state", states);
  4021. bool isecl=state==WUStateCompiling;
  4022. if (aborting())
  4023. state = WUStateAborted;
  4024. else if (state==WUStateRunning || state==WUStatePaused || state==WUStateDebugPaused || state==WUStateDebugRunning || state==WUStateBlocked || state==WUStateCompiling)
  4025. state = WUStateFailed;
  4026. else
  4027. return;
  4028. WARNLOG("checkAgentRunning terminated: %"I64F"d state = %d",(__int64)agent,(int)state);
  4029. Owned<IWorkUnit> w = &lock();
  4030. w->setState(state);
  4031. Owned<IWUException> e = w->createException();
  4032. WUAction action = w->getAction();
  4033. switch (action)
  4034. {
  4035. case WUActionPause:
  4036. case WUActionPauseNow:
  4037. case WUActionResume:
  4038. w->setAction(WUActionUnknown);
  4039. }
  4040. if(isecl)
  4041. {
  4042. e->setExceptionCode(1001);
  4043. e->setExceptionMessage("EclServer terminated unexpectedly");
  4044. }
  4045. else
  4046. {
  4047. e->setExceptionCode(1000);
  4048. e->setExceptionMessage("Workunit terminated unexpectedly");
  4049. }
  4050. }
  4051. }
  4052. }
  4053. }
  4054. WUState CLocalWorkUnit::getState() const
  4055. {
  4056. CriticalBlock block(crit);
  4057. WUState state = (WUState) getEnum(p, "@state", states);
  4058. switch (state)
  4059. {
  4060. case WUStateRunning:
  4061. case WUStateDebugPaused:
  4062. case WUStateDebugRunning:
  4063. case WUStateBlocked:
  4064. case WUStateCompiling:
  4065. if (aborting())
  4066. state = WUStateAborting;
  4067. break;
  4068. case WUStateSubmitted:
  4069. if (aborting())
  4070. state = WUStateAborted;
  4071. break;
  4072. }
  4073. const_cast<CLocalWorkUnit *>(this)->checkAgentRunning(state); //need const_cast as will change state if agent has died
  4074. return state;
  4075. }
  4076. IStringVal& CLocalWorkUnit::getStateEx(IStringVal & str) const
  4077. {
  4078. CriticalBlock block(crit);
  4079. str.set(p->queryProp("@stateEx"));
  4080. return str;
  4081. }
  4082. __int64 CLocalWorkUnit::getAgentSession() const
  4083. {
  4084. CriticalBlock block(crit);
  4085. return p->getPropInt64("@agentSession", -1);
  4086. }
  4087. unsigned CLocalWorkUnit::getAgentPID() const
  4088. {
  4089. CriticalBlock block(crit);
  4090. return p->getPropInt("@agentPID", -1);
  4091. }
  4092. const char * CLocalWorkUnit::queryStateDesc() const
  4093. {
  4094. // MORE - not sure about this - may prefer a separate interface
  4095. CriticalBlock block(crit);
  4096. try
  4097. {
  4098. return getEnumText(getState(), states);
  4099. }
  4100. catch (...)
  4101. {
  4102. return "???";
  4103. }
  4104. }
  4105. mapEnums actions[] = {
  4106. { WUActionUnknown, "unknown" },
  4107. { WUActionCompile, "compile" },
  4108. { WUActionCheck, "check" },
  4109. { WUActionRun, "run" },
  4110. { WUActionExecuteExisting, "execute" },
  4111. { WUActionPause, "pause" },
  4112. { WUActionPauseNow, "pausenow" },
  4113. { WUActionResume, "resume" },
  4114. { WUActionSize, NULL },
  4115. };
  4116. void CLocalWorkUnit::setAction(WUAction value)
  4117. {
  4118. CriticalBlock block(crit);
  4119. setEnum(p, "Action", value, actions);
  4120. }
  4121. WUAction CLocalWorkUnit::getAction() const
  4122. {
  4123. CriticalBlock block(crit);
  4124. return (WUAction) getEnum(p, "Action", actions);
  4125. }
  4126. IStringVal& CLocalWorkUnit::getActionEx(IStringVal & str) const
  4127. {
  4128. CriticalBlock block(crit);
  4129. str.set(p->queryProp("Action"));
  4130. return str;
  4131. }
  4132. IStringVal& CLocalWorkUnit::getApplicationValue(const char *app, const char *propname, IStringVal &str) const
  4133. {
  4134. CriticalBlock block(crit);
  4135. StringBuffer prop("Application/");
  4136. prop.append(app).append('/').append(propname);
  4137. str.set(p->queryProp(prop.str()));
  4138. return str;
  4139. }
  4140. int CLocalWorkUnit::getApplicationValueInt(const char *app, const char *propname, int defVal) const
  4141. {
  4142. CriticalBlock block(crit);
  4143. StringBuffer prop("Application/");
  4144. prop.append(app).append('/').append(propname);
  4145. return p->getPropInt(prop.str(), defVal);
  4146. }
  4147. IConstWUAppValueIterator& CLocalWorkUnit::getApplicationValues() const
  4148. {
  4149. CriticalBlock block(crit);
  4150. appvalues.load(p,"Application/*");
  4151. return *new CArrayIteratorOf<IConstWUAppValue,IConstWUAppValueIterator> (appvalues, 0, (IConstWorkUnit *) this);
  4152. }
  4153. void CLocalWorkUnit::setApplicationValue(const char *app, const char *propname, const char *value, bool overwrite)
  4154. {
  4155. CriticalBlock block(crit);
  4156. StringBuffer prop("Application/");
  4157. prop.append(app).append('/').append(propname);
  4158. if (overwrite || !p->hasProp(prop.str()))
  4159. {
  4160. // MORE - not sure these lines should be needed....
  4161. StringBuffer sp;
  4162. p->setProp(sp.append("Application").str(), "");
  4163. p->setProp(sp.append('/').append(app).str(), "");
  4164. p->setProp(prop.str(), value);
  4165. }
  4166. }
  4167. void CLocalWorkUnit::setApplicationValueInt(const char *app, const char *propname, int value, bool overwrite)
  4168. {
  4169. CriticalBlock block(crit);
  4170. StringBuffer prop("Application/");
  4171. prop.append(app).append('/').append(propname);
  4172. if (overwrite || !p->hasProp(prop.str()))
  4173. {
  4174. // MORE - not sure these lines should be needed....
  4175. StringBuffer sp;
  4176. p->setProp(sp.append("Application").str(), "");
  4177. p->setProp(sp.append('/').append(app).str(), "");
  4178. p->setPropInt(prop.str(), value);
  4179. }
  4180. }
  4181. void CLocalWorkUnit::setPriorityLevel(int level)
  4182. {
  4183. CriticalBlock block(crit);
  4184. p->setPropInt("PriorityFlag", level);
  4185. }
  4186. int CLocalWorkUnit::getPriorityLevel() const
  4187. {
  4188. CriticalBlock block(crit);
  4189. return p->getPropInt("PriorityFlag");
  4190. }
  4191. int calcPriorityValue(const IPropertyTree * p)
  4192. {
  4193. int priority = p->getPropInt("PriorityFlag");
  4194. switch((WUPriorityClass) getEnum(p, "@priorityClass", priorityClasses))
  4195. {
  4196. case PriorityClassLow:
  4197. priority -= 100;
  4198. break;
  4199. case PriorityClassHigh:
  4200. priority += 100;
  4201. break;
  4202. }
  4203. return priority;
  4204. }
  4205. int CLocalWorkUnit::getPriorityValue() const
  4206. {
  4207. CriticalBlock block(crit);
  4208. return calcPriorityValue(p);
  4209. }
  4210. void CLocalWorkUnit::setRescheduleFlag(bool value)
  4211. {
  4212. CriticalBlock block(crit);
  4213. p->setPropInt("RescheduleFlag", (int) value);
  4214. }
  4215. bool CLocalWorkUnit::getRescheduleFlag() const
  4216. {
  4217. CriticalBlock block(crit);
  4218. return p->getPropInt("RescheduleFlag") != 0;
  4219. }
  4220. class NullIStringIterator : public CInterface, extends IStringIterator
  4221. {
  4222. public:
  4223. IMPLEMENT_IINTERFACE;
  4224. bool first() { return false; }
  4225. bool next() { return false; }
  4226. bool isValid() { return false; }
  4227. IStringVal & str(IStringVal & str) { return str; }
  4228. };
  4229. ClusterType getClusterType(const char * platform, ClusterType dft)
  4230. {
  4231. if (stricmp(platform, "thor") == 0)
  4232. return ThorLCRCluster;
  4233. if (stricmp(platform, "thorlcr") == 0)
  4234. return ThorLCRCluster;
  4235. if (stricmp(platform, "hthor") == 0)
  4236. return HThorCluster;
  4237. if (stricmp(platform, "roxie") == 0)
  4238. return RoxieCluster;
  4239. return dft;
  4240. }
  4241. const char *clusterTypeString(ClusterType clusterType, bool lcrSensitive)
  4242. {
  4243. switch (clusterType)
  4244. {
  4245. case ThorLCRCluster:
  4246. if (lcrSensitive)
  4247. return "thorlcr";
  4248. return "thor";
  4249. case RoxieCluster:
  4250. return "roxie";
  4251. case HThorCluster:
  4252. return "hthor";
  4253. }
  4254. throwUnexpected();
  4255. }
  4256. IPropertyTree *queryRoxieProcessTree(IPropertyTree *environment, const char *process)
  4257. {
  4258. if (!process || !*process)
  4259. return NULL;
  4260. VStringBuffer xpath("Software/RoxieCluster[@name=\"%s\"]", process);
  4261. return environment->queryPropTree(xpath.str());
  4262. }
  4263. void getRoxieProcessServers(IPropertyTree *roxie, SocketEndpointArray &endpoints)
  4264. {
  4265. if (!roxie)
  4266. return;
  4267. Owned<IPropertyTreeIterator> servers = roxie->getElements("RoxieServerProcess");
  4268. ForEach(*servers)
  4269. {
  4270. IPropertyTree &server = servers->query();
  4271. const char *netAddress = server.queryProp("@netAddress");
  4272. if (netAddress && *netAddress)
  4273. {
  4274. SocketEndpoint ep(netAddress, server.getPropInt("@port", 9876));
  4275. endpoints.append(ep);
  4276. }
  4277. }
  4278. }
  4279. void getRoxieProcessServers(const char *process, SocketEndpointArray &servers)
  4280. {
  4281. Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
  4282. Owned<IConstEnvironment> env = factory->openEnvironment();
  4283. if (!env)
  4284. return;
  4285. Owned<IPropertyTree> root = &env->getPTree();
  4286. getRoxieProcessServers(queryRoxieProcessTree(root, process), servers);
  4287. }
  4288. class CEnvironmentClusterInfo: public CInterface, implements IConstWUClusterInfo
  4289. {
  4290. StringAttr name;
  4291. StringAttr serverQueue;
  4292. StringAttr agentQueue;
  4293. StringAttr roxieProcess;
  4294. SocketEndpointArray roxieServers;
  4295. StringAttr thorQueue;
  4296. StringArray thorProcesses;
  4297. StringArray primaryThorProcesses;
  4298. StringAttr prefix;
  4299. StringAttr ldapUser;
  4300. StringBuffer ldapPassword;
  4301. ClusterType platform;
  4302. unsigned clusterWidth;
  4303. public:
  4304. IMPLEMENT_IINTERFACE;
  4305. CEnvironmentClusterInfo(const char *_name, const char *_prefix, IPropertyTree *agent, IArrayOf<IPropertyTree> &thors, IPropertyTree *roxie)
  4306. : name(_name), prefix(_prefix)
  4307. {
  4308. StringBuffer queue;
  4309. if (thors.ordinality())
  4310. {
  4311. thorQueue.set(getClusterThorQueueName(queue.clear(), name));
  4312. clusterWidth = 0;
  4313. bool isMultiThor = (thors.length() > 1);
  4314. ForEachItemIn(i,thors)
  4315. {
  4316. IPropertyTree &thor = thors.item(i);
  4317. const char* thorName = thor.queryProp("@name");
  4318. thorProcesses.append(thorName);
  4319. if (!isMultiThor)
  4320. primaryThorProcesses.append(thorName);
  4321. else
  4322. {
  4323. const char *nodeGroup = thor.queryProp("@nodeGroup");
  4324. if (!nodeGroup || strieq(nodeGroup, thorName))
  4325. primaryThorProcesses.append(thorName);
  4326. }
  4327. unsigned nodes = thor.getCount("ThorSlaveProcess");
  4328. if (!nodes)
  4329. throw MakeStringException(WUERR_MismatchClusterSize,"CEnvironmentClusterInfo: Thor cluster can not have 0 slave processes");
  4330. unsigned ts = nodes * thor.getPropInt("@slavesPerNode", 1);
  4331. if (clusterWidth && (ts!=clusterWidth))
  4332. throw MakeStringException(WUERR_MismatchClusterSize,"CEnvironmentClusterInfo: mismatched thor sizes in cluster");
  4333. clusterWidth = ts;
  4334. bool islcr = !thor.getPropBool("@Legacy");
  4335. if (!islcr)
  4336. throw MakeStringException(WUERR_MismatchThorType,"CEnvironmentClusterInfo: Legacy Thor no longer supported");
  4337. }
  4338. platform = ThorLCRCluster;
  4339. }
  4340. else if (roxie)
  4341. {
  4342. roxieProcess.set(roxie->queryProp("@name"));
  4343. platform = RoxieCluster;
  4344. getRoxieProcessServers(roxie, roxieServers);
  4345. clusterWidth = roxieServers.length();
  4346. ldapUser.set(roxie->queryProp("@ldapUser"));
  4347. StringBuffer encPassword = roxie->queryProp("@ldapPassword");
  4348. if (encPassword.length())
  4349. decrypt(ldapPassword, encPassword);
  4350. }
  4351. else
  4352. {
  4353. clusterWidth = 1;
  4354. platform = HThorCluster;
  4355. }
  4356. if (agent)
  4357. {
  4358. assertex(!roxie);
  4359. agentQueue.set(getClusterEclAgentQueueName(queue.clear(), name));
  4360. }
  4361. else if (roxie)
  4362. agentQueue.set(getClusterRoxieQueueName(queue.clear(), name));
  4363. // MORE - does this need to be conditional?
  4364. serverQueue.set(getClusterEclCCServerQueueName(queue.clear(), name));
  4365. }
  4366. IStringVal & getName(IStringVal & str) const
  4367. {
  4368. str.set(name.get());
  4369. return str;
  4370. }
  4371. IStringVal & getScope(IStringVal & str) const
  4372. {
  4373. str.set(prefix.get());
  4374. return str;
  4375. }
  4376. IStringVal & getAgentQueue(IStringVal & str) const
  4377. {
  4378. str.set(agentQueue);
  4379. return str;
  4380. }
  4381. virtual IStringVal & getServerQueue(IStringVal & str) const
  4382. {
  4383. str.set(serverQueue);
  4384. return str;
  4385. }
  4386. IStringVal & getThorQueue(IStringVal & str) const
  4387. {
  4388. str.set(thorQueue);
  4389. return str;
  4390. }
  4391. unsigned getSize() const
  4392. {
  4393. return clusterWidth;
  4394. }
  4395. virtual ClusterType getPlatform() const
  4396. {
  4397. return platform;
  4398. }
  4399. IStringVal & getRoxieProcess(IStringVal & str) const
  4400. {
  4401. str.set(roxieProcess.get());
  4402. return str;
  4403. }
  4404. const StringArray & getThorProcesses() const
  4405. {
  4406. return thorProcesses;
  4407. }
  4408. const StringArray & getPrimaryThorProcesses() const
  4409. {
  4410. return primaryThorProcesses;
  4411. }
  4412. const SocketEndpointArray & getRoxieServers() const
  4413. {
  4414. return roxieServers;
  4415. }
  4416. const char *getLdapUser() const
  4417. {
  4418. return ldapUser.get();
  4419. }
  4420. virtual const char *getLdapPassword() const
  4421. {
  4422. return ldapPassword.str();
  4423. }
  4424. };
  4425. IStringVal &getProcessQueueNames(IStringVal &ret, const char *process, const char *type, const char *suffix)
  4426. {
  4427. if (process)
  4428. {
  4429. Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
  4430. Owned<IConstEnvironment> env = factory->openEnvironment();
  4431. if (env)
  4432. {
  4433. Owned<IPropertyTree> root = &env->getPTree();
  4434. StringBuffer queueNames;
  4435. StringBuffer xpath;
  4436. xpath.appendf("%s[@process=\"%s\"]", type, process);
  4437. Owned<IPropertyTreeIterator> targets = root->getElements("Software/Topology/Cluster");
  4438. ForEach(*targets)
  4439. {
  4440. IPropertyTree &target = targets->query();
  4441. if (target.hasProp(xpath))
  4442. {
  4443. if (queueNames.length())
  4444. queueNames.append(',');
  4445. queueNames.append(target.queryProp("@name")).append(suffix);
  4446. }
  4447. }
  4448. ret.set(queueNames);
  4449. }
  4450. }
  4451. return ret;
  4452. }
  4453. #define ROXIE_QUEUE_EXT ".roxie"
  4454. #define THOR_QUEUE_EXT ".thor"
  4455. #define ECLCCSERVER_QUEUE_EXT ".eclserver"
  4456. #define ECLSERVER_QUEUE_EXT ECLCCSERVER_QUEUE_EXT
  4457. #define ECLSCHEDULER_QUEUE_EXT ".eclscheduler"
  4458. #define ECLAGENT_QUEUE_EXT ".agent"
  4459. extern WORKUNIT_API IStringVal &getEclCCServerQueueNames(IStringVal &ret, const char *process)
  4460. {
  4461. return getProcessQueueNames(ret, process, "EclCCServerProcess", ECLCCSERVER_QUEUE_EXT);
  4462. }
  4463. extern WORKUNIT_API IStringVal &getEclServerQueueNames(IStringVal &ret, const char *process)
  4464. {
  4465. return getProcessQueueNames(ret, process, "EclServerProcess", ECLSERVER_QUEUE_EXT); // shares queue name with EclCCServer
  4466. }
  4467. extern WORKUNIT_API IStringVal &getEclSchedulerQueueNames(IStringVal &ret, const char *process)
  4468. {
  4469. return getProcessQueueNames(ret, process, "EclSchedulerProcess", ECLSCHEDULER_QUEUE_EXT); // Shares deployment/config with EclCCServer
  4470. }
  4471. extern WORKUNIT_API IStringVal &getAgentQueueNames(IStringVal &ret, const char *process)
  4472. {
  4473. return getProcessQueueNames(ret, process, "EclAgentProcess", ECLAGENT_QUEUE_EXT);
  4474. }
  4475. extern WORKUNIT_API IStringVal &getRoxieQueueNames(IStringVal &ret, const char *process)
  4476. {
  4477. return getProcessQueueNames(ret, process, "RoxieCluster", ROXIE_QUEUE_EXT);
  4478. }
  4479. extern WORKUNIT_API IStringVal &getThorQueueNames(IStringVal &ret, const char *process)
  4480. {
  4481. return getProcessQueueNames(ret, process, "ThorCluster", THOR_QUEUE_EXT);
  4482. }
  4483. extern WORKUNIT_API StringBuffer &getClusterThorQueueName(StringBuffer &ret, const char *cluster)
  4484. {
  4485. return ret.append(cluster).append(THOR_QUEUE_EXT);
  4486. }
  4487. extern WORKUNIT_API StringBuffer &getClusterThorGroupName(StringBuffer &ret, const char *cluster)
  4488. {
  4489. Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
  4490. Owned<IConstEnvironment> env = factory->openEnvironment();
  4491. if (env)
  4492. {
  4493. Owned<IPropertyTree> root = &env->getPTree();
  4494. StringBuffer path;
  4495. path.append("Software/ThorCluster[@name=\"").append(cluster).append("\"]");
  4496. IPropertyTree * child = root->queryPropTree(path);
  4497. if (child)
  4498. getClusterGroupName(*child, ret);
  4499. }
  4500. return ret;
  4501. }
  4502. extern WORKUNIT_API StringBuffer &getClusterRoxieQueueName(StringBuffer &ret, const char *cluster)
  4503. {
  4504. return ret.append(cluster).append(ROXIE_QUEUE_EXT);
  4505. }
  4506. extern WORKUNIT_API StringBuffer &getClusterEclCCServerQueueName(StringBuffer &ret, const char *cluster)
  4507. {
  4508. return ret.append(cluster).append(ECLCCSERVER_QUEUE_EXT);
  4509. }
  4510. extern WORKUNIT_API StringBuffer &getClusterEclServerQueueName(StringBuffer &ret, const char *cluster)
  4511. {
  4512. return ret.append(cluster).append(ECLSERVER_QUEUE_EXT);
  4513. }
  4514. extern WORKUNIT_API StringBuffer &getClusterEclAgentQueueName(StringBuffer &ret, const char *cluster)
  4515. {
  4516. return ret.append(cluster).append(ECLAGENT_QUEUE_EXT);
  4517. }
  4518. extern WORKUNIT_API IStringIterator *getTargetClusters(const char *processType, const char *processName)
  4519. {
  4520. Owned<CStringArrayIterator> ret = new CStringArrayIterator;
  4521. Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
  4522. Owned<IConstEnvironment> env = factory->openEnvironment();
  4523. if (env)
  4524. {
  4525. Owned<IPropertyTree> root = &env->getPTree();
  4526. StringBuffer xpath;
  4527. xpath.appendf("%s", processType ? processType : "*");
  4528. if (processName && *processName)
  4529. xpath.appendf("[@process=\"%s\"]", processName);
  4530. Owned<IPropertyTreeIterator> targets = root->getElements("Software/Topology/Cluster");
  4531. ForEach(*targets)
  4532. {
  4533. IPropertyTree &target = targets->query();
  4534. if (target.hasProp(xpath))
  4535. {
  4536. ret->append(target.queryProp("@name"));
  4537. }
  4538. }
  4539. }
  4540. return ret.getClear();
  4541. }
  4542. extern WORKUNIT_API bool isProcessCluster(const char *process)
  4543. {
  4544. if (!process || !*process)
  4545. return false;
  4546. Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
  4547. Owned<IConstEnvironment> env = factory->openEnvironment();
  4548. if (!env)
  4549. return false;
  4550. Owned<IPropertyTree> root = &env->getPTree();
  4551. VStringBuffer xpath("Software/*Cluster[@name=\"%s\"]", process);
  4552. return root->hasProp(xpath.str());
  4553. }
  4554. extern WORKUNIT_API bool isProcessCluster(const char *remoteDali, const char *process)
  4555. {
  4556. if (!remoteDali || !*remoteDali)
  4557. return isProcessCluster(process);
  4558. if (!process || !*process)
  4559. return false;
  4560. Owned<INode> remote = createINode(remoteDali, 7070);
  4561. if (!remote)
  4562. return false;
  4563. //Cannot use getEnvironmentFactory() since it is using a remotedali
  4564. VStringBuffer xpath("Environment/Software/*Cluster[@name=\"%s\"]/@name", process);
  4565. try
  4566. {
  4567. Owned<IPropertyTreeIterator> clusters = querySDS().getElementsRaw(xpath, remote, 1000*60*1);
  4568. return clusters->first();
  4569. }
  4570. catch (IException *E)
  4571. {
  4572. StringBuffer msg;
  4573. E->errorMessage(msg);
  4574. DBGLOG("Exception validating cluster %s/%s: %s", remoteDali, xpath.str(), msg.str());
  4575. E->Release();
  4576. }
  4577. return true;
  4578. }
  4579. IConstWUClusterInfo* getTargetClusterInfo(IPropertyTree *environment, IPropertyTree *cluster)
  4580. {
  4581. const char *clustname = cluster->queryProp("@name");
  4582. // MORE - at the moment configenf specifies eclagent and thor queues by (in effect) placing an 'example' thor or eclagent in the topology
  4583. // that uses the queue that will be used.
  4584. // We should and I hope will change that, at which point the code below gets simpler
  4585. StringBuffer prefix(cluster->queryProp("@prefix"));
  4586. prefix.toLowerCase();
  4587. StringBuffer xpath;
  4588. StringBuffer querySetName;
  4589. IPropertyTree *agent = NULL;
  4590. const char *agentName = cluster->queryProp("EclAgentProcess/@process");
  4591. if (agentName)
  4592. {
  4593. xpath.clear().appendf("Software/EclAgentProcess[@name=\"%s\"]", agentName);
  4594. agent = environment->queryPropTree(xpath.str());
  4595. }
  4596. Owned<IPropertyTreeIterator> ti = cluster->getElements("ThorCluster");
  4597. IArrayOf<IPropertyTree> thors;
  4598. ForEach(*ti)
  4599. {
  4600. const char *thorName = ti->query().queryProp("@process");
  4601. if (thorName)
  4602. {
  4603. xpath.clear().appendf("Software/ThorCluster[@name=\"%s\"]", thorName);
  4604. thors.append(*environment->getPropTree(xpath.str()));
  4605. }
  4606. }
  4607. const char *roxieName = cluster->queryProp("RoxieCluster/@process");
  4608. return new CEnvironmentClusterInfo(clustname, prefix, agent, thors, queryRoxieProcessTree(environment, roxieName));
  4609. }
  4610. IPropertyTree* getTopologyCluster(Owned<IPropertyTree> &envRoot, const char *clustname)
  4611. {
  4612. if (!clustname || !*clustname)
  4613. return NULL;
  4614. Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
  4615. Owned<IConstEnvironment> env = factory->openEnvironment();
  4616. if (!env)
  4617. return NULL;
  4618. envRoot.setown(&env->getPTree());
  4619. StringBuffer xpath;
  4620. xpath.appendf("Software/Topology/Cluster[@name=\"%s\"]", clustname);
  4621. return envRoot->getPropTree(xpath.str());
  4622. }
  4623. bool validateTargetClusterName(const char *clustname)
  4624. {
  4625. Owned<IPropertyTree> envRoot;
  4626. Owned<IPropertyTree> cluster = getTopologyCluster(envRoot, clustname);
  4627. return (cluster.get()!=NULL);
  4628. }
  4629. IConstWUClusterInfo* getTargetClusterInfo(const char *clustname)
  4630. {
  4631. Owned<IPropertyTree> envRoot;
  4632. Owned<IPropertyTree> cluster = getTopologyCluster(envRoot, clustname);
  4633. if (!cluster)
  4634. return NULL;
  4635. return getTargetClusterInfo(envRoot, cluster);
  4636. }
  4637. unsigned getEnvironmentClusterInfo(CConstWUClusterInfoArray &clusters)
  4638. {
  4639. Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
  4640. Owned<IConstEnvironment> env = factory->openEnvironment();
  4641. if (!env)
  4642. return 0;
  4643. Owned<IPropertyTree> root = &env->getPTree();
  4644. return getEnvironmentClusterInfo(root, clusters);
  4645. }
  4646. unsigned getEnvironmentClusterInfo(IPropertyTree* environmentRoot, CConstWUClusterInfoArray &clusters)
  4647. {
  4648. if (!environmentRoot)
  4649. return 0;
  4650. Owned<IPropertyTreeIterator> clusterIter = environmentRoot->getElements("Software/Topology/Cluster");
  4651. ForEach(*clusterIter)
  4652. {
  4653. IPropertyTree &node = clusterIter->query();
  4654. Owned<IConstWUClusterInfo> cluster = getTargetClusterInfo(environmentRoot, &node);
  4655. clusters.append(*cluster.getClear());
  4656. }
  4657. return clusters.ordinality();
  4658. }
  4659. const char *getTargetClusterComponentName(const char *clustname, const char *processType, StringBuffer &name)
  4660. {
  4661. if (!clustname)
  4662. return NULL;
  4663. Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
  4664. Owned<IConstEnvironment> env = factory->openEnvironment();
  4665. if (!env)
  4666. return NULL;
  4667. Owned<IPropertyTree> root = &env->getPTree();
  4668. StringBuffer xpath;
  4669. xpath.appendf("Software/Topology/Cluster[@name=\"%s\"]", clustname);
  4670. Owned<IPropertyTree> cluster = root->getPropTree(xpath.str());
  4671. if (!cluster)
  4672. return NULL;
  4673. StringBuffer xpath1;
  4674. xpath1.appendf("%s/@process", processType);
  4675. name.append(cluster->queryProp(xpath1.str()));
  4676. return name.str();
  4677. }
  4678. unsigned getEnvironmentThorClusterNames(StringArray &thorNames, StringArray &groupNames, StringArray &targetNames, StringArray &queueNames)
  4679. {
  4680. Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
  4681. Owned<IConstEnvironment> env = factory->openEnvironment();
  4682. if (!env)
  4683. return 0;
  4684. Owned<IPropertyTree> root = &env->getPTree();
  4685. Owned<IPropertyTreeIterator> allTargets = root->getElements("Software/Topology/Cluster");
  4686. ForEach(*allTargets)
  4687. {
  4688. IPropertyTree &target = allTargets->query();
  4689. const char *targetName = target.queryProp("@name");
  4690. if (targetName && *targetName)
  4691. {
  4692. Owned<IPropertyTreeIterator> thorClusters = target.getElements("ThorCluster");
  4693. ForEach(*thorClusters)
  4694. {
  4695. const char *thorName = thorClusters->query().queryProp("@process");
  4696. VStringBuffer query("Software/ThorCluster[@name=\"%s\"]",thorName);
  4697. IPropertyTree *thorCluster = root->queryPropTree(query.str());
  4698. if (thorCluster)
  4699. {
  4700. const char *groupName = thorCluster->queryProp("@nodeGroup");
  4701. if (!groupName||!*groupName)
  4702. groupName = thorName;
  4703. thorNames.append(thorName);
  4704. groupNames.append(groupName);
  4705. targetNames.append(targetName);
  4706. StringBuffer queueName(targetName);
  4707. queueNames.append(queueName.append(THOR_QUEUE_EXT));
  4708. }
  4709. }
  4710. }
  4711. }
  4712. return thorNames.ordinality();
  4713. }
  4714. unsigned getEnvironmentHThorClusterNames(StringArray &eclAgentNames, StringArray &groupNames, StringArray &targetNames)
  4715. {
  4716. Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
  4717. Owned<IConstEnvironment> env = factory->openEnvironment();
  4718. if (!env)
  4719. return 0;
  4720. Owned<IPropertyTree> root = &env->getPTree();
  4721. Owned<IPropertyTreeIterator> allEclAgents = root->getElements("Software/EclAgentProcess");
  4722. ForEach(*allEclAgents)
  4723. {
  4724. IPropertyTree &eclAgent = allEclAgents->query();
  4725. const char *eclAgentName = eclAgent.queryProp("@name");
  4726. if (eclAgentName && *eclAgentName)
  4727. {
  4728. Owned<IPropertyTreeIterator> allTargets = root->getElements("Software/Topology/Cluster");
  4729. ForEach(*allTargets)
  4730. {
  4731. IPropertyTree &target = allTargets->query();
  4732. const char *targetName = target.queryProp("@name");
  4733. if (targetName && *targetName)
  4734. {
  4735. StringBuffer xpath;
  4736. xpath.appendf("EclAgentProcess[@process=\"%s\"]", eclAgentName);
  4737. if (target.hasProp(xpath) && !target.hasProp("ThorCluster"))
  4738. {
  4739. StringBuffer groupName("hthor__");
  4740. groupName.append(eclAgentName);
  4741. groupNames.append(groupName);
  4742. eclAgentNames.append(eclAgentName);
  4743. targetNames.append(targetName);
  4744. }
  4745. }
  4746. }
  4747. }
  4748. }
  4749. return eclAgentNames.ordinality();
  4750. }
  4751. IStringVal& CLocalWorkUnit::getScope(IStringVal &str) const
  4752. {
  4753. CriticalBlock block(crit);
  4754. if (p->hasProp("Debug/ForceScope"))
  4755. {
  4756. StringBuffer prefix(p->queryProp("Debug/ForceScope"));
  4757. str.set(prefix.toLowerCase().str());
  4758. }
  4759. else
  4760. {
  4761. Owned <IConstWUClusterInfo> ci = getTargetClusterInfo(p->queryProp("@clusterName"));
  4762. if (ci)
  4763. ci->getScope(str);
  4764. else
  4765. str.clear();
  4766. }
  4767. return str;
  4768. }
  4769. //Queries
  4770. void CLocalWorkUnit::setCodeVersion(unsigned codeVersion, const char * buildVersion, const char * eclVersion)
  4771. {
  4772. CriticalBlock block(crit);
  4773. p->setPropInt("@codeVersion", codeVersion);
  4774. p->setProp("@buildVersion", buildVersion);
  4775. p->setProp("@eclVersion", eclVersion);
  4776. }
  4777. unsigned CLocalWorkUnit::getCodeVersion() const
  4778. {
  4779. CriticalBlock block(crit);
  4780. return p->getPropInt("@codeVersion");
  4781. }
  4782. unsigned CLocalWorkUnit::getWuidVersion() const
  4783. {
  4784. CriticalBlock block(crit);
  4785. return p->getPropInt("@wuidVersion");
  4786. }
  4787. void CLocalWorkUnit::getBuildVersion(IStringVal & buildVersion, IStringVal & eclVersion) const
  4788. {
  4789. CriticalBlock block(crit);
  4790. buildVersion.set(p->queryProp("@buildVersion"));
  4791. eclVersion.set(p->queryProp("@eclVersion"));
  4792. }
  4793. void CLocalWorkUnit::setCloneable(bool value)
  4794. {
  4795. CriticalBlock block(crit);
  4796. p->setPropInt("@cloneable", value);
  4797. }
  4798. void CLocalWorkUnit::setIsClone(bool value)
  4799. {
  4800. CriticalBlock block(crit);
  4801. p->setPropInt("@isClone", value);
  4802. }
  4803. bool CLocalWorkUnit::getCloneable() const
  4804. {
  4805. CriticalBlock block(crit);
  4806. return p->getPropBool("@cloneable", false);
  4807. }
  4808. IUserDescriptor *CLocalWorkUnit::queryUserDescriptor() const
  4809. {
  4810. CriticalBlock block(crit);
  4811. if (!userDesc)
  4812. {
  4813. SCMStringBuffer token, user, password;
  4814. getSecurityToken(token);
  4815. extractToken(token.str(), queryWuid(), user, password);
  4816. userDesc.setown(createUserDescriptor());
  4817. userDesc->set(user.str(), password.str());
  4818. }
  4819. return userDesc;
  4820. }
  4821. bool CLocalWorkUnit::isProtected() const
  4822. {
  4823. CriticalBlock block(crit);
  4824. return p->getPropBool("@protected", false);
  4825. }
  4826. bool CLocalWorkUnit::isPausing() const
  4827. {
  4828. CriticalBlock block(crit);
  4829. if (WUActionPause == getAction())
  4830. {
  4831. switch (getState())
  4832. {
  4833. case WUStateRunning:
  4834. case WUStateAborting:
  4835. return true;
  4836. }
  4837. }
  4838. return false;
  4839. }
  4840. void CLocalWorkUnit::protect(bool protectMode)
  4841. {
  4842. CriticalBlock block(crit);
  4843. p->setPropBool("@protected", protectMode);
  4844. }
  4845. void CLocalWorkUnit::setResultLimit(unsigned value)
  4846. {
  4847. CriticalBlock block(crit);
  4848. p->setPropInt("resultLimit", value);
  4849. }
  4850. unsigned CLocalWorkUnit::getResultLimit() const
  4851. {
  4852. CriticalBlock block(crit);
  4853. return p->getPropInt("resultLimit");
  4854. }
  4855. IStringVal & CLocalWorkUnit::getSnapshot(IStringVal & str) const
  4856. {
  4857. CriticalBlock block(crit);
  4858. str.set(p->queryProp("SNAPSHOT"));
  4859. return str;
  4860. }
  4861. void CLocalWorkUnit::setSnapshot(const char * val)
  4862. {
  4863. CriticalBlock block(crit);
  4864. p->setProp("SNAPSHOT", val);
  4865. }
  4866. static int comparePropTrees(IInterface * const *ll, IInterface * const *rr)
  4867. {
  4868. IPropertyTree *l = (IPropertyTree *) *ll;
  4869. IPropertyTree *r = (IPropertyTree *) *rr;
  4870. return stricmp(l->queryName(), r->queryName());
  4871. };
  4872. unsigned CLocalWorkUnit::calculateHash(unsigned crc)
  4873. {
  4874. // Any other values in the WU that could affect generated code should be crc'ed here
  4875. IPropertyTree *tree = p->queryBranch("Debug");
  4876. if (tree)
  4877. {
  4878. Owned<IPropertyTreeIterator> sub = tree->getElements("*");
  4879. ICopyArrayOf<IPropertyTree> subs;
  4880. for(sub->first(); sub->isValid(); sub->next())
  4881. subs.append(sub->query());
  4882. subs.sort(comparePropTrees);
  4883. ForEachItemIn(idx, subs)
  4884. {
  4885. const char *name = subs.item(idx).queryName();
  4886. const char *val = subs.item(idx).queryProp(NULL);
  4887. crc = crc32(name, (size32_t)strlen(name), crc);
  4888. if (val)
  4889. crc = crc32(val, (size32_t)strlen(val), crc);
  4890. }
  4891. }
  4892. Owned<IConstWUPluginIterator> plugins = &getPlugins();
  4893. for (plugins->first();plugins->isValid();plugins->next())
  4894. {
  4895. IConstWUPlugin &thisplugin = plugins->query();
  4896. SCMStringBuffer version;
  4897. thisplugin.getPluginVersion(version);
  4898. crc = crc32(version.str(), version.length(), crc);
  4899. }
  4900. return crc;
  4901. }
  4902. static void updateProp(IPropertyTree * to, const IPropertyTree * from, const char * xpath)
  4903. {
  4904. if (!to->hasProp(xpath) && from->hasProp(xpath))
  4905. to->setProp(xpath, from->queryProp(xpath));
  4906. }
  4907. static void setProp(IPropertyTree * to, const IPropertyTree * from, const char * xpath)
  4908. {
  4909. if (from->hasProp(xpath))
  4910. to->setProp(xpath, from->queryProp(xpath));
  4911. }
  4912. static void copyTree(IPropertyTree * to, const IPropertyTree * from, const char * xpath)
  4913. {
  4914. IPropertyTree * match = from->getBranch(xpath);
  4915. if (match)
  4916. to->setPropTree(xpath, match);
  4917. }
  4918. void CLocalWorkUnit::copyWorkUnit(IConstWorkUnit *cached, bool all)
  4919. {
  4920. CLocalWorkUnit *from = QUERYINTERFACE(cached, CLocalWorkUnit);
  4921. if (!from)
  4922. {
  4923. CLockedWorkUnit *fl = QUERYINTERFACE(cached, CLockedWorkUnit);
  4924. if (!fl)
  4925. throw MakeStringException(WUERR_InternalUnknownImplementation, "Cached workunit not created using workunit dll");
  4926. from = fl->c;
  4927. }
  4928. // Need to copy the query, the results, and the graphs from the cached query.
  4929. // The cache is made before the query is executed so there is no need to clear them.
  4930. if (!cached->getCloneable())
  4931. throw MakeStringException(WUERR_CannotCloneWorkunit, "Source work unit not marked as clonable");
  4932. const IPropertyTree * fromP = from->p;
  4933. IPropertyTree *pt;
  4934. CriticalBlock block(crit);
  4935. query.clear();
  4936. updateProp(p, fromP, "@jobName");
  4937. copyTree(p, fromP, "Query");
  4938. pt = fromP->getBranch("Application/LibraryModule");
  4939. if (pt)
  4940. {
  4941. ensurePTree(p, "Application");
  4942. p->setPropTree("Application/LibraryModule", pt);
  4943. }
  4944. pt = fromP->queryBranch("Debug");
  4945. if (pt)
  4946. {
  4947. IPropertyTree *curDebug = p->queryPropTree("Debug");
  4948. if (curDebug)
  4949. {
  4950. Owned<IPropertyTreeIterator> elems = pt->getElements("*");
  4951. ForEach(*elems)
  4952. {
  4953. IPropertyTree *elem = &elems->query();
  4954. if (!curDebug->hasProp(elem->queryName()))
  4955. curDebug->setPropTree(elem->queryName(),LINK(elem));
  4956. }
  4957. }
  4958. else
  4959. p->setPropTree("Debug", LINK(pt));
  4960. }
  4961. copyTree(p, fromP, "Plugins");
  4962. copyTree(p, fromP, "Libraries");
  4963. copyTree(p, fromP, "Results");
  4964. copyTree(p, fromP, "Graphs");
  4965. copyTree(p, fromP, "Workflow");
  4966. if (all)
  4967. {
  4968. // 'all' mode is used when setting up a dali WU from the embedded wu in a workunit dll
  4969. // Merge timing info from both branches
  4970. pt = fromP->getBranch("Timings");
  4971. if (pt)
  4972. {
  4973. IPropertyTree *tgtTimings = ensurePTree(p, "Timings");
  4974. mergePTree(tgtTimings, pt);
  4975. pt->Release();
  4976. }
  4977. pt = fromP->getBranch("Statistics");
  4978. if (pt)
  4979. {
  4980. IPropertyTree *tgtStatistics = ensurePTree(p, "Statistics");
  4981. mergePTree(tgtStatistics, pt);
  4982. pt->Release();
  4983. }
  4984. }
  4985. updateProp(p, fromP, "@clusterName");
  4986. updateProp(p, fromP, "allowedclusters");
  4987. updateProp(p, fromP, "@submitID");
  4988. updateProp(p, fromP, "SNAPSHOT");
  4989. //MORE: This is very adhoc. All options that should be cloned should really be in a common branch
  4990. if (all)
  4991. {
  4992. setProp(p, fromP, "PriorityFlag");
  4993. setProp(p, fromP, "@priorityClass");
  4994. setProp(p, fromP, "@protected");
  4995. setProp(p, fromP, "@clusterName");
  4996. updateProp(p, fromP, "@scope");
  4997. }
  4998. //Variables may have been set up as parameters to the query - so need to preserve any values that were supplied.
  4999. pt = fromP->getBranch("Variables");
  5000. if (pt)
  5001. {
  5002. IPropertyTree *ptTgtVariables = ensurePTree(p, "Variables");
  5003. Owned<IPropertyTreeIterator> ptiVariable = pt->getElements("Variable");
  5004. for (ptiVariable->first(); ptiVariable->isValid(); ptiVariable->next())
  5005. {
  5006. IPropertyTree *ptSrcVariable = &ptiVariable->query();
  5007. const char *name = ptSrcVariable->queryProp("@name");
  5008. assertex(name);
  5009. StringBuffer xpath;
  5010. xpath.append("Variable[@name='").append(name).append("']");
  5011. IPropertyTree *ptTgtVariable = ptTgtVariables->queryPropTree(xpath.str());
  5012. IPropertyTree *merged = createPTreeFromIPT(ptSrcVariable); // clone entire source info...
  5013. merged->removeProp("Value"); // except value and status
  5014. merged->setProp("@status", "undefined");
  5015. if (!merged->getPropBool("@isScalar"))
  5016. merged->removeProp("totalRowCount");
  5017. merged->removeProp("rowCount");
  5018. // If there are any other fields that get set ONLY by eclagent, strip them out here...
  5019. if (ptTgtVariable)
  5020. {
  5021. // copy status and Value from what is already set in target
  5022. merged->setProp("@status", ptTgtVariable->queryProp("@status"));
  5023. MemoryBuffer value;
  5024. if (ptTgtVariable->getPropBin("Value", value))
  5025. merged->setPropBin("Value", value.length(), value.toByteArray());
  5026. ptTgtVariable->removeProp(xpath.str());
  5027. // If there are any other fields in a variable that get set by ws_ecl before submitting, copy them across here...
  5028. }
  5029. ptTgtVariables->addPropTree("Variable", merged);
  5030. }
  5031. pt->Release();
  5032. }
  5033. p->setProp("@codeVersion", fromP->queryProp("@codeVersion"));
  5034. p->setPropBool("@cloneable", true);
  5035. p->setPropBool("@isClone", true);
  5036. resetWorkflow(); // the source Workflow section may have had some parts already executed...
  5037. // resetResults(); // probably should be resetting the results as well... rather than waiting for the rerun to overwrite them
  5038. }
  5039. bool CLocalWorkUnit::hasDebugValue(const char *propname) const
  5040. {
  5041. StringBuffer lower;
  5042. lower.append(propname).toLowerCase();
  5043. CriticalBlock block(crit);
  5044. StringBuffer prop("Debug/");
  5045. return p->hasProp(prop.append(lower));
  5046. }
  5047. IStringVal& CLocalWorkUnit::getDebugValue(const char *propname, IStringVal &str) const
  5048. {
  5049. StringBuffer lower;
  5050. lower.append(propname).toLowerCase();
  5051. CriticalBlock block(crit);
  5052. StringBuffer prop("Debug/");
  5053. str.set(p->queryProp(prop.append(lower).str()));
  5054. return str;
  5055. }
  5056. IStringIterator& CLocalWorkUnit::getDebugValues() const
  5057. {
  5058. return getDebugValues(NULL);
  5059. }
  5060. IStringIterator& CLocalWorkUnit::getDebugValues(const char *prop) const
  5061. {
  5062. CriticalBlock block(crit);
  5063. StringBuffer path("Debug/");
  5064. if (prop)
  5065. {
  5066. StringBuffer lower;
  5067. lower.append(prop).toLowerCase();
  5068. path.append(lower);
  5069. }
  5070. else
  5071. path.append("*");
  5072. return *new CStringPTreeTagIterator(p->getElements(path.str()));
  5073. }
  5074. int CLocalWorkUnit::getDebugValueInt(const char *propname, int defVal) const
  5075. {
  5076. StringBuffer lower;
  5077. lower.append(propname).toLowerCase();
  5078. CriticalBlock block(crit);
  5079. StringBuffer prop("Debug/");
  5080. prop.append(lower);
  5081. return p->getPropInt(prop.str(), defVal);
  5082. }
  5083. __int64 CLocalWorkUnit::getDebugValueInt64(const char *propname, __int64 defVal) const
  5084. {
  5085. StringBuffer lower;
  5086. lower.append(propname).toLowerCase();
  5087. CriticalBlock block(crit);
  5088. StringBuffer prop("Debug/");
  5089. prop.append(lower);
  5090. return p->getPropInt64(prop.str(), defVal);
  5091. }
  5092. bool CLocalWorkUnit::getDebugValueBool(const char * propname, bool defVal) const
  5093. {
  5094. StringBuffer lower;
  5095. lower.append(propname).toLowerCase();
  5096. CriticalBlock block(crit);
  5097. StringBuffer prop("Debug/");
  5098. prop.append(lower);
  5099. return p->getPropBool(prop.str(), defVal);
  5100. }
  5101. IStringIterator *CLocalWorkUnit::getLogs(const char *type, const char *instance) const
  5102. {
  5103. VStringBuffer xpath("Process/%s/", type);
  5104. if (instance)
  5105. xpath.append(instance);
  5106. else
  5107. xpath.append("*");
  5108. CriticalBlock block(crit);
  5109. if (p->getPropInt("@wuidVersion") < 1) // legacy wuid
  5110. {
  5111. // NB: instance unused
  5112. if (streq("EclAgent", type))
  5113. return new CStringPTreeIterator(p->getElements("Debug/eclagentlog"));
  5114. else if (streq("Thor", type))
  5115. return new CStringPTreeIterator(p->getElements("Debug/thorlog*"));
  5116. VStringBuffer xpath("Debug/%s", type);
  5117. return new CStringPTreeIterator(p->getElements(xpath.str()));
  5118. }
  5119. else
  5120. return new CStringPTreeAttrIterator(p->getElements(xpath.str()), "@log");
  5121. }
  5122. IPropertyTreeIterator* CLocalWorkUnit::getProcesses(const char *type, const char *instance) const
  5123. {
  5124. VStringBuffer xpath("Process/%s/", type);
  5125. if (instance)
  5126. xpath.append(instance);
  5127. else
  5128. xpath.append("*");
  5129. CriticalBlock block(crit);
  5130. return p->getElements(xpath.str());
  5131. }
  5132. IStringIterator *CLocalWorkUnit::getProcesses(const char *type) const
  5133. {
  5134. VStringBuffer xpath("Process/%s/*", type);
  5135. CriticalBlock block(crit);
  5136. return new CStringPTreeTagIterator(p->getElements(xpath.str()));
  5137. }
  5138. void CLocalWorkUnit::addProcess(const char *type, const char *instance, unsigned pid, const char *log)
  5139. {
  5140. VStringBuffer processType("Process/%s", type);
  5141. VStringBuffer xpath("%s/%s", processType.str(), instance);
  5142. if (log)
  5143. xpath.appendf("[@log=\"%s\"]", log);
  5144. CriticalBlock block(crit);
  5145. if (!p->hasProp(xpath))
  5146. {
  5147. IPropertyTree *node = ensurePTree(p, processType.str());
  5148. node = node->addPropTree(instance, createPTree());
  5149. node->setProp("@log", log);
  5150. node->setPropInt("@pid", pid);
  5151. }
  5152. }
  5153. void CLocalWorkUnit::setDebugValue(const char *propname, const char *value, bool overwrite)
  5154. {
  5155. StringBuffer lower;
  5156. lower.append(propname).toLowerCase();
  5157. CriticalBlock block(crit);
  5158. StringBuffer prop("Debug/");
  5159. prop.append(lower);
  5160. if (overwrite || !p->hasProp(prop.str()))
  5161. {
  5162. // MORE - not sure this line should be needed....
  5163. p->setProp("Debug", "");
  5164. p->setProp(prop.str(), value);
  5165. }
  5166. }
  5167. void CLocalWorkUnit::setDebugValueInt(const char *propname, int value, bool overwrite)
  5168. {
  5169. StringBuffer lower;
  5170. lower.append(propname).toLowerCase();
  5171. CriticalBlock block(crit);
  5172. StringBuffer prop("Debug/");
  5173. prop.append(lower);
  5174. if (overwrite || !p->hasProp(prop.str()))
  5175. {
  5176. // MORE - not sure this line should be needed....
  5177. p->setProp("Debug", "");
  5178. p->setPropInt(prop.str(), value);
  5179. }
  5180. }
  5181. void CLocalWorkUnit::setTracingValue(const char *propname, const char *value)
  5182. {
  5183. CriticalBlock block(crit);
  5184. // MORE - not sure this line should be needed....
  5185. p->setProp("Tracing", "");
  5186. StringBuffer prop("Tracing/");
  5187. p->setProp(prop.append(propname).str(), value);
  5188. }
  5189. void CLocalWorkUnit::setTracingValueInt(const char *propname, int value)
  5190. {
  5191. CriticalBlock block(crit);
  5192. StringBuffer prop("Tracing/");
  5193. p->setPropInt(prop.append(propname).str(), value);
  5194. }
  5195. IConstWUQuery* CLocalWorkUnit::getQuery() const
  5196. {
  5197. // For this to be legally called, we must have the read-able interface. So we are already locked for (at least) read.
  5198. CriticalBlock block(crit);
  5199. if (!query)
  5200. {
  5201. IPropertyTree *s = p->getPropTree("Query");
  5202. if (s)
  5203. query.setown(new CLocalWUQuery(s)); // NB takes ownership of 's'
  5204. }
  5205. return query.getLink();
  5206. }
  5207. IWUQuery* CLocalWorkUnit::updateQuery()
  5208. {
  5209. // For this to be legally called, we must have the write-able interface. So we are already locked for write.
  5210. CriticalBlock block(crit);
  5211. if (!query)
  5212. {
  5213. IPropertyTree *s = p->queryPropTree("Query");
  5214. if (!s)
  5215. s = p->addPropTree("Query", createPTreeFromXMLString("<Query fetchEntire='1'/>"));
  5216. s->Link();
  5217. query.setown(new CLocalWUQuery(s));
  5218. }
  5219. return query.getLink();
  5220. }
  5221. void CLocalWorkUnit::loadPlugins() const
  5222. {
  5223. CriticalBlock block(crit);
  5224. if (!pluginsCached)
  5225. {
  5226. assertex(plugins.length() == 0);
  5227. Owned<IPropertyTreeIterator> r = p->getElements("Plugins/Plugin");
  5228. for (r->first(); r->isValid(); r->next())
  5229. {
  5230. IPropertyTree *rp = &r->query();
  5231. rp->Link();
  5232. plugins.append(*new CLocalWUPlugin(rp));
  5233. }
  5234. pluginsCached = true;
  5235. }
  5236. }
  5237. IConstWUPluginIterator& CLocalWorkUnit::getPlugins() const
  5238. {
  5239. CriticalBlock block(crit);
  5240. loadPlugins();
  5241. return *new CArrayIteratorOf<IConstWUPlugin,IConstWUPluginIterator> (plugins, 0, (IConstWorkUnit *) this);
  5242. }
  5243. void CLocalWorkUnit::loadLibraries() const
  5244. {
  5245. CriticalBlock block(crit);
  5246. if (!librariesCached)
  5247. {
  5248. assertex(libraries.length() == 0);
  5249. Owned<IPropertyTreeIterator> r = p->getElements("Libraries/Library");
  5250. ForEach(*r)
  5251. {
  5252. IPropertyTree *rp = &r->query();
  5253. rp->Link();
  5254. libraries.append(*new CLocalWULibrary(rp));
  5255. }
  5256. librariesCached = true;
  5257. }
  5258. }
  5259. IConstWULibraryIterator& CLocalWorkUnit::getLibraries() const
  5260. {
  5261. CriticalBlock block(crit);
  5262. loadLibraries();
  5263. return *new CArrayIteratorOf<IConstWULibrary,IConstWULibraryIterator> (libraries, 0, (IConstWorkUnit *) this);
  5264. }
  5265. IConstWULibrary * CLocalWorkUnit::getLibraryByName(const char * search) const
  5266. {
  5267. CriticalBlock block(crit);
  5268. loadLibraries();
  5269. ForEachItemIn(idx, libraries)
  5270. {
  5271. SCMStringBuffer name;
  5272. IConstWULibrary &cur = libraries.item(idx);
  5273. cur.getName(name);
  5274. if (stricmp(name.str(), search)==0)
  5275. return &OLINK(cur);
  5276. }
  5277. return NULL;
  5278. }
  5279. StringBuffer &formatGraphTimerLabel(StringBuffer &str, const char *graphName, unsigned subGraphNum, unsigned __int64 subId)
  5280. {
  5281. str.append("Graph ").append(graphName);
  5282. if (subGraphNum) str.append(" - ").append(subGraphNum).append(" (").append(subId).append(")");
  5283. else if (subId) str.append(" - id(").append(subId).append(")");
  5284. return str;
  5285. }
  5286. StringBuffer &formatGraphTimerScope(StringBuffer &str, const char *graphName, unsigned subGraphNum, unsigned __int64 subId)
  5287. {
  5288. str.append(graphName);
  5289. if (subId) str.append(":sg").append(subId);
  5290. return str;
  5291. }
  5292. bool parseGraphTimerLabel(const char *label, StringAttr &graphName, unsigned & graphNum, unsigned &subGraphNum, unsigned &subId)
  5293. {
  5294. // expects format: "Graph <graphname>[ - <subgraphnum> (<subgraphid>)]"
  5295. unsigned len = (size32_t)strlen(label);
  5296. if (len < 6 || (0 != memcmp(label, "Graph ", 6)))
  5297. return false;
  5298. graphNum = 0;
  5299. subGraphNum = 0;
  5300. subId = 0;
  5301. const char *finger = label+6;
  5302. const char *finger2 = strchr(finger, '-');
  5303. if (NULL == finger2) // just graphName
  5304. graphName.set(finger);
  5305. else
  5306. {
  5307. graphName.set(finger, (size32_t)((finger2-1)-finger));
  5308. finger = finger2+2; // skip '-' and space
  5309. finger2 = strchr(finger, ' ');
  5310. if (finger2)
  5311. {
  5312. subGraphNum = atoi_l(finger, (size32_t)(finger2-finger));
  5313. finger = finger2+2; // skip space and '('
  5314. finger2 = strchr(finger, ')');
  5315. if (finger2)
  5316. subId = atoi_l(finger, (size32_t)(finger2-finger));
  5317. }
  5318. else if (((len-(finger-label))>3) && 0 == memcmp(finger, "id(", 3)) // subgraph id only, new format.
  5319. {
  5320. finger += 3;
  5321. finger2 = strchr(finger, ')');
  5322. if (finger2)
  5323. subId = atoi_l(finger, (size32_t)(finger2-finger));
  5324. }
  5325. }
  5326. if (graphName && !memicmp(graphName, "graph", 5))
  5327. graphNum = atoi(graphName + 5);
  5328. return true;
  5329. }
  5330. bool parseGraphScope(const char *scope, StringAttr &graphName, unsigned & graphNum, unsigned &subGraphId)
  5331. {
  5332. if (!MATCHES_CONST_PREFIX(scope, GraphScopePrefix))
  5333. return false;
  5334. graphNum = atoi(scope + CONST_STRLEN(GraphScopePrefix));
  5335. const char * colon = strchr(scope, ':');
  5336. if (!colon)
  5337. {
  5338. graphName.set(scope);
  5339. subGraphId = 0;
  5340. return true;
  5341. }
  5342. const char * subgraph = colon+1;
  5343. graphName.set(scope, (size32_t)(colon - scope));
  5344. if (MATCHES_CONST_PREFIX(subgraph, SubGraphScopePrefix))
  5345. subGraphId = atoi(subgraph+CONST_STRLEN(SubGraphScopePrefix));
  5346. return true;
  5347. }
  5348. class WorkUnitStatisticsIterator : public CArrayIteratorOf<IConstWUStatistic,IConstWUStatisticIterator>
  5349. {
  5350. typedef CArrayIteratorOf<IConstWUStatistic,IConstWUStatisticIterator> PARENT;
  5351. public:
  5352. WorkUnitStatisticsIterator(const IArray &a, aindex_t start, IInterface *owner, const IStatisticsFilter * _filter)
  5353. : PARENT(a,start, owner), filter(_filter)
  5354. {
  5355. }
  5356. virtual bool first()
  5357. {
  5358. if (!PARENT::first())
  5359. return false;
  5360. if (matchesFilter())
  5361. return true;
  5362. return next();
  5363. }
  5364. virtual bool next()
  5365. {
  5366. loop
  5367. {
  5368. if (!PARENT::next())
  5369. return false;
  5370. if (matchesFilter())
  5371. return true;
  5372. }
  5373. }
  5374. protected:
  5375. bool matchesFilter()
  5376. {
  5377. if (!filter)
  5378. return true;
  5379. return query().matches(filter);
  5380. }
  5381. protected:
  5382. Linked<const IStatisticsFilter> filter;
  5383. };
  5384. void CLocalWorkUnit::setStatistic(StatisticCreatorType creatorType, const char * creator, StatisticScopeType scopeType, const char * scope, StatisticKind kind, const char * optDescription, unsigned __int64 value, unsigned __int64 count, unsigned __int64 maxValue, StatsMergeAction mergeAction)
  5385. {
  5386. if (!scope || !*scope) scope = GLOBAL_SCOPE;
  5387. const char * kindName = queryStatisticName(kind);
  5388. StatisticMeasure measure = queryMeasure(kind);
  5389. //creator. scope and name must all be present, and must not contain semi colons.
  5390. assertex(creator && scope);
  5391. CriticalBlock block(crit);
  5392. IPropertyTree * stats = p->queryPropTree("Statistics");
  5393. if (!stats)
  5394. stats = p->addPropTree("Statistics", createPTree("Statistics"));
  5395. IPropertyTree * statTree = NULL;
  5396. if (mergeAction != StatsMergeAppend)
  5397. {
  5398. StringBuffer xpath;
  5399. xpath.append("Statistic[@creator='").append(creator).append("'][@scope='").append(scope).append("'][@kind='").append(kindName).append("']");
  5400. statTree = stats->queryPropTree(xpath.str());
  5401. }
  5402. if (!statTree)
  5403. {
  5404. statTree = stats->addPropTree("Statistic", createPTree("Statistic"));
  5405. statTree->setProp("@creator", creator);
  5406. statTree->setProp("@scope", scope);
  5407. statTree->setProp("@kind", kindName);
  5408. //These items are primarily here to facilitate filtering.
  5409. statTree->setProp("@unit", queryMeasureName(measure));
  5410. statTree->setProp("@c", queryCreatorTypeName(creatorType));
  5411. statTree->setProp("@s", queryScopeTypeName(scopeType));
  5412. statTree->setPropInt64("@ts", getTimeStampNowValue());
  5413. if (optDescription)
  5414. statTree->setProp("@desc", optDescription);
  5415. if (statistics.cached)
  5416. statistics.append(LINK(statTree));
  5417. mergeAction = StatsMergeAppend;
  5418. }
  5419. if (mergeAction != StatsMergeAppend)
  5420. {
  5421. unsigned __int64 oldValue = statTree->getPropInt64("@value", 0);
  5422. unsigned __int64 oldCount = statTree->getPropInt64("@count", 0);
  5423. unsigned __int64 oldMax = statTree->getPropInt64("@max", 0);
  5424. if (oldMax < oldValue)
  5425. oldMax = oldValue;
  5426. statTree->setPropInt64("@value", mergeStatisticValue(oldValue, value, mergeAction));
  5427. statTree->setPropInt64("@count", count + oldCount);
  5428. if (maxValue > oldMax)
  5429. statTree->setPropInt64("@max", maxValue);
  5430. }
  5431. else
  5432. {
  5433. statTree->setPropInt64("@value", value);
  5434. statTree->setPropInt64("@count", count);
  5435. if (maxValue)
  5436. statTree->setPropInt64("@max", maxValue);
  5437. else
  5438. statTree->removeProp("@max");
  5439. }
  5440. }
  5441. IConstWUStatisticIterator& CLocalWorkUnit::getStatistics(const IStatisticsFilter * filter) const
  5442. {
  5443. CriticalBlock block(crit);
  5444. //This should be deleted in version 6.0 when support for 4.x is no longer required
  5445. legacyTimings.load(p,"Timings/*");
  5446. if (legacyTimings.ordinality())
  5447. return *new WorkUnitStatisticsIterator(legacyTimings, 0, (IConstWorkUnit *) this, filter);
  5448. statistics.load(p,"Statistics/*");
  5449. Owned<IConstWUStatisticIterator> localStats = new WorkUnitStatisticsIterator(statistics, 0, (IConstWorkUnit *) this, filter);
  5450. if (filter && !filter->queryMergeSources())
  5451. return *localStats.getClear();
  5452. const char * wuid = p->queryName();
  5453. Owned<IConstWUStatisticIterator> graphStats = new CConstGraphProgressStatisticsIterator(wuid, filter);
  5454. return * new CCompoundIteratorOf<IConstWUStatisticIterator, IConstWUStatistic>(localStats, graphStats);
  5455. }
  5456. IConstWUStatistic * CLocalWorkUnit::getStatistic(const char * creator, const char * scope, StatisticKind kind) const
  5457. {
  5458. //MORE: Optimize this....
  5459. StatisticsFilter filter;
  5460. filter.setCreator(creator);
  5461. filter.setScope(scope);
  5462. filter.setKind(kind);
  5463. Owned<IConstWUStatisticIterator> stats = &getStatistics(&filter);
  5464. if (stats->first())
  5465. return LINK(&stats->query());
  5466. return NULL;
  5467. }
  5468. bool CLocalWorkUnit::getWuDate(unsigned & year, unsigned & month, unsigned& day)
  5469. {
  5470. CriticalBlock block(crit);
  5471. if (sscanf(queryWuid(), "W%4u%2u%2u", &year, &month, &day)==3)
  5472. {
  5473. }
  5474. return false;
  5475. }
  5476. IWUPlugin* CLocalWorkUnit::updatePluginByName(const char *qname)
  5477. {
  5478. CriticalBlock block(crit);
  5479. IConstWUPlugin *existing = getPluginByName(qname);
  5480. if (existing)
  5481. return (IWUPlugin *) existing;
  5482. if (!plugins.length())
  5483. p->addPropTree("Plugins", createPTree("Plugins"));
  5484. IPropertyTree *pl = p->queryPropTree("Plugins");
  5485. IPropertyTree *s = pl->addPropTree("Plugin", createPTree("Plugin"));
  5486. s->Link();
  5487. IWUPlugin* q = new CLocalWUPlugin(s);
  5488. q->Link();
  5489. plugins.append(*q);
  5490. q->setPluginName(qname);
  5491. return q;
  5492. }
  5493. IConstWUPlugin* CLocalWorkUnit::getPluginByName(const char *qname) const
  5494. {
  5495. CriticalBlock block(crit);
  5496. loadPlugins();
  5497. ForEachItemIn(idx, plugins)
  5498. {
  5499. SCMStringBuffer name;
  5500. IConstWUPlugin &cur = plugins.item(idx);
  5501. cur.getPluginName(name);
  5502. if (stricmp(name.str(), qname)==0)
  5503. {
  5504. cur.Link();
  5505. return &cur;
  5506. }
  5507. }
  5508. return NULL;
  5509. }
  5510. IWULibrary* CLocalWorkUnit::updateLibraryByName(const char *qname)
  5511. {
  5512. CriticalBlock block(crit);
  5513. IConstWULibrary *existing = getLibraryByName(qname);
  5514. if (existing)
  5515. return (IWULibrary *) existing;
  5516. if (!libraries.length())
  5517. p->addPropTree("Libraries", createPTree("Libraries"));
  5518. IPropertyTree *pl = p->queryPropTree("Libraries");
  5519. IPropertyTree *s = pl->addPropTree("Library", createPTree("Library"));
  5520. s->Link();
  5521. IWULibrary* q = new CLocalWULibrary(s);
  5522. q->Link();
  5523. libraries.append(*q);
  5524. q->setName(qname);
  5525. return q;
  5526. }
  5527. void CLocalWorkUnit::loadExceptions() const
  5528. {
  5529. CriticalBlock block(crit);
  5530. if (!exceptionsCached)
  5531. {
  5532. assertex(exceptions.length() == 0);
  5533. Owned<IPropertyTreeIterator> r = p->getElements("Exceptions/Exception");
  5534. for (r->first(); r->isValid(); r->next())
  5535. {
  5536. IPropertyTree *rp = &r->query();
  5537. rp->Link();
  5538. exceptions.append(*new CLocalWUException(rp));
  5539. }
  5540. exceptionsCached = true;
  5541. }
  5542. }
  5543. IConstWUExceptionIterator& CLocalWorkUnit::getExceptions() const
  5544. {
  5545. CriticalBlock block(crit);
  5546. loadExceptions();
  5547. return *new CArrayIteratorOf<IConstWUException,IConstWUExceptionIterator> (exceptions, 0, (IConstWorkUnit *) this);
  5548. }
  5549. unsigned CLocalWorkUnit::getExceptionCount() const
  5550. {
  5551. CriticalBlock block(crit);
  5552. loadExceptions();
  5553. return exceptions.length();
  5554. }
  5555. void CLocalWorkUnit::clearExceptions()
  5556. {
  5557. CriticalBlock block(crit);
  5558. // For this to be legally called, we must have the write-able interface. So we are already locked for write.
  5559. exceptions.kill();
  5560. exceptionsCached = true;
  5561. p->removeProp("Exceptions");
  5562. }
  5563. IWUException* CLocalWorkUnit::createException()
  5564. {
  5565. CriticalBlock block(crit);
  5566. // For this to be legally called, we must have the write-able interface. So we are already locked for write.
  5567. loadExceptions();
  5568. if (!exceptions.length())
  5569. p->addPropTree("Exceptions", createPTree("Exceptions"));
  5570. IPropertyTree *r = p->queryPropTree("Exceptions");
  5571. IPropertyTree *s = r->addPropTree("Exception", createPTree("Exception"));
  5572. IWUException* q = new CLocalWUException(LINK(s));
  5573. exceptions.append(*LINK(q));
  5574. Owned<IJlibDateTime> now = createDateTimeNow();
  5575. SCMStringBuffer temp;
  5576. now->getString(temp);
  5577. q->setTimeStamp(temp.str());
  5578. return q;
  5579. }
  5580. IConstWUWebServicesInfo* CLocalWorkUnit::getWebServicesInfo() const
  5581. {
  5582. // For this to be legally called, we must have the read-able interface. So we are already locked for (at least) read.
  5583. CriticalBlock block(crit);
  5584. if (!webServicesInfoCached)
  5585. {
  5586. assertex(!webServicesInfo);
  5587. IPropertyTree *s = p->getPropTree("WebServicesInfo");
  5588. if (s)
  5589. webServicesInfo.setown(new CLocalWUWebServicesInfo(s)); // NB takes ownership of 's'
  5590. webServicesInfoCached = true;
  5591. }
  5592. return webServicesInfo.getLink();
  5593. }
  5594. IWUWebServicesInfo* CLocalWorkUnit::updateWebServicesInfo(bool create)
  5595. {
  5596. // For this to be legally called, we must have the write-able interface. So we are already locked for write.
  5597. CriticalBlock block(crit);
  5598. if (!webServicesInfoCached)
  5599. {
  5600. IPropertyTree *s = p->queryPropTree("WebServicesInfo");
  5601. if (!s)
  5602. {
  5603. if (create)
  5604. s = p->addPropTree("WebServicesInfo", createPTreeFromXMLString("<WebServicesInfo />"));
  5605. else
  5606. return NULL;
  5607. }
  5608. s->Link();
  5609. webServicesInfo.setown(new CLocalWUWebServicesInfo(s));
  5610. webServicesInfoCached = true;
  5611. }
  5612. return webServicesInfo.getLink();
  5613. }
  5614. IConstWURoxieQueryInfo* CLocalWorkUnit::getRoxieQueryInfo() const
  5615. {
  5616. // For this to be legally called, we must have the read-able interface. So we are already locked for (at least) read.
  5617. CriticalBlock block(crit);
  5618. if (!roxieQueryInfoCached)
  5619. {
  5620. assertex(!roxieQueryInfo);
  5621. IPropertyTree *s = p->getPropTree("RoxieQueryInfo");
  5622. if (s)
  5623. roxieQueryInfo.setown(new CLocalWURoxieQueryInfo(s)); // NB takes ownership of 's'
  5624. roxieQueryInfoCached = true;
  5625. }
  5626. return roxieQueryInfo.getLink();
  5627. }
  5628. IWURoxieQueryInfo* CLocalWorkUnit::updateRoxieQueryInfo(const char *wuid, const char *roxieClusterName)
  5629. {
  5630. // For this to be legally called, we must have the write-able interface. So we are already locked for write.
  5631. CriticalBlock block(crit);
  5632. if (!roxieQueryInfoCached)
  5633. {
  5634. IPropertyTree *s = p->queryPropTree("RoxieQueryInfo");
  5635. if (!s)
  5636. s = p->addPropTree("RoxieQueryInfo", createPTreeFromXMLString("<RoxieQueryInfo />"));
  5637. if (wuid && *wuid)
  5638. s->addProp("@wuid", wuid);
  5639. if (roxieClusterName && *roxieClusterName)
  5640. s->addProp("@roxieClusterName", roxieClusterName);
  5641. s->Link();
  5642. roxieQueryInfo.setown(new CLocalWURoxieQueryInfo(s));
  5643. roxieQueryInfoCached = true;
  5644. }
  5645. return roxieQueryInfo.getLink();
  5646. }
  5647. static int compareResults(IInterface * const *ll, IInterface * const *rr)
  5648. {
  5649. CLocalWUResult *l = (CLocalWUResult *) *ll;
  5650. CLocalWUResult *r = (CLocalWUResult *) *rr;
  5651. return l->getResultSequence() - r->getResultSequence();
  5652. }
  5653. void CLocalWorkUnit::loadResults() const
  5654. {
  5655. CriticalBlock block(crit);
  5656. if (!resultsCached)
  5657. {
  5658. assertex(results.length() == 0);
  5659. Owned<IPropertyTreeIterator> r = p->getElements("Results/Result");
  5660. for (r->first(); r->isValid(); r->next())
  5661. {
  5662. IPropertyTree *rp = &r->query();
  5663. rp->Link();
  5664. results.append(*new CLocalWUResult(rp));
  5665. }
  5666. results.sort(compareResults);
  5667. resultsCached = true;
  5668. }
  5669. }
  5670. void CLocalWorkUnit::loadVariables() const
  5671. {
  5672. CriticalBlock block(crit);
  5673. if (!variablesCached)
  5674. {
  5675. assertex(variables.length() == 0);
  5676. Owned<IPropertyTreeIterator> r = p->getElements("Variables/Variable");
  5677. for (r->first(); r->isValid(); r->next())
  5678. {
  5679. IPropertyTree *rp = &r->query();
  5680. rp->Link();
  5681. variables.append(*new CLocalWUResult(rp));
  5682. }
  5683. variablesCached = true;
  5684. }
  5685. }
  5686. void CLocalWorkUnit::loadTemporaries() const
  5687. {
  5688. CriticalBlock block(crit);
  5689. if (!temporariesCached)
  5690. {
  5691. assertex(temporaries.length() == 0);
  5692. Owned<IPropertyTreeIterator> r = p->getElements("Temporaries/Variable");
  5693. for (r->first(); r->isValid(); r->next())
  5694. {
  5695. IPropertyTree *rp = &r->query();
  5696. rp->Link();
  5697. temporaries.append(*new CLocalWUResult(rp));
  5698. }
  5699. temporariesCached = true;
  5700. }
  5701. }
  5702. void CLocalWorkUnit::deleteTemporaries()
  5703. {
  5704. CriticalBlock block(crit);
  5705. if (temporariesCached)
  5706. {
  5707. temporaries.kill();
  5708. temporariesCached = false;
  5709. }
  5710. p->removeProp("Temporaries");
  5711. }
  5712. IWUResult* CLocalWorkUnit::createResult()
  5713. {
  5714. CriticalBlock block(crit);
  5715. // For this to be legally called, we must have the write-able interface. So we are already locked for write.
  5716. loadResults();
  5717. if (!results.length())
  5718. p->addPropTree("Results", createPTree("Results"));
  5719. IPropertyTree *r = p->queryPropTree("Results");
  5720. IPropertyTree *s = r->addPropTree("Result", createPTree());
  5721. s->Link();
  5722. IWUResult* q = new CLocalWUResult(s);
  5723. q->Link();
  5724. results.append(*q);
  5725. return q;
  5726. }
  5727. IWUResult* CLocalWorkUnit::updateResultByName(const char *qname)
  5728. {
  5729. CriticalBlock block(crit);
  5730. IConstWUResult *existing = getResultByName(qname);
  5731. if (existing)
  5732. return (IWUResult *) existing;
  5733. IWUResult* q = createResult();
  5734. q->setResultName(qname);
  5735. return q;
  5736. }
  5737. IWUResult* CLocalWorkUnit::updateResultBySequence(unsigned seq)
  5738. {
  5739. CriticalBlock block(crit);
  5740. IConstWUResult *existing = getResultBySequence(seq);
  5741. if (existing)
  5742. return (IWUResult *) existing;
  5743. IWUResult* q = createResult();
  5744. q->setResultSequence(seq);
  5745. return q;
  5746. }
  5747. IConstWUResultIterator& CLocalWorkUnit::getResults() const
  5748. {
  5749. CriticalBlock block(crit);
  5750. loadResults();
  5751. return *new CArrayIteratorOf<IConstWUResult,IConstWUResultIterator> (results, 0, (IConstWorkUnit *) this);
  5752. }
  5753. IConstWUResult* CLocalWorkUnit::getResultByName(const char *qname) const
  5754. {
  5755. CriticalBlock block(crit);
  5756. loadResults();
  5757. ForEachItemIn(idx, results)
  5758. {
  5759. SCMStringBuffer name;
  5760. IConstWUResult &cur = results.item(idx);
  5761. cur.getResultName(name);
  5762. if (stricmp(name.str(), qname)==0)
  5763. {
  5764. cur.Link();
  5765. return &cur;
  5766. }
  5767. }
  5768. return NULL;
  5769. }
  5770. IConstWUResult* CLocalWorkUnit::getResultBySequence(unsigned seq) const
  5771. {
  5772. CriticalBlock block(crit);
  5773. loadResults();
  5774. ForEachItemIn(idx, results)
  5775. {
  5776. IConstWUResult &cur = results.item(idx);
  5777. if (cur.getResultSequence() == seq)
  5778. {
  5779. cur.Link();
  5780. return &cur;
  5781. }
  5782. }
  5783. return NULL;
  5784. }
  5785. IConstWUResultIterator& CLocalWorkUnit::getVariables() const
  5786. {
  5787. CriticalBlock block(crit);
  5788. loadVariables();
  5789. return *new CArrayIteratorOf<IConstWUResult,IConstWUResultIterator> (variables, 0, (IConstWorkUnit *) this);
  5790. }
  5791. IConstWUResult* CLocalWorkUnit::getGlobalByName(const char *qname) const
  5792. {
  5793. CriticalBlock block(crit);
  5794. if (strcmp(p->queryName(), GLOBAL_WORKUNIT)==0)
  5795. return getVariableByName(qname);
  5796. Owned <IWorkUnit> global = factory->ensureNamedWorkUnit(GLOBAL_WORKUNIT);
  5797. return global->getVariableByName(qname);
  5798. }
  5799. IWUResult* CLocalWorkUnit::updateGlobalByName(const char *qname)
  5800. {
  5801. CriticalBlock block(crit);
  5802. if (strcmp(p->queryName(), GLOBAL_WORKUNIT)==0)
  5803. return updateVariableByName(qname);
  5804. Owned <IWorkUnit> global = factory->ensureNamedWorkUnit(GLOBAL_WORKUNIT);
  5805. return global->updateVariableByName(qname);
  5806. }
  5807. IConstWUResult* CLocalWorkUnit::getVariableByName(const char *qname) const
  5808. {
  5809. CriticalBlock block(crit);
  5810. loadVariables();
  5811. ForEachItemIn(idx, variables)
  5812. {
  5813. SCMStringBuffer name;
  5814. IConstWUResult &cur = variables.item(idx);
  5815. cur.getResultName(name);
  5816. if (stricmp(name.str(), qname)==0)
  5817. {
  5818. cur.Link();
  5819. return &cur;
  5820. }
  5821. }
  5822. return NULL;
  5823. }
  5824. IConstWUResult* CLocalWorkUnit::getTemporaryByName(const char *qname) const
  5825. {
  5826. CriticalBlock block(crit);
  5827. loadTemporaries();
  5828. ForEachItemIn(idx, temporaries)
  5829. {
  5830. SCMStringBuffer name;
  5831. IConstWUResult &cur = temporaries.item(idx);
  5832. cur.getResultName(name);
  5833. if (stricmp(name.str(), qname)==0)
  5834. {
  5835. cur.Link();
  5836. return &cur;
  5837. }
  5838. }
  5839. return NULL;
  5840. }
  5841. IConstWUResultIterator& CLocalWorkUnit::getTemporaries() const
  5842. {
  5843. CriticalBlock block(crit);
  5844. loadTemporaries();
  5845. return *new CArrayIteratorOf<IConstWUResult,IConstWUResultIterator> (temporaries, 0, (IConstWorkUnit *) this);
  5846. }
  5847. IWUResult* CLocalWorkUnit::updateTemporaryByName(const char *qname)
  5848. {
  5849. CriticalBlock block(crit);
  5850. IConstWUResult *existing = getTemporaryByName(qname);
  5851. if (existing)
  5852. return (IWUResult *) existing;
  5853. if (!temporaries.length())
  5854. p->addPropTree("Temporaries", createPTree("Temporaries"));
  5855. IPropertyTree *vars = p->queryPropTree("Temporaries");
  5856. IPropertyTree *s = vars->addPropTree("Variable", createPTree("Variable"));
  5857. s->Link();
  5858. IWUResult* q = new CLocalWUResult(s);
  5859. q->Link();
  5860. temporaries.append(*q);
  5861. q->setResultName(qname);
  5862. return q;
  5863. }
  5864. IWUResult* CLocalWorkUnit::updateVariableByName(const char *qname)
  5865. {
  5866. CriticalBlock block(crit);
  5867. IConstWUResult *existing = getVariableByName(qname);
  5868. if (existing)
  5869. return (IWUResult *) existing;
  5870. if (!variables.length())
  5871. p->addPropTree("Variables", createPTree("Variables"));
  5872. IPropertyTree *vars = p->queryPropTree("Variables");
  5873. IPropertyTree *s = vars->addPropTree("Variable", createPTree("Variable"));
  5874. s->Link();
  5875. IWUResult* q = new CLocalWUResult(s);
  5876. q->Link();
  5877. variables.append(*q);
  5878. q->setResultName(qname);
  5879. return q;
  5880. }
  5881. void CLocalWorkUnit::deleteTempFiles(const char *graph, bool deleteOwned, bool deleteJobOwned)
  5882. {
  5883. CriticalBlock block(crit);
  5884. IPropertyTree *files = p->queryPropTree("Files");
  5885. if (!files) return;
  5886. Owned<IPropertyTreeIterator> iter = files->getElements("File");
  5887. ICopyArrayOf<IPropertyTree> toRemove;
  5888. ForEach (*iter)
  5889. {
  5890. IPropertyTree &file = iter->query();
  5891. WUFileKind fileKind = (WUFileKind) file.getPropInt("@kind", WUFileStandard);
  5892. if(file.getPropBool("@temporary")) fileKind = WUFileTemporary; // @temporary, legacy check
  5893. bool needDelete;
  5894. switch(fileKind)
  5895. {
  5896. case WUFileTemporary:
  5897. if(graph==NULL)
  5898. needDelete = true;
  5899. else
  5900. {
  5901. const char *graphOwner = file.queryProp("@graph");
  5902. needDelete = ((graphOwner==NULL) || (strcmp(graph, graphOwner)==0));
  5903. }
  5904. break;
  5905. case WUFileJobOwned:
  5906. needDelete = ((graph==NULL) && deleteJobOwned);
  5907. break;
  5908. case WUFileOwned:
  5909. needDelete = ((graph==NULL) && deleteOwned);
  5910. break;
  5911. default:
  5912. needDelete = false;
  5913. }
  5914. if(needDelete)
  5915. {
  5916. const char *name = file.queryProp("@name");
  5917. LOG(MCdebugProgress, unknownJob, "Removing workunit file %s from DFS", name);
  5918. queryDistributedFileDirectory().removeEntry(name, queryUserDescriptor());
  5919. toRemove.append(file);
  5920. }
  5921. }
  5922. ForEachItemIn(r, toRemove) files->removeTree(&toRemove.item(r));
  5923. }
  5924. static void _noteFileRead(IDistributedFile *file, IPropertyTree *filesRead)
  5925. {
  5926. IDistributedSuperFile *super = file->querySuperFile();
  5927. StringBuffer fname;
  5928. file->getLogicalName(fname);
  5929. StringBuffer path("File[@name=\"");
  5930. path.append(fname).append("\"]");
  5931. IPropertyTree *fileTree = filesRead->queryPropTree(path.str());
  5932. if (fileTree)
  5933. fileTree->setPropInt("@useCount", fileTree->getPropInt("@useCount")+1);
  5934. else
  5935. {
  5936. StringBuffer cluster;
  5937. file->getClusterName(0,cluster);
  5938. fileTree = createPTree();
  5939. fileTree->setProp("@name", fname.str());
  5940. fileTree->setProp("@cluster", cluster.str());
  5941. fileTree->setPropInt("@useCount", 1);
  5942. fileTree = filesRead->addPropTree("File", fileTree);
  5943. }
  5944. if (super)
  5945. {
  5946. Owned<IDistributedFileIterator> iter = super->getSubFileIterator(false);
  5947. ForEach (*iter)
  5948. {
  5949. IDistributedFile &file = iter->query();
  5950. StringBuffer fname;
  5951. file.getLogicalName(fname);
  5952. Owned<IPropertyTree> subfile = createPTree();
  5953. subfile->setProp("@name", fname.str());
  5954. fileTree->addPropTree("Subfile", subfile.getClear());
  5955. _noteFileRead(&file, filesRead);
  5956. }
  5957. }
  5958. }
  5959. void CLocalWorkUnit::noteFileRead(IDistributedFile *file)
  5960. {
  5961. CriticalBlock block(crit);
  5962. IPropertyTree *files = p->queryPropTree("FilesRead");
  5963. if (!files)
  5964. files = p->addPropTree("FilesRead", createPTree());
  5965. _noteFileRead(file, files);
  5966. }
  5967. static void addFile(IPropertyTree *files, const char *fileName, const char *cluster, unsigned usageCount, WUFileKind fileKind, const char *graphOwner)
  5968. {
  5969. StringBuffer path("File[@name=\"");
  5970. path.append(fileName).append("\"]");
  5971. if (cluster)
  5972. path.append("[@cluster=\"").append(cluster).append("\"]");
  5973. IPropertyTree *file = files->queryPropTree(path.str());
  5974. if (file) files->removeTree(file);
  5975. file = createPTree();
  5976. file->setProp("@name", fileName);
  5977. if (cluster)
  5978. file->setProp("@cluster", cluster);
  5979. if (graphOwner)
  5980. file->setProp("@graph", graphOwner);
  5981. file->setPropInt("@kind", (unsigned)fileKind);
  5982. if (WUFileTemporary == fileKind)
  5983. file->setPropInt("@usageCount", usageCount);
  5984. files->addPropTree("File", file);
  5985. }
  5986. void CLocalWorkUnit::addFile(const char *fileName, StringArray *clusters, unsigned usageCount, WUFileKind fileKind, const char *graphOwner)
  5987. {
  5988. CriticalBlock block(crit);
  5989. IPropertyTree *files = p->queryPropTree("Files");
  5990. if (!files)
  5991. files = p->addPropTree("Files", createPTree());
  5992. if (!clusters)
  5993. addFile(fileName, NULL, usageCount, fileKind, graphOwner);
  5994. else
  5995. {
  5996. ForEachItemIn(c, *clusters)
  5997. ::addFile(files, fileName, clusters->item(c), usageCount, fileKind, graphOwner);
  5998. }
  5999. }
  6000. void CLocalWorkUnit::releaseFile(const char *fileName)
  6001. {
  6002. StringBuffer path("File[@name=\"");
  6003. path.append(fileName).append("\"]");
  6004. CriticalBlock block(crit);
  6005. IPropertyTree *files = p->queryPropTree("Files");
  6006. if (!files) return;
  6007. Owned<IPropertyTreeIterator> fiter = files->getElements(path.str());
  6008. ForEach (*fiter)
  6009. {
  6010. IPropertyTree *file = &fiter->query();
  6011. unsigned usageCount = file->getPropInt("@usageCount");
  6012. if (usageCount > 1)
  6013. file->setPropInt("@usageCount", usageCount-1);
  6014. else
  6015. {
  6016. StringAttr name(file->queryProp("@name"));
  6017. files->removeTree(file);
  6018. if (!name.isEmpty()&&(1 == usageCount))
  6019. {
  6020. if (queryDistributedFileDirectory().removeEntry(fileName, queryUserDescriptor()))
  6021. LOG(MCdebugProgress, unknownJob, "Removed (released) file %s from DFS", name.get());
  6022. }
  6023. }
  6024. }
  6025. }
  6026. void CLocalWorkUnit::clearGraphProgress()
  6027. {
  6028. CConstGraphProgress::deleteWuidProgress(p->queryName());
  6029. }
  6030. void CLocalWorkUnit::resetBeforeGeneration()
  6031. {
  6032. CriticalBlock block(crit);
  6033. //Remove all associated files
  6034. Owned<IWUQuery> q = updateQuery();
  6035. q->removeAssociatedFiles();
  6036. //Remove any pre-existing workflow information
  6037. workflowIterator.clear();
  6038. p->removeProp("Workflow");
  6039. }
  6040. unsigned CLocalWorkUnit::queryFileUsage(const char *fileName) const
  6041. {
  6042. StringBuffer path("Files/File[@name=\"");
  6043. path.append(fileName).append("\"]/@usageCount");
  6044. CriticalBlock block(crit);
  6045. return p->getPropInt(path.str());
  6046. }
  6047. IPropertyTree *CLocalWorkUnit::getDiskUsageStats()
  6048. {
  6049. return p->getPropTree("DiskUsageStats");
  6050. }
  6051. void CLocalWorkUnit::addDiskUsageStats(__int64 _avgNodeUsage, unsigned _minNode, __int64 _minNodeUsage, unsigned _maxNode, __int64 _maxNodeUsage, __int64 _graphId)
  6052. {
  6053. IPropertyTree *stats = p->queryPropTree("DiskUsageStats");
  6054. offset_t maxNodeUsage;
  6055. if (stats)
  6056. maxNodeUsage = stats->getPropInt64("@maxNodeUsage");
  6057. else
  6058. {
  6059. stats = p->addPropTree("DiskUsageStats", createPTree());
  6060. maxNodeUsage = 0;
  6061. }
  6062. if ((offset_t)_maxNodeUsage > maxNodeUsage)
  6063. {
  6064. // record all details at time of max node usage.
  6065. stats->setPropInt("@minNode", _minNode);
  6066. stats->setPropInt("@maxNode", _maxNode);
  6067. stats->setPropInt64("@minNodeUsage", _minNodeUsage);
  6068. stats->setPropInt64("@maxNodeUsage", _maxNodeUsage);
  6069. stats->setPropInt64("@graphId", _graphId);
  6070. if (_avgNodeUsage)
  6071. {
  6072. unsigned _skewHi = (unsigned)((100 * (_maxNodeUsage-_avgNodeUsage))/_avgNodeUsage);
  6073. unsigned _skewLo = (unsigned)((100 * (_avgNodeUsage-_minNodeUsage))/_avgNodeUsage);
  6074. stats->setPropInt("@skewHi", _skewHi);
  6075. stats->setPropInt("@skewLo", _skewLo);
  6076. }
  6077. }
  6078. }
  6079. IPropertyTreeIterator & CLocalWorkUnit::getFileIterator() const
  6080. {
  6081. CriticalBlock block(crit);
  6082. return * p->getElements("Files/File");
  6083. }
  6084. IPropertyTreeIterator & CLocalWorkUnit::getFilesReadIterator() const
  6085. {
  6086. CriticalBlock block(crit);
  6087. return * p->getElements("FilesRead/File");
  6088. }
  6089. //=================================================================================================
  6090. bool CLocalWorkUnit::switchThorQueue(const char *cluster, IQueueSwitcher *qs)
  6091. {
  6092. CriticalBlock block(crit);
  6093. if (qs->isAuto()&&!getAllowAutoQueueSwitch())
  6094. return false;
  6095. Owned<IConstWUClusterInfo> newci = getTargetClusterInfo(cluster);
  6096. if (!newci)
  6097. return false;
  6098. StringBuffer currentcluster;
  6099. if (!p->getProp("@clusterName",currentcluster))
  6100. return false;
  6101. Owned<IConstWUClusterInfo> curci = getTargetClusterInfo(currentcluster.str());
  6102. if (!curci)
  6103. return false;
  6104. SCMStringBuffer curqname;
  6105. curci->getThorQueue(curqname);
  6106. const char *wuid = p->queryName();
  6107. void *qi = qs->getQ(curqname.str(),wuid);
  6108. if (!qi)
  6109. return false;
  6110. setClusterName(cluster);
  6111. SCMStringBuffer newqname;
  6112. newci->getThorQueue(newqname);
  6113. qs->putQ(newqname.str(),wuid,qi);
  6114. return true;
  6115. }
  6116. //=================================================================================================
  6117. IPropertyTree *CLocalWorkUnit::getUnpackedTree(bool includeProgress) const
  6118. {
  6119. Owned<IPropertyTree> ret = createPTreeFromIPT(p);
  6120. Owned<IConstWUGraphIterator> graphIter = &getGraphs(GraphTypeAny);
  6121. ForEach(*graphIter)
  6122. {
  6123. IConstWUGraph &graph = graphIter->query();
  6124. Owned<IPropertyTree> graphTree = graph.getXGMMLTree(includeProgress);
  6125. SCMStringBuffer gName;
  6126. graph.getName(gName);
  6127. StringBuffer xpath("Graphs/Graph[@name=\"");
  6128. xpath.append(gName.s).append("\"]/xgmml");
  6129. IPropertyTree *xgmml = ret->queryPropTree(xpath.str());
  6130. if (xgmml) // don't know of any reason it shouldn't exist
  6131. {
  6132. xgmml->removeProp("graphBin");
  6133. xgmml->setPropTree("graph", graphTree.getClear());
  6134. }
  6135. }
  6136. return ret.getClear();
  6137. }
  6138. void CLocalWorkUnit::loadGraphs() const
  6139. {
  6140. CriticalBlock block(crit);
  6141. if (!cachedGraphs.get())
  6142. {
  6143. MemoryBuffer buf;
  6144. IPropertyTree *t = p->queryPropTree("PackedGraphs");
  6145. if (t&&t->getPropBin(NULL,buf)) {
  6146. cachedGraphs.setown(createPTree(buf));
  6147. }
  6148. else
  6149. cachedGraphs.set(p->queryPropTree("Graphs"));
  6150. if (cachedGraphs.get())
  6151. {
  6152. Owned<IPropertyTreeIterator> iter = cachedGraphs->getElements("Graph");
  6153. ForEach(*iter)
  6154. {
  6155. IPropertyTree &graph = iter->query();
  6156. graphs.append(*new CLocalWUGraph(*this, LINK(&graph)));
  6157. }
  6158. }
  6159. }
  6160. }
  6161. mapEnums graphTypes[] = {
  6162. { GraphTypeAny, "unknown" },
  6163. { GraphTypeProgress, "progress" },
  6164. { GraphTypeEcl, "ECL" },
  6165. { GraphTypeActivities, "activities" },
  6166. { GraphTypeSubProgress, "subgraph" },
  6167. { GraphTypeSize, NULL },
  6168. };
  6169. CLocalWUGraph::CLocalWUGraph(const CLocalWorkUnit &_owner, IPropertyTree *props) : p(props), owner(_owner)
  6170. {
  6171. wuidVersion = owner.getWuidVersion();
  6172. }
  6173. IStringVal& CLocalWUGraph::getName(IStringVal &str) const
  6174. {
  6175. str.set(p->queryProp("@name"));
  6176. return str;
  6177. }
  6178. IStringVal& CLocalWUGraph::getLabel(IStringVal &str) const
  6179. {
  6180. if (wuidVersion >= 2)
  6181. {
  6182. str.set(p->queryProp("@label"));
  6183. return str;
  6184. }
  6185. else
  6186. {
  6187. Owned<IPropertyTree> xgmml = getXGMMLTree(false);
  6188. str.set(xgmml->queryProp("@label"));
  6189. return str;
  6190. }
  6191. }
  6192. WUGraphState CLocalWUGraph::getState() const
  6193. {
  6194. Owned<IConstWUGraphProgress> graphProgress = owner.getGraphProgress(p->queryProp("@name"));
  6195. if (!graphProgress)
  6196. return WUGraphUnknown;
  6197. return graphProgress->queryGraphState();
  6198. }
  6199. IStringVal& CLocalWUGraph::getXGMML(IStringVal &str, bool mergeProgress) const
  6200. {
  6201. Owned<IPropertyTree> xgmml = getXGMMLTree(mergeProgress);
  6202. if (xgmml)
  6203. {
  6204. StringBuffer x;
  6205. toXML(xgmml, x);
  6206. str.set(x.str());
  6207. }
  6208. return str;
  6209. }
  6210. unsigned CLocalWorkUnit::getGraphCount() const
  6211. {
  6212. CriticalBlock block(crit);
  6213. if (p->hasProp("Graphs"))
  6214. {
  6215. return p->queryPropTree("Graphs")->numChildren();
  6216. }
  6217. return 0;
  6218. }
  6219. unsigned CLocalWorkUnit::getSourceFileCount() const
  6220. {
  6221. CriticalBlock block(crit);
  6222. if (p->hasProp("FilesRead"))
  6223. {
  6224. return p->queryPropTree("FilesRead")->numChildren();
  6225. }
  6226. return 0;
  6227. }
  6228. unsigned CLocalWorkUnit::getResultCount() const
  6229. {
  6230. CriticalBlock block(crit);
  6231. if (p->hasProp("Results"))
  6232. {
  6233. return p->queryPropTree("Results")->numChildren();
  6234. }
  6235. return 0;
  6236. }
  6237. unsigned CLocalWorkUnit::getVariableCount() const
  6238. {
  6239. CriticalBlock block(crit);
  6240. if (p->hasProp("Variables"))
  6241. {
  6242. return p->queryPropTree("Variables")->numChildren();
  6243. }
  6244. return 0;
  6245. }
  6246. unsigned CLocalWorkUnit::getApplicationValueCount() const
  6247. {
  6248. CriticalBlock block(crit);
  6249. if (p->hasProp("Application"))
  6250. {
  6251. return p->queryPropTree("Application")->numChildren();
  6252. }
  6253. return 0;
  6254. }
  6255. IStringVal &CLocalWorkUnit::getXmlParams(IStringVal &str) const
  6256. {
  6257. CriticalBlock block(crit);
  6258. IPropertyTree *paramTree = p->queryPropTree("Parameters");
  6259. if (paramTree)
  6260. {
  6261. StringBuffer temp;
  6262. toXML(paramTree, temp);
  6263. str.set(temp.str());
  6264. }
  6265. return str;
  6266. }
  6267. const IPropertyTree *CLocalWorkUnit::getXmlParams() const
  6268. {
  6269. CriticalBlock block(crit);
  6270. return p->getPropTree("Parameters");
  6271. }
  6272. void CLocalWorkUnit::setXmlParams(const char *params)
  6273. {
  6274. CriticalBlock block(crit);
  6275. p->setPropTree("Parameters", createPTreeFromXMLString(params));
  6276. }
  6277. void CLocalWorkUnit::setXmlParams(IPropertyTree *tree)
  6278. {
  6279. CriticalBlock block(crit);
  6280. p->setPropTree("Parameters", tree);
  6281. }
  6282. unsigned __int64 CLocalWorkUnit::getHash() const
  6283. {
  6284. CriticalBlock block(crit);
  6285. return p->getPropInt64("@hash");
  6286. }
  6287. void CLocalWorkUnit::setHash(unsigned __int64 hash)
  6288. {
  6289. CriticalBlock block(crit);
  6290. p->setPropInt64("@hash", hash);
  6291. }
  6292. IConstWUGraphMetaIterator& CLocalWorkUnit::getGraphsMeta(WUGraphType type) const
  6293. {
  6294. /* NB: this method should be 'cheap', loadGraphs() creates IConstWUGraph interfaces to the graphs
  6295. * it does not actually pull the graph data. We only use IConstWUGraphMeta here, which never probes the xgmml
  6296. * This method also connects to the graph progress (/GraphProgress/<wuid>) using a single connection, in order
  6297. * to get state information, it does not pull all the progress data.
  6298. */
  6299. CriticalBlock block(crit);
  6300. loadGraphs();
  6301. class CConstWUGraphMetaIterator: public CInterface, implements IConstWUGraphMetaIterator, implements IConstWUGraphMeta
  6302. {
  6303. StringAttr wuid;
  6304. WUGraphType type;
  6305. Owned<IConstWUGraphIterator> graphIter;
  6306. IConstWUGraph *curGraph;
  6307. Owned<IConstWUGraphProgress> curGraphProgress;
  6308. Owned<IRemoteConnection> progressConn;
  6309. bool match()
  6310. {
  6311. return (GraphTypeAny == type) || (type == graphIter->query().getType());
  6312. }
  6313. void setCurrent(IConstWUGraph &graph)
  6314. {
  6315. curGraph = &graph;
  6316. SCMStringBuffer graphName;
  6317. curGraph->getName(graphName);
  6318. if (progressConn)
  6319. {
  6320. IPropertyTree *progress = progressConn->queryRoot()->queryPropTree(graphName.str());
  6321. if (progress)
  6322. {
  6323. curGraphProgress.setown(new CConstGraphProgress(wuid, graphName.str(), progress));
  6324. return;
  6325. }
  6326. }
  6327. curGraphProgress.clear();
  6328. }
  6329. public:
  6330. IMPLEMENT_IINTERFACE;
  6331. CConstWUGraphMetaIterator(const char *_wuid, IConstWUGraphIterator *_graphIter, WUGraphType _type)
  6332. : wuid(_wuid), graphIter(_graphIter), type(_type)
  6333. {
  6334. curGraph = NULL;
  6335. StringBuffer progressPath;
  6336. progressPath.append("/GraphProgress/").append(wuid);
  6337. progressConn.setown(querySDS().connect(progressPath.str(), myProcessSession(), RTM_NONE, SDS_LOCK_TIMEOUT));
  6338. }
  6339. virtual bool first()
  6340. {
  6341. curGraph = NULL;
  6342. curGraphProgress.clear();
  6343. if (!graphIter->first())
  6344. return false;
  6345. if (match())
  6346. {
  6347. setCurrent(graphIter->query());
  6348. return true;
  6349. }
  6350. return next();
  6351. }
  6352. virtual bool next()
  6353. {
  6354. while (graphIter->next())
  6355. {
  6356. if (match())
  6357. {
  6358. setCurrent(graphIter->query());
  6359. return true;
  6360. }
  6361. }
  6362. curGraph = NULL;
  6363. curGraphProgress.clear();
  6364. return false;
  6365. }
  6366. virtual bool isValid()
  6367. {
  6368. return NULL != curGraph;
  6369. }
  6370. virtual IConstWUGraphMeta & query()
  6371. {
  6372. return *this;
  6373. }
  6374. // IConstWUGraphMeta
  6375. virtual IStringVal & getName(IStringVal & ret) const
  6376. {
  6377. return curGraph->getName(ret);
  6378. }
  6379. virtual IStringVal & getLabel(IStringVal & ret) const
  6380. {
  6381. return curGraph->getLabel(ret);
  6382. }
  6383. virtual IStringVal & getTypeName(IStringVal & ret) const
  6384. {
  6385. return curGraph->getTypeName(ret);
  6386. }
  6387. virtual WUGraphType getType() const
  6388. {
  6389. return curGraph->getType();
  6390. }
  6391. virtual WUGraphState getState() const
  6392. {
  6393. if (!curGraphProgress)
  6394. return WUGraphUnknown;
  6395. return curGraphProgress->queryGraphState();
  6396. }
  6397. };
  6398. IConstWUGraphIterator *graphIter = new CArrayIteratorOf<IConstWUGraph,IConstWUGraphIterator> (graphs, 0, (IConstWorkUnit *) this);
  6399. return * new CConstWUGraphMetaIterator(p->queryName(), graphIter, type);
  6400. }
  6401. IConstWUGraphIterator& CLocalWorkUnit::getGraphs(WUGraphType type) const
  6402. {
  6403. CriticalBlock block(crit);
  6404. loadGraphs();
  6405. IConstWUGraphIterator *giter = new CArrayIteratorOf<IConstWUGraph,IConstWUGraphIterator> (graphs, 0, (IConstWorkUnit *) this);
  6406. if (type!=GraphTypeAny) {
  6407. class CConstWUGraphIterator: public CInterface, implements IConstWUGraphIterator
  6408. {
  6409. WUGraphType type;
  6410. Owned<IConstWUGraphIterator> base;
  6411. bool match()
  6412. {
  6413. return base->query().getType()==type;
  6414. }
  6415. public:
  6416. IMPLEMENT_IINTERFACE;
  6417. CConstWUGraphIterator(IConstWUGraphIterator *_base,WUGraphType _type)
  6418. : base(_base)
  6419. {
  6420. type = _type;
  6421. }
  6422. bool first()
  6423. {
  6424. if (!base->first())
  6425. return false;
  6426. if (match())
  6427. return true;
  6428. return next();
  6429. }
  6430. bool next()
  6431. {
  6432. while (base->next())
  6433. if (match())
  6434. return true;
  6435. return false;
  6436. }
  6437. virtual bool isValid()
  6438. {
  6439. return base->isValid();
  6440. }
  6441. IConstWUGraph & query()
  6442. {
  6443. return base->query();
  6444. }
  6445. };
  6446. giter = new CConstWUGraphIterator(giter,type);
  6447. }
  6448. return *giter;
  6449. }
  6450. IConstWUGraph* CLocalWorkUnit::getGraph(const char *qname) const
  6451. {
  6452. CriticalBlock block(crit);
  6453. loadGraphs();
  6454. ForEachItemIn(idx, graphs)
  6455. {
  6456. SCMStringBuffer name;
  6457. IConstWUGraph &cur = graphs.item(idx);
  6458. cur.getName(name);
  6459. if (stricmp(name.str(), qname)==0)
  6460. {
  6461. cur.Link();
  6462. return &cur;
  6463. }
  6464. }
  6465. return NULL;
  6466. }
  6467. IWUGraph* CLocalWorkUnit::createGraph()
  6468. {
  6469. // For this to be legally called, we must have the write-able interface. So we are already locked for write.
  6470. CriticalBlock block(crit);
  6471. ensureGraphsUnpacked();
  6472. loadGraphs();
  6473. if (!graphs.ordinality())
  6474. p->addPropTree("Graphs", createPTree("Graphs"));
  6475. IPropertyTree *r = p->queryPropTree("Graphs");
  6476. IPropertyTree *s = r->addPropTree("Graph", createPTree());
  6477. s->Link();
  6478. IWUGraph* q = new CLocalWUGraph(*this, s);
  6479. q->Link();
  6480. graphs.append(*q);
  6481. return q;
  6482. }
  6483. IWUGraph * CLocalWorkUnit::createGraph(const char * name, WUGraphType type, IPropertyTree *xgmml)
  6484. {
  6485. CriticalBlock block(crit);
  6486. ensureGraphsUnpacked();
  6487. Linked<IConstWUGraph> existing = getGraph(name);
  6488. if (existing)
  6489. throwUnexpected();
  6490. if (!graphs.length())
  6491. p->addPropTree("Graphs", createPTree("Graphs"));
  6492. IPropertyTree *r = p->queryPropTree("Graphs");
  6493. IPropertyTree *s = r->addPropTree("Graph", createPTree());
  6494. IWUGraph* q = new CLocalWUGraph(*this, LINK(s));
  6495. graphs.append(*LINK(q));
  6496. q->setName(name);
  6497. q->setXGMMLTree(xgmml);
  6498. q->setType(type);
  6499. return q;
  6500. }
  6501. IWUGraph * CLocalWorkUnit::updateGraph(const char * name)
  6502. {
  6503. CriticalBlock block(crit);
  6504. ensureGraphsUnpacked();
  6505. return (IWUGraph *)getGraph(name);
  6506. }
  6507. IConstWUGraphProgress *CLocalWorkUnit::getGraphProgress(const char *name) const
  6508. {
  6509. CriticalBlock block(crit);
  6510. return new CConstGraphProgress(p->queryName(), name);
  6511. }
  6512. void CLocalWUGraph::setName(const char *str)
  6513. {
  6514. p->setProp("@name", str);
  6515. progress.clear();
  6516. progress.setown(new CConstGraphProgress(owner.queryWuid(), str));
  6517. }
  6518. void CLocalWUGraph::setLabel(const char *str)
  6519. {
  6520. p->setProp("@label", str);
  6521. }
  6522. void CLocalWUGraph::setXGMML(const char *str)
  6523. {
  6524. setXGMMLTree(createPTreeFromXMLString(str));
  6525. }
  6526. void CLocalWUGraph::setXGMMLTree(IPropertyTree *_graph, bool compress)
  6527. {
  6528. assertex(strcmp(_graph->queryName(), "graph")==0);
  6529. IPropertyTree *xgmml = p->setPropTree("xgmml", createPTree());
  6530. if (compress)
  6531. {
  6532. MemoryBuffer mb;
  6533. _graph->serialize(mb);
  6534. xgmml->setPropBin("graphBin", mb.length(), mb.toByteArray());
  6535. graph.setown(_graph);
  6536. }
  6537. else
  6538. xgmml->setPropTree("graph", _graph);
  6539. }
  6540. static void expandAttributes(IPropertyTree & targetNode, IPropertyTree & progressNode)
  6541. {
  6542. Owned<IAttributeIterator> aIter = progressNode.getAttributes();
  6543. ForEach (*aIter)
  6544. {
  6545. const char *aName = aIter->queryName()+1;
  6546. if (0 != stricmp("id", aName)) // "id" reserved.
  6547. {
  6548. IPropertyTree *att = targetNode.addPropTree("att", createPTree());
  6549. att->setProp("@name", aName);
  6550. att->setProp("@value", aIter->queryValue());
  6551. }
  6552. }
  6553. }
  6554. void CLocalWUGraph::mergeProgress(IPropertyTree &rootNode, IPropertyTree &progressTree, const unsigned &progressV) const
  6555. {
  6556. IPropertyTree *graphNode = rootNode.queryPropTree("att/graph");
  6557. if (!graphNode) return;
  6558. unsigned nodeId = rootNode.getPropInt("@id");
  6559. StringBuffer progressNodePath("node[@id=\"");
  6560. progressNodePath.append(nodeId).append("\"]");
  6561. IPropertyTree *progressNode = progressTree.queryPropTree(progressNodePath.str());
  6562. if (progressNode)
  6563. {
  6564. expandAttributes(*graphNode, *progressNode);
  6565. Owned<IPropertyTreeIterator> edges = progressNode->getElements("edge");
  6566. ForEach (*edges)
  6567. {
  6568. IPropertyTree &edge = edges->query();
  6569. StringBuffer edgePath("edge[@id=\"");
  6570. edgePath.append(edge.queryProp("@id")).append("\"]");
  6571. IPropertyTree *graphEdge = graphNode->queryPropTree(edgePath.str());
  6572. if (graphEdge)
  6573. {
  6574. if (progressV < 1)
  6575. mergePTree(graphEdge, &edge);
  6576. else
  6577. { // must translate to XGMML format
  6578. expandAttributes(*graphEdge, edge);
  6579. // This is really only here, so that our progress format can use non-attribute values, which have different efficiency qualifies (e.g. can be external by dali)
  6580. Owned<IPropertyTreeIterator> iter = edge.getElements("*");
  6581. ForEach (*iter)
  6582. {
  6583. IPropertyTree &t = iter->query();
  6584. IPropertyTree *att = graphEdge->addPropTree("att", createPTree());
  6585. att->setProp("@name", t.queryName());
  6586. att->setProp("@value", t.queryProp(NULL));
  6587. }
  6588. }
  6589. }
  6590. }
  6591. Owned<IPropertyTreeIterator> nodes = progressNode->getElements("node");
  6592. ForEach (*nodes)
  6593. {
  6594. IPropertyTree &node = nodes->query();
  6595. StringBuffer nodePath("node[@id=\"");
  6596. nodePath.append(node.queryProp("@id")).append("\"]");
  6597. IPropertyTree *_node = graphNode->queryPropTree(nodePath.str());
  6598. if (_node)
  6599. {
  6600. if (progressV < 1)
  6601. mergePTree(_node, &node);
  6602. else
  6603. { // must translate to XGMML format
  6604. expandAttributes(*_node, node);
  6605. }
  6606. }
  6607. }
  6608. }
  6609. Owned<IPropertyTreeIterator> iter = graphNode->getElements("node");
  6610. ForEach (*iter)
  6611. mergeProgress(iter->query(), progressTree, progressV);
  6612. }
  6613. IPropertyTree * CLocalWUGraph::getXGMMLTreeRaw() const
  6614. {
  6615. return p->getPropTree("xgmml");
  6616. }
  6617. IPropertyTree * CLocalWUGraph::getXGMMLTree(bool doMergeProgress) const
  6618. {
  6619. if (!graph)
  6620. {
  6621. // NB: although graphBin introduced in wuidVersion==2,
  6622. // daliadmin can retrospectively compress existing graphs, so need to check for all versions
  6623. MemoryBuffer mb;
  6624. if (p->getPropBin("xgmml/graphBin", mb))
  6625. graph.setown(createPTree(mb));
  6626. else
  6627. graph.setown(p->getBranch("xgmml/graph"));
  6628. if (!graph)
  6629. return NULL;
  6630. }
  6631. if (!doMergeProgress)
  6632. return graph.getLink();
  6633. else
  6634. {
  6635. Owned<IPropertyTree> copy = createPTreeFromIPT(graph);
  6636. Owned<IConstWUGraphProgress> _progress;
  6637. if (progress) _progress.set(progress);
  6638. else
  6639. _progress.setown(new CConstGraphProgress(owner.queryWuid(), p->queryProp("@name")));
  6640. //MORE: Eventually this should directly access the new stats structure
  6641. unsigned progressV = _progress->queryFormatVersion();
  6642. Owned<IPropertyTree> progressTree = _progress->getProgressTree();
  6643. Owned<IPropertyTreeIterator> nodeIterator = copy->getElements("node");
  6644. ForEach (*nodeIterator)
  6645. mergeProgress(nodeIterator->query(), *progressTree, progressV);
  6646. return copy.getClear();
  6647. }
  6648. }
  6649. WUGraphType CLocalWUGraph::getType() const
  6650. {
  6651. return (WUGraphType) getEnum(p, "@type", graphTypes);
  6652. }
  6653. IStringVal & CLocalWUGraph::getTypeName(IStringVal &str) const
  6654. {
  6655. str.set(p->queryProp("@type"));
  6656. if (!str.length())
  6657. str.set("unknown");
  6658. return str;
  6659. }
  6660. void CLocalWUGraph::setType(WUGraphType _type)
  6661. {
  6662. setEnum(p, "@type", _type, graphTypes);
  6663. }
  6664. //=================================================================================================
  6665. mapEnums queryFileTypes[] = {
  6666. { FileTypeCpp, "cpp" },
  6667. { FileTypeDll, "dll" },
  6668. { FileTypeResText, "res" },
  6669. { FileTypeHintXml, "hint" },
  6670. { FileTypeXml, "xml" },
  6671. { FileTypeSize, NULL },
  6672. };
  6673. CLocalWUAssociated::CLocalWUAssociated(IPropertyTree *props) : p(props)
  6674. {
  6675. }
  6676. WUFileType CLocalWUAssociated::getType() const
  6677. {
  6678. return (WUFileType)getEnum(p, "@type", queryFileTypes);
  6679. }
  6680. IStringVal & CLocalWUAssociated::getDescription(IStringVal & str) const
  6681. {
  6682. str.set(p->queryProp("@desc"));
  6683. return str;
  6684. }
  6685. IStringVal & CLocalWUAssociated::getIp(IStringVal & str) const
  6686. {
  6687. str.set(p->queryProp("@ip"));
  6688. return str;
  6689. }
  6690. IStringVal & CLocalWUAssociated::getName(IStringVal & str) const
  6691. {
  6692. str.set(p->queryProp("@filename"));
  6693. return str;
  6694. }
  6695. IStringVal & CLocalWUAssociated::getNameTail(IStringVal & str) const
  6696. {
  6697. str.set(pathTail(p->queryProp("@filename")));
  6698. return str;
  6699. }
  6700. unsigned CLocalWUAssociated::getCrc() const
  6701. {
  6702. return p->getPropInt("@crc", 0);
  6703. }
  6704. //=================================================================================================
  6705. CLocalWUQuery::CLocalWUQuery(IPropertyTree *props) : p(props)
  6706. {
  6707. associatedCached = false;
  6708. }
  6709. mapEnums queryTypes[] = {
  6710. { QueryTypeUnknown, "unknown" },
  6711. { QueryTypeEcl, "ECL" },
  6712. { QueryTypeSql, "SQL" },
  6713. { QueryTypeXml, "XML" },
  6714. { QueryTypeAttribute, "Attribute" },
  6715. { QueryTypeSize, NULL },
  6716. };
  6717. WUQueryType CLocalWUQuery::getQueryType() const
  6718. {
  6719. return (WUQueryType) getEnum(p, "@type", queryTypes);
  6720. }
  6721. void CLocalWUQuery::setQueryType(WUQueryType qt)
  6722. {
  6723. setEnum(p, "@type", qt, queryTypes);
  6724. }
  6725. IStringVal& CLocalWUQuery::getQueryText(IStringVal &str) const
  6726. {
  6727. str.set(p->queryProp("Text"));
  6728. return str;
  6729. }
  6730. IStringVal& CLocalWUQuery::getQueryShortText(IStringVal &str) const
  6731. {
  6732. const char * text = p->queryProp("Text");
  6733. if (isArchiveQuery(text))
  6734. {
  6735. Owned<IPropertyTree> xml = createPTreeFromXMLString(text, ipt_caseInsensitive);
  6736. const char * path = xml->queryProp("Query/@attributePath");
  6737. if (path)
  6738. {
  6739. IPropertyTree * resolved = resolveDefinitionInArchive(xml, path);
  6740. if (resolved)
  6741. str.set(resolved->queryProp(NULL));
  6742. }
  6743. else
  6744. str.set(xml->queryProp("Query"));
  6745. }
  6746. else
  6747. str.set(text);
  6748. return str;
  6749. }
  6750. IStringVal& CLocalWUQuery::getQueryName(IStringVal &str) const
  6751. {
  6752. str.set(p->queryProp("@name"));
  6753. return str;
  6754. }
  6755. IStringVal & CLocalWUQuery::getQueryMainDefinition(IStringVal & str) const
  6756. {
  6757. str.set(p->queryProp("@main"));
  6758. return str;
  6759. }
  6760. IStringVal& CLocalWUQuery::getQueryDllName(IStringVal &str) const
  6761. {
  6762. Owned<IConstWUAssociatedFile> entry = getAssociatedFile(FileTypeDll, 0);
  6763. if (entry)
  6764. entry->getNameTail(str);
  6765. return str;
  6766. }
  6767. IStringVal& CLocalWUQuery::getQueryCppName(IStringVal &str) const
  6768. {
  6769. Owned<IConstWUAssociatedFile> entry = getAssociatedFile(FileTypeCpp, 0);
  6770. if (entry)
  6771. entry->getName(str);
  6772. return str;
  6773. }
  6774. IStringVal& CLocalWUQuery::getQueryResTxtName(IStringVal &str) const
  6775. {
  6776. Owned<IConstWUAssociatedFile> entry = getAssociatedFile(FileTypeResText, 0);
  6777. if (entry)
  6778. entry->getName(str);
  6779. return str;
  6780. }
  6781. unsigned CLocalWUQuery::getQueryDllCrc() const
  6782. {
  6783. Owned<IConstWUAssociatedFile> entry = getAssociatedFile(FileTypeDll, 0);
  6784. if (entry)
  6785. return entry->getCrc();
  6786. return 0;
  6787. }
  6788. void CLocalWUQuery::setQueryText(const char *text)
  6789. {
  6790. p->setProp("Text", text);
  6791. }
  6792. void CLocalWUQuery::setQueryName(const char *qname)
  6793. {
  6794. p->setProp("@name", qname);
  6795. }
  6796. void CLocalWUQuery::setQueryMainDefinition(const char * str)
  6797. {
  6798. p->setProp("@main", str);
  6799. }
  6800. void CLocalWUQuery::addAssociatedFile(WUFileType type, const char * name, const char * ip, const char * desc, unsigned crc)
  6801. {
  6802. CriticalBlock block(crit);
  6803. loadAssociated();
  6804. if (!associated.length())
  6805. p->addPropTree("Associated", createPTree("Associated"));
  6806. IPropertyTree *pl = p->queryPropTree("Associated");
  6807. IPropertyTree *s = pl->addPropTree("File", createPTree("File"));
  6808. setEnum(s, "@type", type, queryFileTypes);
  6809. s->setProp("@filename", name);
  6810. s->setProp("@ip", ip);
  6811. s->setProp("@desc", desc);
  6812. if (crc)
  6813. s->setPropInt("@crc", crc);
  6814. IConstWUAssociatedFile * q = new CLocalWUAssociated(LINK(s));
  6815. associated.append(*q);
  6816. }
  6817. void CLocalWUQuery::removeAssociatedFiles()
  6818. {
  6819. associatedCached = false;
  6820. associated.kill();
  6821. p->removeProp("Associated");
  6822. }
  6823. IConstWUAssociatedFile * CLocalWUQuery::getAssociatedFile(WUFileType type, unsigned index) const
  6824. {
  6825. CriticalBlock block(crit);
  6826. loadAssociated();
  6827. ForEachItemIn(idx, associated)
  6828. {
  6829. CLocalWUAssociated &cur = static_cast<CLocalWUAssociated &>(associated.item(idx));
  6830. if (cur.getType() == type)
  6831. {
  6832. if (index-- == 0)
  6833. return &OLINK(cur);
  6834. }
  6835. }
  6836. return NULL;
  6837. }
  6838. void CLocalWUQuery::addSpecialCaseAssociated(WUFileType type, const char * propname, unsigned crc) const
  6839. {
  6840. const char * name = p->queryProp(propname);
  6841. if (name)
  6842. {
  6843. IPropertyTree *s = createPTree("File");
  6844. setEnum(s, "@type", type, queryFileTypes);
  6845. s->setProp("@filename", name);
  6846. if (crc)
  6847. s->setPropInt("@crc", crc);
  6848. associated.append(*new CLocalWUAssociated(s));
  6849. }
  6850. }
  6851. void CLocalWUQuery::loadAssociated() const
  6852. {
  6853. CriticalBlock block(crit);
  6854. if (!associatedCached)
  6855. {
  6856. assertex(associated.length() == 0);
  6857. addSpecialCaseAssociated(FileTypeDll, "DllName", p->getPropInt("DllCrc", 0));
  6858. addSpecialCaseAssociated(FileTypeCpp, "CppName", 0);
  6859. addSpecialCaseAssociated(FileTypeResText, "ResTxtName", 0);
  6860. Owned<IPropertyTreeIterator> r = p->getElements("Associated/File");
  6861. for (r->first(); r->isValid(); r->next())
  6862. {
  6863. IPropertyTree *rp = &r->query();
  6864. rp->Link();
  6865. associated.append(*new CLocalWUAssociated(rp));
  6866. }
  6867. associatedCached = true;
  6868. }
  6869. }
  6870. IConstWUAssociatedFileIterator& CLocalWUQuery::getAssociatedFiles() const
  6871. {
  6872. CriticalBlock block(crit);
  6873. loadAssociated();
  6874. return *new CArrayIteratorOf<IConstWUAssociatedFile,IConstWUAssociatedFileIterator> (associated, 0, (IConstWUQuery *) this);
  6875. }
  6876. //========================================================================================
  6877. CLocalWUWebServicesInfo::CLocalWUWebServicesInfo(IPropertyTree *props) : p(props)
  6878. {
  6879. }
  6880. IStringVal& CLocalWUWebServicesInfo::getModuleName(IStringVal &str) const
  6881. {
  6882. str.set(p->queryProp("@module"));
  6883. return str;
  6884. }
  6885. IStringVal& CLocalWUWebServicesInfo::getAttributeName(IStringVal &str) const
  6886. {
  6887. str.set(p->queryProp("@attribute"));
  6888. return str;
  6889. }
  6890. IStringVal& CLocalWUWebServicesInfo::getDefaultName(IStringVal &str) const
  6891. {
  6892. str.set(p->queryProp("@defaultName"));
  6893. return str;
  6894. }
  6895. unsigned CLocalWUWebServicesInfo::getWebServicesCRC() const
  6896. {
  6897. return (unsigned) p->getPropInt("@crc");
  6898. }
  6899. IStringVal& CLocalWUWebServicesInfo::getInfo(const char *name, IStringVal &str) const
  6900. {
  6901. if (!name)
  6902. {
  6903. StringBuffer ws_info;
  6904. ws_info.appendf("<%s ", p->queryName());
  6905. Owned<IAttributeIterator> attrs = p->getAttributes();
  6906. for(attrs->first(); attrs->isValid(); attrs->next())
  6907. {
  6908. const char *name = attrs->queryName()+1;
  6909. const char *value = attrs->queryValue();
  6910. ws_info.appendf("%s='%s' ", name, value);
  6911. }
  6912. ws_info.append("> \n");
  6913. Owned<IPropertyTreeIterator> info = p->getElements("*");
  6914. ForEach(*info)
  6915. {
  6916. IPropertyTree &item = info->query();
  6917. const char *name = item.queryName();
  6918. if (name)
  6919. {
  6920. MemoryBuffer mb;
  6921. bool isbin = p->isBinary(name);
  6922. if (isbin)
  6923. {
  6924. p->getPropBin(name,mb);
  6925. if (mb.length())
  6926. {
  6927. unsigned len = 0;
  6928. mb.read(len);
  6929. StringBuffer encodedString;
  6930. StringBuffer val(len, (const char *) mb.readDirect(len));
  6931. encodeXML(val, encodedString);
  6932. ws_info.appendf("<%s>%s</%s>", name, encodedString.str(), name);
  6933. }
  6934. }
  6935. else
  6936. {
  6937. StringBuffer tmp;
  6938. toXML(&item, tmp);
  6939. ws_info.append(tmp.str());
  6940. }
  6941. }
  6942. }
  6943. ws_info.appendf("</%s>", p->queryName());
  6944. str.setLen(ws_info.str(), ws_info.length());
  6945. }
  6946. else
  6947. {
  6948. MemoryBuffer mb;
  6949. p->getPropBin(name,mb);
  6950. if (mb.length())
  6951. {
  6952. unsigned len;
  6953. mb.read(len);
  6954. str.setLen((const char *) mb.readDirect(len), len);
  6955. }
  6956. }
  6957. return str;
  6958. }
  6959. void CLocalWUWebServicesInfo::setModuleName(const char *mname)
  6960. {
  6961. p->setProp("@module", mname);
  6962. }
  6963. void CLocalWUWebServicesInfo::setAttributeName(const char *aname)
  6964. {
  6965. p->setProp("@attribute", aname);
  6966. }
  6967. void CLocalWUWebServicesInfo::setDefaultName(const char *dname)
  6968. {
  6969. p->setProp("@defaultName", dname);
  6970. }
  6971. void CLocalWUWebServicesInfo::setWebServicesCRC(unsigned crc)
  6972. {
  6973. p->setPropInt("@crc", crc);
  6974. }
  6975. void CLocalWUWebServicesInfo::setInfo(const char *name, const char *info)
  6976. {
  6977. MemoryBuffer m;
  6978. unsigned len = (size32_t)strlen(info);
  6979. serializeLPString(len, info, m);
  6980. p->setPropBin(name, m.length(), m.toByteArray());
  6981. }
  6982. //========================================================================================
  6983. CLocalWURoxieQueryInfo::CLocalWURoxieQueryInfo(IPropertyTree *props) : p(props)
  6984. {
  6985. }
  6986. IStringVal& CLocalWURoxieQueryInfo::getQueryInfo(IStringVal &str) const
  6987. {
  6988. IPropertyTree *queryTree = p->queryPropTree("query");
  6989. if (queryTree)
  6990. {
  6991. StringBuffer temp;
  6992. toXML(queryTree, temp);
  6993. str.set(temp.str());
  6994. }
  6995. return str;
  6996. }
  6997. IStringVal& CLocalWURoxieQueryInfo::getDefaultPackageInfo(IStringVal &str) const
  6998. {
  6999. MemoryBuffer mb;
  7000. p->getPropBin("RoxiePackages",mb);
  7001. if (mb.length())
  7002. {
  7003. unsigned len;
  7004. mb.read(len);
  7005. str.setLen((const char *) mb.readDirect(len), len);
  7006. }
  7007. return str;
  7008. }
  7009. IStringVal& CLocalWURoxieQueryInfo::getRoxieClusterName(IStringVal &str) const
  7010. {
  7011. const char *val = p->queryProp("@roxieClusterName");
  7012. if (val)
  7013. str.set(val);
  7014. return str;
  7015. }
  7016. IStringVal& CLocalWURoxieQueryInfo::getWuid(IStringVal &str) const
  7017. {
  7018. const char *val = p->queryProp("@wuid");
  7019. if (val)
  7020. str.set(val);
  7021. return str;
  7022. }
  7023. void CLocalWURoxieQueryInfo::setQueryInfo(const char *info)
  7024. {
  7025. IPropertyTree *queryTree = p->queryPropTree("query");
  7026. if (queryTree)
  7027. p->removeTree(queryTree);
  7028. IPropertyTree * tempTree = p->addPropTree("query", createPTreeFromXMLString(info));
  7029. if (!p->hasProp("@roxieClusterName"))
  7030. {
  7031. const char *roxieClusterName = tempTree->queryProp("@roxieName");
  7032. if (roxieClusterName && *roxieClusterName)
  7033. p->addProp("@roxieClusterName", roxieClusterName);
  7034. }
  7035. if (!p->hasProp("@wuid"))
  7036. {
  7037. const char *wuid = tempTree->queryProp("Query/@wuid");
  7038. if (wuid && *wuid)
  7039. p->addProp("@wuid", wuid);
  7040. }
  7041. }
  7042. void CLocalWURoxieQueryInfo::setDefaultPackageInfo(const char *info, int len)
  7043. {
  7044. MemoryBuffer m;
  7045. serializeLPString(len, info, m);
  7046. p->setPropBin("RoxiePackages", m.length(), m.toByteArray());
  7047. }
  7048. void CLocalWURoxieQueryInfo::setRoxieClusterName(const char *info)
  7049. {
  7050. p->setProp("@roxieClusterName", info);
  7051. }
  7052. void CLocalWURoxieQueryInfo::setWuid(const char *info)
  7053. {
  7054. p->setProp("@wuid", info);
  7055. }
  7056. //========================================================================================
  7057. CLocalWUResult::CLocalWUResult(IPropertyTree *props) : p(props)
  7058. {
  7059. }
  7060. mapEnums resultStatuses[] = {
  7061. { ResultStatusUndefined, "undefined" },
  7062. { ResultStatusCalculated, "calculated" },
  7063. { ResultStatusSupplied, "supplied" },
  7064. { ResultStatusFailed, "failed" },
  7065. { ResultStatusPartial, "partial" },
  7066. { ResultStatusSize, NULL }
  7067. };
  7068. WUResultStatus CLocalWUResult::getResultStatus() const
  7069. {
  7070. return (WUResultStatus ) getEnum(p, "@status", resultStatuses);
  7071. }
  7072. IStringVal& CLocalWUResult::getResultName(IStringVal &str) const
  7073. {
  7074. str.set(p->queryProp("@name"));
  7075. return str;
  7076. }
  7077. int CLocalWUResult::getResultSequence() const
  7078. {
  7079. return p->getPropInt("@sequence", -1);
  7080. }
  7081. bool CLocalWUResult::isResultScalar() const
  7082. {
  7083. return p->getPropInt("@isScalar", 1) != 0;
  7084. }
  7085. bool findSize(int size, IntArray &sizes)
  7086. {
  7087. ForEachItemIn(idx, sizes)
  7088. {
  7089. if (sizes.item(idx)==size)
  7090. return true;
  7091. }
  7092. return false;
  7093. }
  7094. void CLocalWUResult::getSchema(TypeInfoArray &types, StringAttrArray &names, IStringVal * eclText) const
  7095. {
  7096. MemoryBuffer schema;
  7097. p->getPropBin("SchemaRaw", schema);
  7098. if (schema.length())
  7099. {
  7100. for (;;)
  7101. {
  7102. StringAttr name;
  7103. schema.read(name);
  7104. if (*schema.readDirect(0)==type_void)
  7105. break;
  7106. names.append(*new StringAttrItem(name));
  7107. types.append(*deserializeType(schema)); // MORE - nested records!
  7108. }
  7109. schema.skip(1);
  7110. if (schema.length() != schema.getPos())
  7111. {
  7112. unsigned eclLen;
  7113. schema.read(eclLen);
  7114. const char * schemaData = (const char *)schema.readDirect(eclLen);
  7115. if (eclText)
  7116. {
  7117. eclText->setLen(schemaData, eclLen);
  7118. if ((eclLen == 0) && names.ordinality())
  7119. {
  7120. const char * firstName = names.item(0).text;
  7121. StringBuffer temp;
  7122. temp.append("RECORD ");
  7123. types.item(0).getECLType(temp);
  7124. temp.append(" value{NAMED('").append(firstName).append("')}").append("; END;");
  7125. eclText->set(temp.str());
  7126. }
  7127. }
  7128. }
  7129. }
  7130. }
  7131. void readRow(StringBuffer &out, MemoryBuffer &in, TypeInfoArray &types, StringAttrArray &names)
  7132. {
  7133. ForEachItemIn(idx, types)
  7134. {
  7135. StringAttrItem &name = names.item(idx);
  7136. ITypeInfo &type = types.item(idx);
  7137. unsigned size = type.getSize();
  7138. switch(type.getTypeCode())
  7139. {
  7140. case type_data:
  7141. if (size==UNKNOWN_LENGTH)
  7142. {
  7143. if (in.remaining() < sizeof(int))
  7144. throw MakeStringException(WUERR_CorruptResult, "corrupt workunit information");
  7145. in.read(size);
  7146. }
  7147. outputXmlData(size, in.readDirect(size), name.text, out);
  7148. break;
  7149. case type_string:
  7150. if (size==UNKNOWN_LENGTH)
  7151. {
  7152. if (in.remaining() < sizeof(int))
  7153. throw MakeStringException(WUERR_CorruptResult, "corrupt workunit information");
  7154. in.read(size);
  7155. }
  7156. outputXmlString(size, (const char *) in.readDirect(size), name.text, out);
  7157. break;
  7158. case type_varstring:
  7159. {
  7160. if (size == UNKNOWN_LENGTH)
  7161. size = (size32_t)strlen((const char *) in.readDirect(0))+1;
  7162. const char * text = (const char *) in.readDirect(size);
  7163. outputXmlString((size32_t)strlen(text), text, name.text, out);
  7164. break;
  7165. }
  7166. case type_unicode:
  7167. {
  7168. unsigned len = type.getStringLen();
  7169. if (size==UNKNOWN_LENGTH)
  7170. in.read(len);
  7171. outputXmlUnicode(len, (UChar const *) in.readDirect(len*2), name.text, out);
  7172. }
  7173. break;
  7174. case type_utf8:
  7175. {
  7176. unsigned len = type.getStringLen();
  7177. if (size==UNKNOWN_LENGTH)
  7178. {
  7179. in.read(len);
  7180. size = rtlUtf8Size(len, in.readDirect(0));
  7181. }
  7182. outputXmlUtf8(len, (const char *) in.readDirect(size), name.text, out);
  7183. }
  7184. break;
  7185. case type_qstring:
  7186. {
  7187. unsigned len = type.getStringLen();
  7188. if (size==UNKNOWN_LENGTH)
  7189. in.read(len);
  7190. unsigned outlen;
  7191. char *outstr;
  7192. rtlQStrToStrX(outlen, outstr, len, (const char *) in.readDirect(rtlQStrSize(len)));
  7193. outputXmlString(outlen, outstr, name.text, out);
  7194. free(outstr);
  7195. break;
  7196. }
  7197. case type_int:
  7198. case type_swapint:
  7199. if (type.isSigned())
  7200. {
  7201. const unsigned char *raw = (const unsigned char *) in.readDirect(size);
  7202. unsigned __int64 cval8 = 0;
  7203. //MORE: I think this is wrong - swapped doesn't mean little/big/
  7204. if (type.isSwappedEndian())
  7205. {
  7206. unsigned idx = 0;
  7207. if (raw[idx] & 0x80)
  7208. cval8 = (__int64)-1;
  7209. while (size--)
  7210. cval8 = (cval8 << 8) | raw[idx++];
  7211. }
  7212. else
  7213. {
  7214. if (raw[size-1] & 0x80)
  7215. cval8 = (__int64)-1;
  7216. while (size--)
  7217. cval8 = (cval8 << 8) | raw[size];
  7218. }
  7219. outputXmlInt((__int64) cval8, name.text, out);
  7220. }
  7221. else
  7222. {
  7223. const unsigned char *raw = (const unsigned char *) in.readDirect(size);
  7224. unsigned __int64 cval8 = 0;
  7225. if (type.isSwappedEndian())
  7226. {
  7227. unsigned idx = 0;
  7228. while (size--)
  7229. cval8 = (cval8 << 8) | raw[idx++];
  7230. }
  7231. else
  7232. {
  7233. while (size--)
  7234. cval8 = (cval8 << 8) | raw[size];
  7235. }
  7236. outputXmlUInt(cval8, name.text, out);
  7237. }
  7238. break;
  7239. case type_boolean:
  7240. bool cvalb;
  7241. in.read(cvalb);
  7242. outputXmlBool(cvalb, name.text, out);
  7243. break;
  7244. case type_decimal:
  7245. if (type.isSigned())
  7246. outputXmlDecimal(in.readDirect(size), size, type.getPrecision(), name.text, out);
  7247. else
  7248. outputXmlUDecimal(in.readDirect(size), size, type.getPrecision(), name.text, out);
  7249. break;
  7250. case type_real:
  7251. double cvald;
  7252. switch(size)
  7253. {
  7254. case 4:
  7255. float cvalf;
  7256. in.read(cvalf);
  7257. cvald = cvalf;
  7258. break;
  7259. case 8:
  7260. in.read(cvald);
  7261. break;
  7262. }
  7263. outputXmlReal(cvald, name.text, out);
  7264. break;
  7265. default:
  7266. assertex(!"unexpected type in raw record");
  7267. break;
  7268. }
  7269. }
  7270. }
  7271. IStringVal& CLocalWUResult::getResultXml(IStringVal &str) const
  7272. {
  7273. TypeInfoArray types;
  7274. StringAttrArray names;
  7275. getSchema(types, names);
  7276. StringBuffer xml;
  7277. const char * name = p->queryProp("@name");
  7278. if (name)
  7279. xml.appendf("<Dataset name=\'%s\'>\n", name);
  7280. else
  7281. xml.append("<Dataset>\n");
  7282. if (p->hasProp("Value"))
  7283. {
  7284. MemoryBuffer raw;
  7285. p->getPropBin("Value", raw);
  7286. unsigned __int64 numrows = getResultRowCount();
  7287. while (numrows--)
  7288. {
  7289. xml.append(" <Row>");
  7290. readRow(xml, raw, types, names);
  7291. xml.append("</Row>\n");
  7292. }
  7293. }
  7294. else if (p->hasProp("xmlValue"))
  7295. {
  7296. xml.append(" <Row>");
  7297. appendXMLTag(xml, name, p->queryProp("xmlValue"));
  7298. xml.append("</Row>\n");
  7299. }
  7300. xml.append("</Dataset>\n");
  7301. str.set(xml.str());
  7302. return str;
  7303. }
  7304. IProperties *CLocalWUResult::queryResultXmlns()
  7305. {
  7306. CriticalBlock block(crit);
  7307. if (xmlns)
  7308. return xmlns;
  7309. xmlns.setown(createProperties());
  7310. Owned<IAttributeIterator> it = p->getAttributes();
  7311. unsigned prefixLen = strlen("@xmlns");
  7312. ForEach(*it)
  7313. {
  7314. const char *name = it->queryName();
  7315. if (!strncmp("@xmlns", name, prefixLen))
  7316. {
  7317. if (name[prefixLen]==':') //normal case
  7318. xmlns->setProp(name+prefixLen+1, it->queryValue());
  7319. else if (!name[prefixLen]) //special case, unprefixed namespace
  7320. xmlns->setProp("xmlns", it->queryValue());
  7321. }
  7322. }
  7323. return xmlns;
  7324. }
  7325. unsigned CLocalWUResult::getResultFetchSize() const
  7326. {
  7327. return p->getPropInt("fetchSize", 100);
  7328. }
  7329. __int64 CLocalWUResult::getResultTotalRowCount() const
  7330. {
  7331. return p->getPropInt64("totalRowCount", -1);
  7332. }
  7333. __int64 CLocalWUResult::getResultRowCount() const
  7334. {
  7335. return p->getPropInt64("rowCount", 0);
  7336. }
  7337. void CLocalWUResult::getResultDataset(IStringVal & ecl, IStringVal & defs) const
  7338. {
  7339. ecl.set(p->queryProp("datasetEcl"));
  7340. defs.set(p->queryProp("datasetEclDefs"));
  7341. }
  7342. IStringVal& CLocalWUResult::getResultLogicalName(IStringVal & val) const
  7343. {
  7344. val.set(p->queryProp("logicalName"));
  7345. return val;
  7346. }
  7347. IStringVal& CLocalWUResult::getResultKeyField(IStringVal & ecl) const
  7348. {
  7349. ecl.set(p->queryProp("keyField"));
  7350. return ecl;
  7351. }
  7352. unsigned CLocalWUResult::getResultRequestedRows() const
  7353. {
  7354. return p->getPropInt("requestedRows", 1);
  7355. }
  7356. IStringVal& CLocalWUResult::getResultEclSchema(IStringVal & str) const
  7357. {
  7358. TypeInfoArray types;
  7359. StringAttrArray names;
  7360. getSchema(types, names, &str);
  7361. return str;
  7362. }
  7363. IStringVal& CLocalWUResult::getResultRecordSizeEntry(IStringVal & str) const
  7364. {
  7365. str.set(p->queryProp("@recordSizeEntry"));
  7366. return str;
  7367. }
  7368. IStringVal& CLocalWUResult::getResultTransformerEntry(IStringVal & str) const
  7369. {
  7370. str.set(p->queryProp("@transformerEntry"));
  7371. return str;
  7372. }
  7373. __int64 CLocalWUResult::getResultRowLimit() const
  7374. {
  7375. return p->getPropInt64("@rowLimit");
  7376. }
  7377. IStringVal& CLocalWUResult::getResultFilename(IStringVal & str) const
  7378. {
  7379. str.set(p->queryProp("@tempFilename"));
  7380. return str;
  7381. }
  7382. void CLocalWUResult::setResultStatus(WUResultStatus status)
  7383. {
  7384. setEnum(p, "@status", status, resultStatuses);
  7385. if (status==ResultStatusUndefined)
  7386. p->removeProp("Value");
  7387. }
  7388. void CLocalWUResult::setResultName(const char *s)
  7389. {
  7390. p->setProp("@name", s);
  7391. }
  7392. void CLocalWUResult::setResultSequence(unsigned seq)
  7393. {
  7394. p->setPropInt("@sequence", seq);
  7395. }
  7396. void CLocalWUResult::setResultSchemaRaw(unsigned size, const void *schema)
  7397. {
  7398. p->setPropBin("SchemaRaw", size, schema);
  7399. }
  7400. void CLocalWUResult::setResultXmlns(const char *prefix, const char *uri)
  7401. {
  7402. StringBuffer xpath("@xmlns");
  7403. if (prefix && *prefix)
  7404. xpath.append(':').append(prefix);
  7405. p->setProp(xpath, uri);
  7406. }
  7407. void CLocalWUResult::setResultScalar(bool isScalar)
  7408. {
  7409. p->setPropInt("@isScalar", (int) isScalar);
  7410. if (isScalar)
  7411. setResultTotalRowCount(1);
  7412. }
  7413. void CLocalWUResult::setResultRaw(unsigned len, const void *data, WUResultFormat format)
  7414. {
  7415. p->setPropBin("Value", len, data);
  7416. setResultStatus(ResultStatusSupplied);
  7417. setResultFormat(format);
  7418. }
  7419. void CLocalWUResult::setResultFormat(WUResultFormat format)
  7420. {
  7421. switch (format)
  7422. {
  7423. case ResultFormatXml:
  7424. p->setProp("@format","xml");
  7425. break;
  7426. case ResultFormatXmlSet:
  7427. p->setProp("@format","xmlSet");
  7428. break;
  7429. case ResultFormatCsv:
  7430. p->setProp("@format","csv");
  7431. break;
  7432. default:
  7433. p->removeProp("@format");
  7434. break;
  7435. }
  7436. }
  7437. void CLocalWUResult::setResultXML(const char *val)
  7438. {
  7439. p->setProp("xmlValue", val);
  7440. }
  7441. void CLocalWUResult::addResultRaw(unsigned len, const void *data, WUResultFormat format)
  7442. {
  7443. p->appendPropBin("Value", len, data);
  7444. setResultStatus(ResultStatusPartial);
  7445. const char *existingFormat = p->queryProp("@format");
  7446. const char *formatStr = NULL;
  7447. switch (format)
  7448. {
  7449. case ResultFormatXml:
  7450. formatStr = "xml";
  7451. break;
  7452. case ResultFormatXmlSet:
  7453. formatStr = "xmlSet";
  7454. break;
  7455. case ResultFormatCsv:
  7456. formatStr = "csv";
  7457. break;
  7458. default:
  7459. p->removeProp("@format");
  7460. break;
  7461. }
  7462. if (format)
  7463. {
  7464. if (existingFormat)
  7465. {
  7466. if (0 != stricmp(formatStr, existingFormat))
  7467. throw MakeStringException(WUERR_ResultFormatMismatch, "addResult format %s, does not match existing format %s", formatStr, existingFormat);
  7468. }
  7469. else
  7470. p->setProp("@format", formatStr);
  7471. }
  7472. }
  7473. void CLocalWUResult::setResultFetchSize(unsigned rows)
  7474. {
  7475. p->setPropInt("fetchSize", rows);
  7476. }
  7477. void CLocalWUResult::setResultTotalRowCount(__int64 rows)
  7478. {
  7479. p->setPropInt64("totalRowCount", rows);
  7480. }
  7481. void CLocalWUResult::setResultRowCount(__int64 rows)
  7482. {
  7483. p->setPropInt64("rowCount", rows);
  7484. }
  7485. void CLocalWUResult::setResultDataset(const char *ecl, const char *defs)
  7486. {
  7487. p->setProp("datasetEcl", ecl);
  7488. p->setProp("datasetEclDefs", defs);
  7489. }
  7490. void CLocalWUResult::setResultLogicalName(const char *logicalName)
  7491. {
  7492. p->setProp("logicalName", logicalName);
  7493. }
  7494. void CLocalWUResult::setResultKeyField(const char *ecl)
  7495. {
  7496. p->setProp("keyField", ecl);
  7497. }
  7498. void CLocalWUResult::setResultRequestedRows(unsigned rows)
  7499. {
  7500. p->setPropInt("requestedRows", rows);
  7501. }
  7502. void CLocalWUResult::setResultRecordSizeEntry(const char * entry)
  7503. {
  7504. p->setProp("@recordSizeEntry", entry);
  7505. }
  7506. void CLocalWUResult::setResultTransformerEntry(const char * entry)
  7507. {
  7508. p->setProp("@transformerEntry", entry);
  7509. }
  7510. void CLocalWUResult::setResultRowLimit(__int64 value)
  7511. {
  7512. p->setPropInt64("@rowLimit", value);
  7513. }
  7514. void CLocalWUResult::setResultFilename(const char * name)
  7515. {
  7516. p->setProp("@tempFilename", name);
  7517. }
  7518. // MORE - it's an undetected error if we call getResult... of a type that does not match schema
  7519. __int64 CLocalWUResult::getResultInt() const
  7520. {
  7521. __int64 result = 0;
  7522. MemoryBuffer s;
  7523. p->getPropBin("Value", s);
  7524. if (s.length())
  7525. s.read(result);
  7526. else
  7527. result = p->getPropInt64("xmlValue");
  7528. return result;
  7529. }
  7530. bool CLocalWUResult::getResultBool() const
  7531. {
  7532. bool result = false;
  7533. MemoryBuffer s;
  7534. p->getPropBin("Value", s);
  7535. if (s.length())
  7536. s.read(result);
  7537. else
  7538. result = p->getPropBool("xmlValue");
  7539. return result;
  7540. }
  7541. double CLocalWUResult::getResultReal() const
  7542. {
  7543. double result = 0;
  7544. MemoryBuffer s;
  7545. p->getPropBin("Value", s);
  7546. if (s.length())
  7547. s.read(result);
  7548. else
  7549. {
  7550. const char *xmlVal = p->queryProp("xmlValue");
  7551. if (xmlVal)
  7552. result = atof(xmlVal);
  7553. }
  7554. return result;
  7555. }
  7556. void CLocalWUResult::getResultDecimal(void * val, unsigned len, unsigned precision, bool isSigned) const
  7557. {
  7558. MemoryBuffer s;
  7559. p->getPropBin("Value", s);
  7560. if (s.length())
  7561. {
  7562. assertex(s.length() == len);
  7563. s.read(len, val);
  7564. }
  7565. else
  7566. {
  7567. const char *xmlVal = p->queryProp("xmlValue");
  7568. if (xmlVal)
  7569. {
  7570. Decimal d;
  7571. d.setString(strlen(xmlVal), xmlVal);
  7572. if (isSigned)
  7573. d.getDecimal(len, precision, val);
  7574. else
  7575. d.getUDecimal(len, precision, val);
  7576. }
  7577. else
  7578. memset(val, 0, len);
  7579. }
  7580. }
  7581. IStringVal& CLocalWUResult::getResultString(IStringVal & str) const
  7582. {
  7583. MemoryBuffer s;
  7584. p->getPropBin("Value", s);
  7585. if (s.length())
  7586. {
  7587. unsigned len;
  7588. s.read(len);
  7589. str.setLen((const char *) s.readDirect(len), len);
  7590. }
  7591. else
  7592. {
  7593. p->getPropBin("xmlValue", s);
  7594. if (p->isBinary("xmlValue"))
  7595. str.setLen(s.toByteArray(), s.length());
  7596. else
  7597. {
  7598. char *ascii = rtlUtf8ToVStr(rtlUtf8Length(s.length(), s.toByteArray()), s.toByteArray());
  7599. str.set(ascii);
  7600. rtlFree(ascii);
  7601. }
  7602. }
  7603. return str;
  7604. }
  7605. WUResultFormat CLocalWUResult::getResultFormat() const
  7606. {
  7607. const char * format = p->queryProp("@format");
  7608. if (!format)
  7609. return ResultFormatRaw;
  7610. else if (strcmp(format, "xml") == 0)
  7611. return ResultFormatXml;
  7612. else if (strcmp(format, "xmlSet") == 0)
  7613. return ResultFormatXmlSet;
  7614. else if (strcmp(format, "csv") == 0)
  7615. return ResultFormatCsv;
  7616. else
  7617. throw MakeStringException(WUERR_InvalidResultFormat, "Unrecognised result format %s", format);
  7618. }
  7619. IDataVal& CLocalWUResult::getResultRaw(IDataVal & data, IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const
  7620. {
  7621. MemoryBuffer s;
  7622. p->getPropBin("Value", s);
  7623. unsigned len = s.length();
  7624. if (len)
  7625. {
  7626. WUResultFormat format = getResultFormat();
  7627. if (format == ResultFormatXml || format == ResultFormatXmlSet)
  7628. {
  7629. if (!xmlTransformer)
  7630. throw MakeStringException(WUERR_MissingFormatTranslator, "No transformer supplied to translate XML format result");
  7631. xmlTransformer->transform(data, len, s.readDirect(len), format == ResultFormatXml);
  7632. }
  7633. else if (format == ResultFormatCsv)
  7634. {
  7635. if (!csvTransformer)
  7636. throw MakeStringException(WUERR_MissingFormatTranslator, "No transformer supplied to translate Csv format result");
  7637. csvTransformer->transform(data, len, s.readDirect(len), true);
  7638. }
  7639. else
  7640. data.setLen(s.readDirect(len), len);
  7641. }
  7642. else
  7643. data.clear();
  7644. return data;
  7645. }
  7646. unsigned CLocalWUResult::getResultHash() const
  7647. {
  7648. MemoryBuffer s;
  7649. p->getPropBin("Value", s);
  7650. unsigned len = s.length();
  7651. const byte * data = (const byte *)s.toByteArray();
  7652. return ~hashc(data, len, ~0);
  7653. }
  7654. IDataVal& CLocalWUResult::getResultUnicode(IDataVal & data) const
  7655. {
  7656. MemoryBuffer s;
  7657. p->getPropBin("Value", s);
  7658. if (s.length())
  7659. {
  7660. unsigned len;
  7661. s.read(len);
  7662. data.setLen(s.readDirect(len*2), len*2);
  7663. }
  7664. else
  7665. {
  7666. StringBuffer utf8;
  7667. if (p->getProp("xmlValue", utf8))
  7668. {
  7669. unsigned outlen;
  7670. UChar *out;
  7671. rtlUtf8ToUnicodeX(outlen, out, utf8.length(), utf8.str());
  7672. data.setLen(out, outlen*2);
  7673. rtlFree(out);
  7674. }
  7675. else
  7676. data.clear();
  7677. }
  7678. return data;
  7679. }
  7680. __int64 CLocalWUResult::getResultRawSize(IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const
  7681. {
  7682. WUResultFormat format = getResultFormat();
  7683. if (format == ResultFormatRaw)
  7684. {
  7685. //MORE: This should not load the whole property...
  7686. MemoryBuffer s;
  7687. p->getPropBin("Value", s);
  7688. return s.length();
  7689. }
  7690. else
  7691. {
  7692. MemoryBuffer temp;
  7693. MemoryBuffer2IDataVal adaptor(temp);
  7694. getResultRaw(adaptor, xmlTransformer, csvTransformer);
  7695. return temp.length();
  7696. }
  7697. }
  7698. IDataVal& CLocalWUResult::getResultRaw(IDataVal & data, __int64 from, __int64 length, IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const
  7699. {
  7700. WUResultFormat format = getResultFormat();
  7701. if (format != ResultFormatRaw)
  7702. {
  7703. MemoryBuffer temp;
  7704. MemoryBuffer2IDataVal adaptor(temp);
  7705. getResultRaw(adaptor, xmlTransformer, csvTransformer);
  7706. unsigned len = temp.length();
  7707. if (from > len) from = len;
  7708. if (from + length > len) length = len - from;
  7709. data.setLen(temp.readDirect(len) + from, (size32_t)length);
  7710. return data;
  7711. }
  7712. else
  7713. {
  7714. //MORE: This should not load the whole property, and should be different from the code above...
  7715. MemoryBuffer s;
  7716. p->getPropBin("Value", s);
  7717. unsigned len = s.length();
  7718. if (from > len) from = len;
  7719. if (from + length > len) length = len - from;
  7720. data.setLen(s.readDirect(len) + from, (size32_t)length);
  7721. return data;
  7722. }
  7723. }
  7724. bool CLocalWUResult::getResultIsAll() const
  7725. {
  7726. return p->getPropBool("@isAll", false);
  7727. }
  7728. // MORE - it's an undetected error if we call setResult... of a type that does not match schema
  7729. void CLocalWUResult::setResultInt(__int64 val)
  7730. {
  7731. // Note: we always serialize scalar integer results as int8, and schema must reflect this
  7732. MemoryBuffer m;
  7733. serializeInt8(val, m);
  7734. p->setPropBin("Value", m.length(), m.toByteArray());
  7735. setResultRowCount(1);
  7736. setResultTotalRowCount(1);
  7737. }
  7738. void CLocalWUResult::setResultUInt(unsigned __int64 val)
  7739. {
  7740. setResultInt((__int64) val);
  7741. }
  7742. void CLocalWUResult::setResultReal(double val)
  7743. {
  7744. // Note: we always serialize scalar real results as real8, and schema must reflect this
  7745. MemoryBuffer m;
  7746. serializeReal8(val, m);
  7747. p->setPropBin("Value", m.length(), m.toByteArray());
  7748. setResultRowCount(1);
  7749. setResultTotalRowCount(1);
  7750. }
  7751. void CLocalWUResult::setResultBool(bool val)
  7752. {
  7753. MemoryBuffer m;
  7754. serializeBool(val, m);
  7755. p->setPropBin("Value", m.length(), m.toByteArray());
  7756. setResultRowCount(1);
  7757. setResultTotalRowCount(1);
  7758. }
  7759. void CLocalWUResult::setResultString(const char *val, unsigned len)
  7760. {
  7761. // Note: we always serialize scalar strings with length prefix, and schema must reflect this
  7762. MemoryBuffer m;
  7763. serializeLPString(len, val, m);
  7764. p->setPropBin("Value", m.length(), m.toByteArray());
  7765. setResultRowCount(1);
  7766. setResultTotalRowCount(1);
  7767. }
  7768. void CLocalWUResult::setResultUnicode(const void *val, unsigned len)
  7769. {
  7770. // Note: we always serialize scalar strings with length prefix, and schema must reflect this
  7771. MemoryBuffer m;
  7772. m.append(len).append(len*2, val);
  7773. p->setPropBin("Value", m.length(), m.toByteArray());
  7774. setResultRowCount(1);
  7775. setResultTotalRowCount(1);
  7776. }
  7777. void CLocalWUResult::setResultData(const void *val, unsigned len)
  7778. {
  7779. // Note: we always serialize scalar data with length prefix, and schema must reflect this
  7780. MemoryBuffer m;
  7781. serializeLPString(len, (const char *)val, m);
  7782. p->setPropBin("Value", m.length(), m.toByteArray());
  7783. setResultRowCount(1);
  7784. setResultTotalRowCount(1);
  7785. }
  7786. void CLocalWUResult::setResultDecimal(const void *val, unsigned len)
  7787. {
  7788. // Note: serialized as data but with length known from schema
  7789. MemoryBuffer m;
  7790. serializeFixedData(len, val, m);
  7791. p->setPropBin("Value", m.length(), m.toByteArray());
  7792. setResultRowCount(1);
  7793. setResultTotalRowCount(1);
  7794. }
  7795. void CLocalWUResult::setResultRow(unsigned len, const void * data)
  7796. {
  7797. p->setPropBin("Value", len, data);
  7798. setResultRowCount(1);
  7799. setResultTotalRowCount(1);
  7800. setResultFormat(ResultFormatRaw);
  7801. }
  7802. void CLocalWUResult::setResultIsAll(bool value)
  7803. {
  7804. p->setPropBool("@isAll", value);
  7805. }
  7806. //==========================================================================================
  7807. CLocalWUPlugin::CLocalWUPlugin(IPropertyTree *props) : p(props)
  7808. {
  7809. }
  7810. IStringVal& CLocalWUPlugin::getPluginName(IStringVal &str) const
  7811. {
  7812. str.set(p->queryProp("@dllname"));
  7813. return str;
  7814. }
  7815. IStringVal& CLocalWUPlugin::getPluginVersion(IStringVal &str) const
  7816. {
  7817. str.set(p->queryProp("@version"));
  7818. return str;
  7819. }
  7820. void CLocalWUPlugin::setPluginName(const char *str)
  7821. {
  7822. p->setProp("@dllname", str);
  7823. }
  7824. void CLocalWUPlugin::setPluginVersion(const char *str)
  7825. {
  7826. p->setProp("@version", str);
  7827. }
  7828. //==========================================================================================
  7829. CLocalWULibrary::CLocalWULibrary(IPropertyTree *props) : p(props)
  7830. {
  7831. }
  7832. IStringVal& CLocalWULibrary::getName(IStringVal &str) const
  7833. {
  7834. str.set(p->queryProp("@name"));
  7835. return str;
  7836. }
  7837. void CLocalWULibrary::setName(const char *str)
  7838. {
  7839. p->setProp("@name", str);
  7840. }
  7841. //==========================================================================================
  7842. CLocalWUException::CLocalWUException(IPropertyTree *props) : p(props)
  7843. {
  7844. }
  7845. IStringVal& CLocalWUException::getExceptionSource(IStringVal &str) const
  7846. {
  7847. str.set(p->queryProp("@source"));
  7848. return str;
  7849. }
  7850. IStringVal& CLocalWUException::getExceptionMessage(IStringVal &str) const
  7851. {
  7852. str.set(p->queryProp(NULL));
  7853. return str;
  7854. }
  7855. unsigned CLocalWUException::getExceptionCode() const
  7856. {
  7857. return p->getPropInt("@code", 0);
  7858. }
  7859. WUExceptionSeverity CLocalWUException::getSeverity() const
  7860. {
  7861. return (WUExceptionSeverity)p->getPropInt("@severity", ExceptionSeverityError);
  7862. }
  7863. IStringVal & CLocalWUException::getTimeStamp(IStringVal & dt) const
  7864. {
  7865. dt.set(p->queryProp("@time"));
  7866. return dt;
  7867. }
  7868. IStringVal & CLocalWUException::getExceptionFileName(IStringVal & str) const
  7869. {
  7870. str.set(p->queryProp("@filename"));
  7871. return str;
  7872. }
  7873. unsigned CLocalWUException::getExceptionLineNo() const
  7874. {
  7875. return p->getPropInt("@row", 0);
  7876. }
  7877. unsigned CLocalWUException::getExceptionColumn() const
  7878. {
  7879. return p->getPropInt("@col", 0);
  7880. }
  7881. void CLocalWUException::setExceptionSource(const char *str)
  7882. {
  7883. p->setProp("@source", str);
  7884. }
  7885. void CLocalWUException::setExceptionMessage(const char *str)
  7886. {
  7887. p->setProp(NULL, str);
  7888. }
  7889. void CLocalWUException::setExceptionCode(unsigned code)
  7890. {
  7891. p->setPropInt("@code", code);
  7892. }
  7893. void CLocalWUException::setSeverity(WUExceptionSeverity level)
  7894. {
  7895. p->setPropInt("@severity", level);
  7896. }
  7897. void CLocalWUException::setTimeStamp(const char *str)
  7898. {
  7899. p->setProp("@time", str);
  7900. }
  7901. void CLocalWUException::setExceptionFileName(const char *str)
  7902. {
  7903. p->setProp("@filename", str);
  7904. }
  7905. void CLocalWUException::setExceptionLineNo(unsigned r)
  7906. {
  7907. p->setPropInt("@row", r);
  7908. }
  7909. void CLocalWUException::setExceptionColumn(unsigned c)
  7910. {
  7911. p->setPropInt("@col", c);
  7912. }
  7913. //==========================================================================================
  7914. CLocalWUAppValue::CLocalWUAppValue(IPropertyTree *props,unsigned child): p(props)
  7915. {
  7916. StringAttrBuilder propPath(prop);
  7917. propPath.append("*[").append(child).append("]");
  7918. }
  7919. IStringVal & CLocalWUAppValue::getApplication(IStringVal & str) const
  7920. {
  7921. str.set(p->queryName());
  7922. return str;
  7923. }
  7924. IStringVal & CLocalWUAppValue::getName(IStringVal & str) const
  7925. {
  7926. IPropertyTree* val=p->queryPropTree(prop.str());
  7927. if(val)
  7928. str.set(val->queryName());
  7929. return str;
  7930. }
  7931. IStringVal & CLocalWUAppValue::getValue(IStringVal & str) const
  7932. {
  7933. str.set(p->queryProp(prop.str()));
  7934. return str;
  7935. }
  7936. //==========================================================================================
  7937. CLocalWUStatistic::CLocalWUStatistic(IPropertyTree *props) : p(props)
  7938. {
  7939. }
  7940. IStringVal & CLocalWUStatistic::getCreator(IStringVal & str) const
  7941. {
  7942. const char * creator = p->queryProp("@creator");
  7943. str.set(creator);
  7944. return str;
  7945. }
  7946. IStringVal & CLocalWUStatistic::getDescription(IStringVal & str, bool createDefault) const
  7947. {
  7948. const char * desc = p->queryProp("@desc");
  7949. if (desc)
  7950. {
  7951. str.set(desc); // legacy and in case it is overridden
  7952. }
  7953. else if (createDefault)
  7954. {
  7955. StatisticKind kind = getKind();
  7956. assertex(kind != StKindNone);
  7957. const char * scope = p->queryProp("@scope");
  7958. assertex(scope);
  7959. //Clean up the format of the scope when converting it to a description
  7960. StringBuffer descriptionText;
  7961. if (streq(scope, GLOBAL_SCOPE))
  7962. {
  7963. const char * creator = p->queryProp("@creator");
  7964. descriptionText.append(creator).append(":");
  7965. queryLongStatisticName(descriptionText, kind);
  7966. }
  7967. else
  7968. {
  7969. loop
  7970. {
  7971. char c = *scope++;
  7972. if (!c)
  7973. break;
  7974. if (c == ':')
  7975. descriptionText.append(": ");
  7976. else
  7977. descriptionText.append(c);
  7978. }
  7979. if (kind != StTimeElapsed)
  7980. queryLongStatisticName(descriptionText.append(": "), kind);
  7981. }
  7982. str.set(descriptionText);
  7983. }
  7984. else
  7985. str.clear();
  7986. return str;
  7987. }
  7988. IStringVal & CLocalWUStatistic::getType(IStringVal & str) const
  7989. {
  7990. StatisticKind kind = getKind();
  7991. if (kind != StKindNone)
  7992. str.set(queryStatisticName(kind));
  7993. return str;
  7994. }
  7995. IStringVal & CLocalWUStatistic::getFormattedValue(IStringVal & str) const
  7996. {
  7997. StringBuffer formatted;
  7998. formatStatistic(formatted, getValue(), getMeasure());
  7999. str.set(formatted);
  8000. return str;
  8001. }
  8002. StatisticCreatorType CLocalWUStatistic::getCreatorType() const
  8003. {
  8004. return queryCreatorType(p->queryProp("@c"));
  8005. }
  8006. StatisticScopeType CLocalWUStatistic::getScopeType() const
  8007. {
  8008. return queryScopeType(p->queryProp("@s"));
  8009. }
  8010. StatisticKind CLocalWUStatistic::getKind() const
  8011. {
  8012. return queryStatisticKind(p->queryProp("@kind"));
  8013. }
  8014. IStringVal & CLocalWUStatistic::getScope(IStringVal & str) const
  8015. {
  8016. const char * scope = p->queryProp("@scope");
  8017. str.set(scope);
  8018. return str;
  8019. }
  8020. StatisticMeasure CLocalWUStatistic::getMeasure() const
  8021. {
  8022. return queryMeasure(p->queryProp("@unit"));
  8023. }
  8024. unsigned __int64 CLocalWUStatistic::getValue() const
  8025. {
  8026. return p->getPropInt64("@value", 0);
  8027. }
  8028. unsigned __int64 CLocalWUStatistic::getCount() const
  8029. {
  8030. return p->getPropInt64("@count", 0);
  8031. }
  8032. unsigned __int64 CLocalWUStatistic::getMax() const
  8033. {
  8034. return p->getPropInt64("@max", 0);
  8035. }
  8036. unsigned __int64 CLocalWUStatistic::getTimestamp() const
  8037. {
  8038. return p->getPropInt64("@ts", 0);
  8039. }
  8040. bool CLocalWUStatistic::matches(const IStatisticsFilter * filter) const
  8041. {
  8042. if (!filter)
  8043. return true;
  8044. const char * creator = p->queryProp("@creator");
  8045. const char * scope = p->queryProp("@scope");
  8046. return filter->matches(getCreatorType(), creator, getScopeType(), scope, getMeasure(), getKind());
  8047. }
  8048. //==========================================================================================
  8049. CLocalWULegacyTiming::CLocalWULegacyTiming(IPropertyTree *props) : p(props)
  8050. {
  8051. }
  8052. IStringVal & CLocalWULegacyTiming::getCreator(IStringVal & str) const
  8053. {
  8054. str.clear();
  8055. return str;
  8056. }
  8057. IStringVal & CLocalWULegacyTiming::getDescription(IStringVal & str, bool createDefault) const
  8058. {
  8059. str.set(p->queryProp("@name"));
  8060. return str;
  8061. }
  8062. IStringVal & CLocalWULegacyTiming::getType(IStringVal & str) const
  8063. {
  8064. str.set(queryStatisticName(StTimeElapsed));
  8065. return str;
  8066. }
  8067. IStringVal & CLocalWULegacyTiming::getFormattedValue(IStringVal & str) const
  8068. {
  8069. StringBuffer formatted;
  8070. formatStatistic(formatted, getValue(), getMeasure());
  8071. str.set(formatted);
  8072. return str;
  8073. }
  8074. StatisticCreatorType CLocalWULegacyTiming::getCreatorType() const
  8075. {
  8076. return SCTunknown;
  8077. }
  8078. StatisticScopeType CLocalWULegacyTiming::getScopeType() const
  8079. {
  8080. return SSTnone;
  8081. }
  8082. StatisticKind CLocalWULegacyTiming::getKind() const
  8083. {
  8084. return StTimeElapsed;
  8085. }
  8086. IStringVal & CLocalWULegacyTiming::getScope(IStringVal & str) const
  8087. {
  8088. str.clear();
  8089. return str;
  8090. }
  8091. StatisticMeasure CLocalWULegacyTiming::getMeasure() const
  8092. {
  8093. return SMeasureTimeNs;
  8094. }
  8095. unsigned __int64 CLocalWULegacyTiming::getValue() const
  8096. {
  8097. return p->getPropInt64("@duration", 0) * 1000000;
  8098. }
  8099. unsigned __int64 CLocalWULegacyTiming::getCount() const
  8100. {
  8101. return p->getPropInt64("@count", 0);
  8102. }
  8103. unsigned __int64 CLocalWULegacyTiming::getMax() const
  8104. {
  8105. return p->getPropInt64("@max", 0);
  8106. }
  8107. unsigned __int64 CLocalWULegacyTiming::getTimestamp() const
  8108. {
  8109. return 0;
  8110. }
  8111. bool CLocalWULegacyTiming::matches(const IStatisticsFilter * filter) const
  8112. {
  8113. if (!filter)
  8114. return true;
  8115. const char * creator = p->queryProp("@creator");
  8116. const char * scope = p->queryProp("@scope");
  8117. return filter->matches(SCTall, NULL, SSTall, NULL, getMeasure(), getKind());
  8118. }
  8119. //==========================================================================================
  8120. extern WORKUNIT_API ILocalWorkUnit * createLocalWorkUnit(const char *xml)
  8121. {
  8122. Owned<CLocalWorkUnit> cw = new CLocalWorkUnit("W_LOCAL", xml, (ISecManager*)NULL, NULL);
  8123. ILocalWorkUnit* ret = QUERYINTERFACE(&cw->lockRemote(false), ILocalWorkUnit);
  8124. return ret;
  8125. }
  8126. extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool unpack, bool includeProgress)
  8127. {
  8128. const CLocalWorkUnit *w = QUERYINTERFACE(wu, const CLocalWorkUnit);
  8129. if (!w)
  8130. {
  8131. const CLockedWorkUnit *wl = QUERYINTERFACE(wu, const CLockedWorkUnit);
  8132. if (wl)
  8133. w = wl->c;
  8134. }
  8135. if (w)
  8136. {
  8137. Linked<IPropertyTree> p;
  8138. if (unpack||includeProgress)
  8139. p.setown(w->getUnpackedTree(includeProgress));
  8140. else
  8141. p.set(w->p);
  8142. toXML(p, str, 0, XML_Format|XML_SortTags);
  8143. }
  8144. else
  8145. str.append("Unrecognized workunit format");
  8146. return str;
  8147. }
  8148. extern WORKUNIT_API IStringVal& exportWorkUnitToXML(const IConstWorkUnit *wu, IStringVal &str, bool unpack, bool includeProgress)
  8149. {
  8150. StringBuffer x;
  8151. str.set(exportWorkUnitToXML(wu,x,unpack, includeProgress).str());
  8152. return str;
  8153. }
  8154. extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool unpack, bool includeProgress)
  8155. {
  8156. const CLocalWorkUnit *w = QUERYINTERFACE(wu, const CLocalWorkUnit);
  8157. if (!w)
  8158. {
  8159. const CLockedWorkUnit *wl = QUERYINTERFACE(wu, const CLockedWorkUnit);
  8160. if (wl)
  8161. w = wl->c;
  8162. }
  8163. if (w)
  8164. {
  8165. Linked<IPropertyTree> p;
  8166. if (unpack||includeProgress)
  8167. p.setown(w->getUnpackedTree(includeProgress));
  8168. else
  8169. p.set(w->p);
  8170. saveXML(filename, p, 0, XML_Format|XML_SortTags|extraXmlFlags);
  8171. }
  8172. }
  8173. extern WORKUNIT_API void submitWorkUnit(const char *wuid, const char *username, const char *password)
  8174. {
  8175. MemoryBuffer buffer;
  8176. Owned<INamedQueueConnection> conn = createNamedQueueConnection(0); // MORE - security token?
  8177. Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
  8178. Owned<IWorkUnit> workunit = factory->updateWorkUnit(wuid);
  8179. assertex(workunit);
  8180. SCMStringBuffer token;
  8181. createToken(wuid, username, password, token);
  8182. workunit->setSecurityToken(token.str());
  8183. StringAttr clusterName(workunit->queryClusterName());
  8184. if (!clusterName.length())
  8185. throw MakeStringException(WUERR_InvalidCluster, "No target cluster specified");
  8186. workunit->commit();
  8187. workunit.clear();
  8188. Owned<IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(clusterName.str());
  8189. if (!clusterInfo)
  8190. throw MakeStringException(WUERR_InvalidCluster, "Unknown cluster %s", clusterName.str());
  8191. SCMStringBuffer serverQueue;
  8192. clusterInfo->getServerQueue(serverQueue);
  8193. assertex(serverQueue.length());
  8194. Owned<IJobQueue> queue = createJobQueue(serverQueue.str());
  8195. if (!queue.get())
  8196. throw MakeStringException(WUERR_InvalidQueue, "Could not create workunit queue");
  8197. IJobQueueItem *item = createJobQueueItem(wuid);
  8198. queue->enqueue(item);
  8199. }
  8200. extern WORKUNIT_API void abortWorkUnit(const char *wuid)
  8201. {
  8202. StringBuffer xpath("/WorkUnitAborts/");
  8203. xpath.append(wuid);
  8204. Owned<IRemoteConnection> acon = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE, SDS_LOCK_TIMEOUT);
  8205. acon->queryRoot()->setPropInt(NULL, 1);
  8206. }
  8207. extern WORKUNIT_API void secSubmitWorkUnit(const char *wuid, ISecManager &secmgr, ISecUser &secuser)
  8208. {
  8209. if (checkWuSecAccess(wuid, secmgr, &secuser, SecAccess_Write, "Submit", true, true))
  8210. submitWorkUnit(wuid, secuser.getName(), secuser.credentials().getPassword());
  8211. }
  8212. extern WORKUNIT_API void secAbortWorkUnit(const char *wuid, ISecManager &secmgr, ISecUser &secuser)
  8213. {
  8214. if (checkWuSecAccess(wuid, secmgr, &secuser, SecAccess_Write, "Submit", true, true))
  8215. abortWorkUnit(wuid);
  8216. }
  8217. extern WORKUNIT_API void submitWorkUnit(const char *wuid, ISecManager *secmgr, ISecUser *secuser)
  8218. {
  8219. if (secmgr && secuser)
  8220. return secSubmitWorkUnit(wuid, *secmgr, *secuser);
  8221. if (secuser)
  8222. return submitWorkUnit(wuid, secuser->getName(), secuser->credentials().getPassword());
  8223. submitWorkUnit(wuid, "", "");
  8224. }
  8225. extern WORKUNIT_API void abortWorkUnit(const char *wuid, ISecManager *secmgr, ISecUser *secuser)
  8226. {
  8227. if (secmgr && secuser)
  8228. return secAbortWorkUnit(wuid, *secmgr, *secuser);
  8229. abortWorkUnit(wuid);
  8230. }
  8231. bool CLocalWorkUnit::hasWorkflow() const
  8232. {
  8233. return p->hasProp("Workflow");
  8234. }
  8235. unsigned CLocalWorkUnit::queryEventScheduledCount() const
  8236. {
  8237. CriticalBlock block(crit);
  8238. return p->getPropInt("Workflow/@eventScheduledCount", 0);
  8239. }
  8240. void CLocalWorkUnit::incEventScheduledCount()
  8241. {
  8242. CriticalBlock block(crit);
  8243. p->setPropInt("Workflow/@eventScheduledCount", p->getPropInt("Workflow/@eventScheduledCount", 0)+1);
  8244. }
  8245. IPropertyTree * CLocalWorkUnit::queryWorkflowTree() const
  8246. {
  8247. CriticalBlock block(crit);
  8248. return p->queryPropTree("Workflow");
  8249. }
  8250. IConstWorkflowItemIterator* CLocalWorkUnit::getWorkflowItems() const
  8251. {
  8252. // For this to be legally called, we must have the read-able interface. So we are already locked for (at least) read.
  8253. CriticalBlock block(crit);
  8254. if(!workflowIteratorCached)
  8255. {
  8256. assertex(!workflowIterator);
  8257. Owned<IPropertyTree> s = p->getPropTree("Workflow");
  8258. if(s)
  8259. workflowIterator.setown(createWorkflowItemIterator(s));
  8260. workflowIteratorCached = true;
  8261. }
  8262. return workflowIterator.getLink();
  8263. }
  8264. IWorkflowItemArray * CLocalWorkUnit::getWorkflowClone() const
  8265. {
  8266. unsigned count = 0;
  8267. Owned<IConstWorkflowItemIterator> iter = getWorkflowItems();
  8268. for(iter->first(); iter->isValid(); iter->next())
  8269. count++;
  8270. Owned<IWorkflowItemArray> array = createWorkflowItemArray(count);
  8271. for(iter->first(); iter->isValid(); iter->next())
  8272. array->addClone(iter->query());
  8273. return array.getLink();
  8274. }
  8275. IWorkflowItem * CLocalWorkUnit::addWorkflowItem(unsigned wfid, WFType type, WFMode mode, unsigned success, unsigned failure, unsigned recovery, unsigned retriesAllowed, unsigned contingencyFor)
  8276. {
  8277. // For this to be legally called, we must have the write-able interface. So we are already locked for write.
  8278. CriticalBlock block(crit);
  8279. workflowIterator.clear();
  8280. workflowIteratorCached = false;
  8281. IPropertyTree * s = p->queryPropTree("Workflow");
  8282. if(!s)
  8283. s = p->addPropTree("Workflow", createPTree("Workflow"));
  8284. return createWorkflowItem(s, wfid, type, mode, success, failure, recovery, retriesAllowed, contingencyFor);
  8285. }
  8286. IWorkflowItemIterator * CLocalWorkUnit::updateWorkflowItems()
  8287. {
  8288. // For this to be legally called, we must have the write-able interface. So we are already locked for write.
  8289. CriticalBlock block(crit);
  8290. if(!workflowIterator)
  8291. {
  8292. IPropertyTree * s = p->queryPropTree("Workflow");
  8293. if(!s)
  8294. s = p->addPropTree("Workflow", createPTree("Workflow"));
  8295. workflowIterator.setown(createWorkflowItemIterator(s));
  8296. workflowIteratorCached = true;
  8297. }
  8298. return workflowIterator.getLink();
  8299. }
  8300. void CLocalWorkUnit::syncRuntimeWorkflow(IWorkflowItemArray * array)
  8301. {
  8302. Owned<IWorkflowItemIterator> iter = updateWorkflowItems();
  8303. Owned<IWorkflowItem> item;
  8304. for(iter->first(); iter->isValid(); iter->next())
  8305. {
  8306. item.setown(iter->get());
  8307. item->syncRuntimeData(array->queryWfid(item->queryWfid()));
  8308. }
  8309. workflowIterator.clear();
  8310. workflowIteratorCached = false;
  8311. }
  8312. void CLocalWorkUnit::resetWorkflow()
  8313. {
  8314. if (hasWorkflow())
  8315. {
  8316. Owned<IWorkflowItemIterator> iter = updateWorkflowItems();
  8317. Owned<IWorkflowItem> wf;
  8318. for(iter->first(); iter->isValid(); iter->next())
  8319. {
  8320. wf.setown(iter->get());
  8321. wf->reset();
  8322. }
  8323. workflowIterator.clear();
  8324. workflowIteratorCached = false;
  8325. }
  8326. }
  8327. void CLocalWorkUnit::schedule()
  8328. {
  8329. CriticalBlock block(crit);
  8330. if(queryEventScheduledCount() == 0) return;
  8331. switch(getState())
  8332. {
  8333. case WUStateCompleted:
  8334. setState(WUStateWait);
  8335. break;
  8336. case WUStateFailed:
  8337. case WUStateArchived:
  8338. case WUStateAborting:
  8339. case WUStateAborted:
  8340. case WUStateScheduled:
  8341. throw MakeStringException(WUERR_CannotSchedule, "Cannot schedule workunit in this state");
  8342. }
  8343. StringBuffer rootPath;
  8344. rootPath.append("/Schedule/").append(queryClusterName());
  8345. Owned<IRemoteConnection> conn = querySDS().connect(rootPath.str(), myProcessSession(), RTM_LOCK_WRITE | RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
  8346. Owned<IPropertyTree> root = conn->getRoot();
  8347. if(!root->hasChildren())
  8348. {
  8349. StringBuffer addPath;
  8350. addPath.append("/Schedulers/").append(queryClusterName());
  8351. Owned<IRemoteConnection> addConn = querySDS().connect(addPath.str(), myProcessSession(), RTM_LOCK_WRITE | RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
  8352. }
  8353. char const * wuid = p->queryName();
  8354. StringBuffer xpath("*/*/");
  8355. ncnameEscape(wuid, xpath);
  8356. bool more;
  8357. do more = root->removeProp(xpath.str()); while(more);
  8358. Owned<IConstWorkflowItemIterator> iter = getWorkflowItems();
  8359. Owned<IWorkflowEvent> event;
  8360. Owned<IPropertyTree> branch1, branch2;
  8361. for(iter->first(); iter->isValid(); iter->next())
  8362. {
  8363. event.setown(iter->query()->getScheduleEvent());
  8364. if(!event) continue;
  8365. ncnameEscape(event->queryName(), xpath.clear());
  8366. ensurePTree(root, xpath.str());
  8367. branch1.setown(root->getPropTree(xpath.str()));
  8368. ncnameEscape(event->queryText(), xpath.clear());
  8369. ensurePTree(branch1, xpath.str());
  8370. branch2.setown(branch1->getPropTree(xpath.str()));
  8371. ncnameEscape(wuid, xpath.clear());
  8372. ensurePTree(branch2, xpath.str());
  8373. }
  8374. }
  8375. void CLocalWorkUnit::deschedule()
  8376. {
  8377. if(queryEventScheduledCount() == 0) return;
  8378. if(getState() == WUStateWait)
  8379. setState(WUStateCompleted);
  8380. doDescheduleWorkkunit(p->queryName());
  8381. }
  8382. mapEnums localFileUploadTypes[] = {
  8383. { UploadTypeFileSpray, "FileSpray" },
  8384. { UploadTypeWUResult, "WUResult" },
  8385. { UploadTypeWUResultCsv, "WUResultCsv" },
  8386. { UploadTypeWUResultXml, "WUResultXml" },
  8387. { UploadTypeSize, NULL }
  8388. };
  8389. class CLocalFileUpload : public CInterface, implements IConstLocalFileUpload
  8390. {
  8391. public:
  8392. CLocalFileUpload(IPropertyTree * _tree) : tree(_tree) {}
  8393. CLocalFileUpload(unsigned id, LocalFileUploadType type, char const * source, char const * destination, char const * eventTag)
  8394. {
  8395. tree.setown(createPTree());
  8396. tree->setPropInt("@id", id);
  8397. setEnum(tree, "@type", type, localFileUploadTypes);
  8398. tree->setProp("@source", source);
  8399. tree->setProp("@destination", destination);
  8400. if (eventTag)
  8401. tree->setProp("@eventTag", eventTag);
  8402. }
  8403. IMPLEMENT_IINTERFACE;
  8404. IPropertyTree * getTree() { return tree.getLink(); }
  8405. virtual unsigned queryID() const { return tree->getPropInt("@id"); }
  8406. virtual LocalFileUploadType queryType() const { return (LocalFileUploadType)getEnum(tree, "@type", localFileUploadTypes); }
  8407. virtual IStringVal & getSource(IStringVal & ret) const { ret.set(tree->queryProp("@source")); return ret; }
  8408. virtual IStringVal & getDestination(IStringVal & ret) const { ret.set(tree->queryProp("@destination")); return ret; }
  8409. virtual IStringVal & getEventTag(IStringVal & ret) const { if(tree->hasProp("@eventTag")) ret.set(tree->queryProp("@eventTag")); else ret.clear(); return ret; }
  8410. private:
  8411. Owned<IPropertyTree> tree;
  8412. };
  8413. class CLocalFileUploadIterator : public CInterface, implements IConstLocalFileUploadIterator
  8414. {
  8415. public:
  8416. CLocalFileUploadIterator(IPropertyTree * _tree) : tree(_tree), iter(tree->getElements("LocalFileUpload")) {}
  8417. IMPLEMENT_IINTERFACE;
  8418. bool first() { return iter->first(); }
  8419. bool isValid() { return iter->isValid(); }
  8420. bool next() { return iter->next(); }
  8421. IConstLocalFileUpload * get() { return new CLocalFileUpload(&iter->get()); }
  8422. private:
  8423. Owned<IPropertyTree> tree;
  8424. Owned<IPropertyTreeIterator> iter;
  8425. };
  8426. IConstLocalFileUploadIterator * CLocalWorkUnit::getLocalFileUploads() const
  8427. {
  8428. // For this to be legally called, we must have the read-able interface. So we are already locked for (at least) read.
  8429. CriticalBlock block(crit);
  8430. Owned<IPropertyTree> s = p->getPropTree("LocalFileUploads");
  8431. if(s)
  8432. return new CLocalFileUploadIterator(s.getClear());
  8433. else
  8434. return NULL;
  8435. }
  8436. bool CLocalWorkUnit::requiresLocalFileUpload() const
  8437. {
  8438. SCMStringBuffer dest;
  8439. Owned<IConstWUResult> result;
  8440. Owned<IConstLocalFileUploadIterator> iter(getLocalFileUploads());
  8441. if(!iter)
  8442. return false;
  8443. for(iter->first(); iter->isValid(); iter->next())
  8444. {
  8445. Owned<IConstLocalFileUpload> upload(iter->get());
  8446. switch(upload->queryType())
  8447. {
  8448. case UploadTypeWUResult:
  8449. case UploadTypeWUResultCsv:
  8450. case UploadTypeWUResultXml:
  8451. upload->getDestination(dest);
  8452. result.setown(getResultByName(dest.str()));
  8453. if(!result)
  8454. return true;
  8455. break;
  8456. default:
  8457. throw MakeStringException(WUERR_InvalidUploadFormat, "Unsupported local file upload type %s", getEnumText(upload->queryType(), localFileUploadTypes));
  8458. }
  8459. }
  8460. return false;
  8461. }
  8462. unsigned CLocalWorkUnit::addLocalFileUpload(LocalFileUploadType type, char const * source, char const * destination, char const * eventTag)
  8463. {
  8464. // For this to be legally called, we must have the write-able interface. So we are already locked for write.
  8465. CriticalBlock block(crit);
  8466. IPropertyTree * s = p->queryPropTree("LocalFileUploads");
  8467. if(!s)
  8468. s = p->addPropTree("LocalFileUploads", createPTree());
  8469. unsigned id = s->numChildren();
  8470. Owned<CLocalFileUpload> upload = new CLocalFileUpload(id, type, source, destination, eventTag);
  8471. s->addPropTree("LocalFileUpload", upload->getTree());
  8472. return id;
  8473. }
  8474. #if 0
  8475. void testConstWorkflow(IConstWorkflowItem * cwf, bool * okay, bool * dep)
  8476. {
  8477. DBGLOG("Test workflow const iface %u", cwf->queryWfid());
  8478. unsigned deps = 0;
  8479. Owned<IWorkflowDependencyIterator> diter;
  8480. switch(cwf->queryWfid())
  8481. {
  8482. case 1:
  8483. assertex(!cwf->isScheduled());
  8484. assertex(cwf->queryType() == WFTypeNormal);
  8485. assertex(cwf->queryState() == WFStateNull);
  8486. diter.setown(cwf->getDependencies());
  8487. for(diter->first(); diter->isValid(); diter->next())
  8488. deps++;
  8489. assertex(deps==0);
  8490. okay[0] = true;
  8491. break;
  8492. case 2:
  8493. assertex(!cwf->isScheduled());
  8494. assertex(cwf->queryType() == WFTypeRecovery);
  8495. assertex(cwf->queryState() == WFStateSkip);
  8496. okay[1] = true;
  8497. break;
  8498. case 3:
  8499. assertex(cwf->queryContingencyFor() == 4);
  8500. okay[2] = true;
  8501. break;
  8502. case 4:
  8503. assertex(cwf->isScheduled());
  8504. assertex(cwf->queryType() == WFTypeNormal);
  8505. assertex(cwf->queryState() == WFStateReqd);
  8506. assertex(cwf->querySuccess() == 0);
  8507. assertex(cwf->queryFailure() == 3);
  8508. assertex(cwf->queryRecovery() == 2);
  8509. assertex(cwf->queryRetriesAllowed() == 10);
  8510. assertex(cwf->queryRetriesRemaining() == 10);
  8511. diter.setown(cwf->getDependencies());
  8512. for(diter->first(); diter->isValid(); diter->next())
  8513. {
  8514. dep[diter->query()-1] = true;
  8515. deps++;
  8516. }
  8517. assertex(deps==2);
  8518. assertex(dep[0]);
  8519. assertex(dep[1]);
  8520. okay[3] = true;
  8521. break;
  8522. case 5:
  8523. assertex(cwf->isScheduled());
  8524. assertex(!cwf->isScheduledNow());
  8525. assertex(cwf->querySchedulePriority() == 75);
  8526. assertex(cwf->queryScheduleCount() == 5);
  8527. assertex(cwf->queryScheduleCountRemaining() == 5);
  8528. okay[4] = true;
  8529. break;
  8530. case 6:
  8531. assertex(cwf->isScheduled());
  8532. assertex(!cwf->isScheduledNow());
  8533. assertex(cwf->querySchedulePriority() == 25);
  8534. assertex(!cwf->hasScheduleCount());
  8535. okay[5] = true;
  8536. break;
  8537. default:
  8538. assertex(!"unknown wfid in test");
  8539. }
  8540. }
  8541. void testRuntimeWorkflow(IRuntimeWorkflowItem * rwf, bool * okay)
  8542. {
  8543. DBGLOG("Test workflow runtime iface %u", rwf->queryWfid());
  8544. switch(rwf->queryWfid())
  8545. {
  8546. case 1:
  8547. case 2:
  8548. case 3:
  8549. okay[rwf->queryWfid()-1] = true;
  8550. break;
  8551. case 4:
  8552. {
  8553. unsigned tries = 0;
  8554. while(rwf->testAndDecRetries())
  8555. tries++;
  8556. assertex(tries == 10);
  8557. assertex(rwf->queryRetriesRemaining() == 0);
  8558. rwf->setState(WFStateFail);
  8559. assertex(rwf->queryState() == WFStateFail);
  8560. rwf->reset();
  8561. assertex(rwf->queryRetriesRemaining() == 10);
  8562. assertex(rwf->queryState() == WFStateReqd);
  8563. }
  8564. okay[3] = true;
  8565. break;
  8566. case 5:
  8567. {
  8568. assertex(rwf->queryScheduleCountRemaining() == 5);
  8569. unsigned count = 0;
  8570. do count++; while(rwf->decAndTestScheduleCountRemaining());
  8571. assertex(count == 5);
  8572. assertex(rwf->queryScheduleCountRemaining() == 0);
  8573. rwf->reset();
  8574. assertex(rwf->queryScheduleCountRemaining() == 5);
  8575. }
  8576. okay[4] = true;
  8577. break;
  8578. case 6:
  8579. {
  8580. assertex(!rwf->hasScheduleCount());
  8581. unsigned count;
  8582. for(count=0; count<20; count++)
  8583. assertex(rwf->decAndTestScheduleCountRemaining());
  8584. }
  8585. okay[5] = true;
  8586. break;
  8587. default:
  8588. assertex(!"unknown wfid in test");
  8589. }
  8590. }
  8591. void testWorkflow()
  8592. {
  8593. DBGLOG("workunit.cpp : testWorkflow");
  8594. CLocalWorkUnit wu("W-WF-TEST", 0, 0, 0);
  8595. Owned<IWorkflowItem> wf;
  8596. wf.setown(wu.addWorkflowItem(1, WFTypeNormal, 0, 0, 0, 0, 0));
  8597. wf.setown(wu.addWorkflowItem(2, WFTypeRecovery, 0, 0, 0, 0, 0));
  8598. wf.setown(wu.addWorkflowItem(3, WFTypeFailure, 0, 0, 0, 0, 4));
  8599. wf.setown(wu.addWorkflowItem(4, WFTypeNormal, 0, 3, 2, 10, 0));
  8600. wf->setScheduledNow();
  8601. wf->addDependency(1);
  8602. wf.setown(wu.addWorkflowItem(5, WFTypeNormal, 0, 0, 0, 0, 0));
  8603. wf->setScheduledOn("test", "foo*");
  8604. wf->setSchedulePriority(75);
  8605. wf->setScheduleCount(5);
  8606. wf.setown(wu.addWorkflowItem(6, WFTypeNormal, 0, 0, 0, 0, 0));
  8607. wf->setScheduledOn("test", "bar*");
  8608. wf->setSchedulePriority(25);
  8609. unsigned const n = 6;
  8610. bool okay[n];
  8611. bool dep[n];
  8612. unsigned i;
  8613. for(i=0; i<n; i++)
  8614. okay[i] = dep[i] = 0;
  8615. Owned<IConstWorkflowItemIterator> citer(wu.getWorkflowItems());
  8616. for(citer->first(); citer->isValid(); citer->next())
  8617. testConstWorkflow(citer->query(), okay, dep);
  8618. for(i=0; i<n; i++)
  8619. {
  8620. assertex(okay[i]);
  8621. okay[i] = false;
  8622. }
  8623. Owned<IWorkflowItemIterator> miter(wu.updateWorkflowItems());
  8624. for(miter->first(); miter->isValid(); miter->next())
  8625. {
  8626. Owned<IRuntimeWorkflowItem> rwf(miter->get());
  8627. testRuntimeWorkflow(rwf, okay);
  8628. }
  8629. for(i=0; i<n; i++)
  8630. {
  8631. assertex(okay[i]);
  8632. okay[i] = dep[i] = false;
  8633. }
  8634. Owned<IWorkflowItemArray> array(wu.getWorkflowClone());
  8635. unsigned wfid;
  8636. for(wfid = 1; array->isValid(wfid); wfid++)
  8637. testConstWorkflow(&array->queryWfid(wfid), okay, dep);
  8638. for(i=0; i<n; i++)
  8639. {
  8640. assertex(okay[i]);
  8641. okay[i] = false;
  8642. }
  8643. for(wfid = 1; array->isValid(wfid); wfid++)
  8644. testRuntimeWorkflow(&array->queryWfid(wfid), okay);
  8645. for(i=0; i<n; i++)
  8646. {
  8647. assertex(okay[i]);
  8648. okay[i] = false;
  8649. }
  8650. }
  8651. #endif
  8652. //------------------------------------------------------------------------------------------
  8653. class WorkUnitWaiter : public CInterface, implements ISDSSubscription, implements IAbortHandler
  8654. {
  8655. Semaphore changed;
  8656. SubscriptionId change;
  8657. public:
  8658. IMPLEMENT_IINTERFACE;
  8659. WorkUnitWaiter(const char *xpath)
  8660. {
  8661. change = querySDS().subscribe(xpath, *this, false);
  8662. aborted = false;
  8663. }
  8664. ~WorkUnitWaiter()
  8665. {
  8666. assertex(change==0);
  8667. }
  8668. void notify(SubscriptionId id, const char *xpath, SDSNotifyFlags flags, unsigned valueLen, const void *valueData)
  8669. {
  8670. changed.signal();
  8671. }
  8672. bool wait(unsigned timeout)
  8673. {
  8674. return changed.wait(timeout) && !aborted;
  8675. }
  8676. bool onAbort()
  8677. {
  8678. aborted = true;
  8679. changed.signal();
  8680. return false;
  8681. }
  8682. void unsubscribe()
  8683. {
  8684. querySDS().unsubscribe(change);
  8685. change = 0;
  8686. }
  8687. bool aborted;
  8688. };
  8689. static WUState _waitForWorkUnit(const char * wuid, unsigned timeout, bool compiled, bool returnOnWaitState)
  8690. {
  8691. StringBuffer wuRoot;
  8692. getXPath(wuRoot, wuid);
  8693. Owned<WorkUnitWaiter> waiter = new WorkUnitWaiter(wuRoot.str());
  8694. LocalIAbortHandler abortHandler(*waiter);
  8695. WUState ret = WUStateUnknown;
  8696. Owned<IRemoteConnection> conn = querySDS().connect(wuRoot.str(), myProcessSession(), 0, SDS_LOCK_TIMEOUT);
  8697. if (conn)
  8698. {
  8699. unsigned start = msTick();
  8700. loop
  8701. {
  8702. ret = (WUState) getEnum(conn->queryRoot(), "@state", states);
  8703. switch (ret)
  8704. {
  8705. case WUStateCompiled:
  8706. case WUStateUploadingFiles:
  8707. if (!compiled)
  8708. break;
  8709. // fall into
  8710. case WUStateCompleted:
  8711. case WUStateFailed:
  8712. case WUStateAborted:
  8713. waiter->unsubscribe();
  8714. return ret;
  8715. case WUStateWait:
  8716. if(returnOnWaitState)
  8717. {
  8718. waiter->unsubscribe();
  8719. return ret;
  8720. }
  8721. break;
  8722. case WUStateCompiling:
  8723. case WUStateRunning:
  8724. case WUStateDebugPaused:
  8725. case WUStateDebugRunning:
  8726. case WUStateBlocked:
  8727. case WUStateAborting:
  8728. if (queryDaliServerVersion().compare("2.1")>=0)
  8729. {
  8730. SessionId agent = conn->queryRoot()->getPropInt64("@agentSession", -1);
  8731. if((agent>0) && querySessionManager().sessionStopped(agent, 0))
  8732. {
  8733. waiter->unsubscribe();
  8734. conn->reload();
  8735. ret = (WUState) getEnum(conn->queryRoot(), "@state", states);
  8736. bool isEcl = false;
  8737. switch (ret)
  8738. {
  8739. case WUStateCompiling:
  8740. isEcl = true;
  8741. // drop into
  8742. case WUStateRunning:
  8743. case WUStateBlocked:
  8744. ret = WUStateFailed;
  8745. break;
  8746. case WUStateAborting:
  8747. ret = WUStateAborted;
  8748. break;
  8749. default:
  8750. return ret;
  8751. }
  8752. WARNLOG("_waitForWorkUnit terminated: %"I64F"d state = %d",(__int64)agent,(int)ret);
  8753. Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
  8754. Owned<IWorkUnit> wu = factory->updateWorkUnit(wuid);
  8755. wu->setState(ret);
  8756. Owned<IWUException> e = wu->createException();
  8757. e->setExceptionCode(isEcl ? 1001 : 1000);
  8758. e->setExceptionMessage(isEcl ? "EclServer terminated unexpectedly" : "Workunit terminated unexpectedly");
  8759. return ret;
  8760. }
  8761. }
  8762. break;
  8763. }
  8764. unsigned waited = msTick() - start;
  8765. if (timeout==-1)
  8766. {
  8767. waiter->wait(20000); // recheck state every 20 seconds even if no timeout, in case eclagent has crashed.
  8768. if (waiter->aborted)
  8769. {
  8770. ret = WUStateUnknown; // MORE - throw an exception?
  8771. break;
  8772. }
  8773. }
  8774. else if (waited > timeout || !waiter->wait(timeout-waited))
  8775. {
  8776. ret = WUStateUnknown; // MORE - throw an exception?
  8777. break;
  8778. }
  8779. conn->reload();
  8780. }
  8781. }
  8782. waiter->unsubscribe();
  8783. return ret;
  8784. }
  8785. extern WUState waitForWorkUnitToComplete(const char * wuid, int timeout, bool returnOnWaitState)
  8786. {
  8787. return _waitForWorkUnit(wuid, (unsigned)timeout, false, returnOnWaitState);
  8788. }
  8789. extern WORKUNIT_API WUState secWaitForWorkUnitToComplete(const char * wuid, ISecManager &secmgr, ISecUser &secuser, int timeout, bool returnOnWaitState)
  8790. {
  8791. if (checkWuSecAccess(wuid, secmgr, &secuser, SecAccess_Read, "Wait for Complete", false, true))
  8792. return waitForWorkUnitToComplete(wuid, timeout, returnOnWaitState);
  8793. return WUStateUnknown;
  8794. }
  8795. extern bool waitForWorkUnitToCompile(const char * wuid, int timeout)
  8796. {
  8797. switch(_waitForWorkUnit(wuid, (unsigned)timeout, true, true))
  8798. {
  8799. case WUStateCompiled:
  8800. case WUStateCompleted:
  8801. case WUStateWait:
  8802. case WUStateUploadingFiles:
  8803. return true;
  8804. default:
  8805. return false;
  8806. }
  8807. }
  8808. extern WORKUNIT_API bool secWaitForWorkUnitToCompile(const char * wuid, ISecManager &secmgr, ISecUser &secuser, int timeout)
  8809. {
  8810. if (checkWuSecAccess(wuid, secmgr, &secuser, SecAccess_Read, "Wait for Compile", false, true))
  8811. return waitForWorkUnitToCompile(wuid, timeout);
  8812. return false;
  8813. }
  8814. extern WORKUNIT_API bool secDebugWorkunit(const char * wuid, ISecManager &secmgr, ISecUser &secuser, const char *command, StringBuffer &response)
  8815. {
  8816. if (strnicmp(command, "<debug:", 7) == 0 && checkWuSecAccess(wuid, secmgr, &secuser, SecAccess_Read, "Debug", false, true))
  8817. {
  8818. Owned<IConstWorkUnit> wu = factory->secOpenWorkUnit(wuid, false, &secmgr, &secuser);
  8819. SCMStringBuffer ip;
  8820. unsigned port;
  8821. port = wu->getDebugAgentListenerPort();
  8822. wu->getDebugAgentListenerIP(ip);
  8823. SocketEndpoint debugEP(ip.str(), port);
  8824. Owned<ISocket> socket = ISocket::connect_timeout(debugEP, 1000);
  8825. unsigned len = (size32_t)strlen(command);
  8826. unsigned revlen = len;
  8827. _WINREV(revlen);
  8828. socket->write(&revlen, sizeof(revlen));
  8829. socket->write(command, len);
  8830. for (;;)
  8831. {
  8832. socket->read(&len, sizeof(len));
  8833. _WINREV(len);
  8834. if (len == 0)
  8835. break;
  8836. if (len & 0x80000000)
  8837. {
  8838. throwUnexpected();
  8839. }
  8840. char * mem = (char*) response.reserve(len);
  8841. socket->read(mem, len);
  8842. }
  8843. return true;
  8844. }
  8845. return false;
  8846. }
  8847. void updateSuppliedXmlParams(IWorkUnit * w)
  8848. {
  8849. Owned<const IPropertyTree> params = w->getXmlParams();
  8850. if (!params)
  8851. return;
  8852. Owned<IPropertyTreeIterator> elems = params->getElements("*");
  8853. ForEach(*elems)
  8854. {
  8855. IPropertyTree & curVal = elems->query();
  8856. const char *name = curVal.queryName();
  8857. Owned<IWUResult> r = updateWorkUnitResult(w, name, -1);
  8858. if (r)
  8859. {
  8860. StringBuffer s;
  8861. if (r->isResultScalar() && !curVal.hasChildren())
  8862. {
  8863. curVal.getProp(".", s);
  8864. r->setResultXML(s);
  8865. r->setResultStatus(ResultStatusSupplied);
  8866. }
  8867. else
  8868. {
  8869. toXML(&curVal, s);
  8870. bool isSet = (curVal.hasProp("Item") || curVal.hasProp("string"));
  8871. r->setResultRaw(s.length(), s.str(), isSet ? ResultFormatXmlSet : ResultFormatXml);
  8872. }
  8873. }
  8874. else
  8875. DBGLOG("WARNING: no matching variable in workunit for input parameter %s", name);
  8876. }
  8877. }
  8878. IWUResult * updateWorkUnitResult(IWorkUnit * w, const char *name, unsigned sequence)
  8879. {
  8880. switch ((int)sequence)
  8881. {
  8882. case ResultSequenceStored:
  8883. return w->updateVariableByName(name);
  8884. case ResultSequencePersist:
  8885. return w->updateGlobalByName(name);
  8886. case ResultSequenceInternal:
  8887. case ResultSequenceOnce:
  8888. return w->updateTemporaryByName(name);
  8889. default:
  8890. return w->updateResultBySequence(sequence);
  8891. }
  8892. }
  8893. IConstWUResult * getWorkUnitResult(IConstWorkUnit * w, const char *name, unsigned sequence)
  8894. {
  8895. switch ((int)sequence)
  8896. {
  8897. case ResultSequenceStored:
  8898. return w->getVariableByName(name);
  8899. case ResultSequencePersist:
  8900. return w->getGlobalByName(name);
  8901. case ResultSequenceInternal:
  8902. case ResultSequenceOnce:
  8903. return w->getTemporaryByName(name);
  8904. default:
  8905. if (name && name[0])
  8906. return w->getResultByName(name);//name takes precedence over sequence
  8907. else
  8908. return w->getResultBySequence(sequence);
  8909. }
  8910. }
  8911. extern WORKUNIT_API bool getWorkUnitCreateTime(const char *wuid,CDateTime &time)
  8912. {
  8913. if (wuid) {
  8914. char prefchar;
  8915. unsigned year,month,day,hour,min,sec;
  8916. if (sscanf(wuid, "%c%4u%2u%2u-%2u%2u%2u", &prefchar, &year, &month, &day, &hour, &min, &sec)==7) {
  8917. time.set(year, month, day, hour, min, sec, 0, true);
  8918. // time.setDate(year, month, day);
  8919. // time.setTime(hour, min, sec, 0, true); // for some reason time is local
  8920. return true;
  8921. }
  8922. }
  8923. return false;
  8924. }
  8925. extern WORKUNIT_API IStringVal& createToken(const char *wuid, const char *user, const char *password, IStringVal &str)
  8926. {
  8927. StringBuffer wu, token("X");
  8928. wu.append(wuid).append(';').append(user).append(';').append(password);
  8929. encrypt(token,wu.str());
  8930. str.set(token.str());
  8931. return str;
  8932. }
  8933. // This will be replaced by something more secure!
  8934. extern WORKUNIT_API void extractToken(const char *token, const char *wuid, IStringVal &user, IStringVal &password)
  8935. {
  8936. if (token && *token)
  8937. {
  8938. StringBuffer wu;
  8939. decrypt(wu, token+1);
  8940. const char *finger = strchr(wu.str(),';');
  8941. if (finger && strnicmp(wuid, wu.str(), finger-wu.str())==0)
  8942. {
  8943. const char *finger1 = strchr(++finger,';');
  8944. if(finger1)
  8945. {
  8946. user.setLen(finger, (size32_t)(finger1-finger));
  8947. finger1++;
  8948. password.setLen(finger1, (size32_t)(wu.str() + wu.length() - finger1));
  8949. return;
  8950. }
  8951. }
  8952. throw MakeStringException(WUERR_InvalidSecurityToken, "Invalid call to extractToken");
  8953. }
  8954. }
  8955. extern WORKUNIT_API WUState getWorkUnitState(const char* state)
  8956. {
  8957. return (WUState) getEnum(state, states);
  8958. }
  8959. const LogMsgCategory MCschedconn = MCprogress(1000); // Category used to inform about schedule synchronization
  8960. class CWorkflowScheduleConnection : public CInterface, implements IWorkflowScheduleConnection
  8961. {
  8962. public:
  8963. CWorkflowScheduleConnection(char const * wuid)
  8964. {
  8965. basexpath.append("/WorkflowSchedule/").append(wuid);
  8966. flagxpath.append(basexpath.str()).append("/Active");
  8967. }
  8968. IMPLEMENT_IINTERFACE;
  8969. virtual void lock()
  8970. {
  8971. LOG(MCschedconn, "Locking base schedule connection");
  8972. baseconn.setown(querySDS().connect(basexpath.str(), myProcessSession(), RTM_CREATE_QUERY | RTM_LOCK_WRITE, INFINITE));
  8973. if(!baseconn)
  8974. throw MakeStringException(WUERR_ScheduleLockFailed, "Could not get base workflow schedule lock");
  8975. }
  8976. virtual void unlock()
  8977. {
  8978. LOG(MCschedconn, "Unlocking base schedule connection");
  8979. baseconn.clear();
  8980. }
  8981. virtual void setActive()
  8982. {
  8983. LOG(MCschedconn, "Setting active flag in schedule connection");
  8984. flagconn.setown(querySDS().connect(flagxpath.str(), myProcessSession(), RTM_CREATE | RTM_LOCK_WRITE | RTM_DELETE_ON_DISCONNECT, INFINITE));
  8985. if(!flagconn)
  8986. throw MakeStringException(WUERR_ScheduleLockFailed, "Could not get active workflow schedule lock");
  8987. }
  8988. virtual void resetActive()
  8989. {
  8990. LOG(MCschedconn, "Resetting active flag in schedule connection");
  8991. flagconn.clear();
  8992. }
  8993. virtual bool queryActive()
  8994. {
  8995. return baseconn->queryRoot()->hasProp("Active");
  8996. }
  8997. virtual bool pull(IWorkflowItemArray * workflow)
  8998. {
  8999. assertex(baseconn);
  9000. Owned<IPropertyTree> root = baseconn->getRoot();
  9001. Owned<IPropertyTree> eventQueue = root->getPropTree("EventQueue");
  9002. if(!eventQueue) return false;
  9003. if(!eventQueue->hasProp("Item")) return false;
  9004. {
  9005. Owned<IPropertyTreeIterator> eventItems = eventQueue->getElements("Item");
  9006. Owned<IPropertyTree> eventItem;
  9007. Owned<IRuntimeWorkflowItemIterator> wfItems = workflow->getSequenceIterator();
  9008. Owned<IRuntimeWorkflowItem> wfItem;
  9009. for(eventItems->first(); eventItems->isValid(); eventItems->next())
  9010. {
  9011. eventItem.setown(&eventItems->get());
  9012. const char * eventName = eventItem->queryProp("@name");
  9013. const char * eventText = eventItem->queryProp("@text");
  9014. for(wfItems->first(); wfItems->isValid(); wfItems->next())
  9015. {
  9016. wfItem.setown(wfItems->get());
  9017. if(wfItem->queryState() != WFStateWait)
  9018. continue;
  9019. Owned<IWorkflowEvent> targetEvent = wfItem->getScheduleEvent();
  9020. if(!targetEvent || !targetEvent->matches(eventName, eventText))
  9021. continue;
  9022. wfItem->setEvent(eventName, eventText);
  9023. wfItem->setState(WFStateReqd);
  9024. resetDependentsState(workflow, *wfItem);
  9025. }
  9026. }
  9027. }
  9028. bool more;
  9029. do
  9030. more = eventQueue->removeProp("Item");
  9031. while(more);
  9032. return true;
  9033. }
  9034. virtual void push(char const * name, char const * text)
  9035. {
  9036. assertex(baseconn);
  9037. Owned<IPropertyTree> root = baseconn->getRoot();
  9038. ensurePTree(root, "EventQueue");
  9039. Owned<IPropertyTree> eventQueue = root->getPropTree("EventQueue");
  9040. Owned<IPropertyTree> eventItem = createPTree();
  9041. eventItem->setProp("@name", name);
  9042. eventItem->setProp("@text", text);
  9043. eventQueue->addPropTree("Item", eventItem.getLink());
  9044. }
  9045. virtual void remove()
  9046. {
  9047. if (baseconn)
  9048. {
  9049. baseconn->close(true);
  9050. baseconn.clear();
  9051. }
  9052. }
  9053. private:
  9054. void resetItemStateAndDependents(IWorkflowItemArray * workflow, unsigned wfid) const
  9055. {
  9056. if (wfid)
  9057. resetItemStateAndDependents(workflow, workflow->queryWfid(wfid));
  9058. }
  9059. void resetItemStateAndDependents(IWorkflowItemArray * workflow, IRuntimeWorkflowItem & item) const
  9060. {
  9061. switch(item.queryState())
  9062. {
  9063. case WFStateDone:
  9064. case WFStateFail:
  9065. {
  9066. item.setState(WFStateNull);
  9067. resetItemStateAndDependents(workflow, item.queryPersistWfid());
  9068. resetDependentsState(workflow, item);
  9069. break;
  9070. }
  9071. }
  9072. }
  9073. void resetDependentsState(IWorkflowItemArray * workflow, IRuntimeWorkflowItem & item) const
  9074. {
  9075. Owned<IWorkflowDependencyIterator> iter(item.getDependencies());
  9076. for(iter->first(); iter->isValid(); iter->next())
  9077. {
  9078. IRuntimeWorkflowItem & dep = workflow->queryWfid(iter->query());
  9079. resetItemStateAndDependents(workflow, dep);
  9080. }
  9081. }
  9082. private:
  9083. StringBuffer basexpath;
  9084. StringBuffer flagxpath;
  9085. Owned<IRemoteConnection> baseconn;
  9086. Owned<IRemoteConnection> flagconn;
  9087. };
  9088. extern WORKUNIT_API IWorkflowScheduleConnection * getWorkflowScheduleConnection(char const * wuid)
  9089. {
  9090. return new CWorkflowScheduleConnection(wuid);
  9091. }
  9092. extern WORKUNIT_API IExtendedWUInterface * queryExtendedWU(IWorkUnit * wu)
  9093. {
  9094. return QUERYINTERFACE(wu, IExtendedWUInterface);
  9095. }
  9096. extern WORKUNIT_API void addExceptionToWorkunit(IWorkUnit * wu, WUExceptionSeverity severity, const char * source, unsigned code, const char * text, const char * filename, unsigned lineno, unsigned column)
  9097. {
  9098. Owned<IWUException> we = wu->createException();
  9099. we->setSeverity(severity);
  9100. we->setExceptionMessage(text);
  9101. if (source)
  9102. we->setExceptionSource(source);
  9103. if (code)
  9104. we->setExceptionCode(code);
  9105. if (filename)
  9106. we->setExceptionFileName(filename);
  9107. if (lineno)
  9108. {
  9109. we->setExceptionLineNo(lineno);
  9110. if (column)
  9111. we->setExceptionColumn(lineno);
  9112. }
  9113. }
  9114. const char * skipLeadingXml(const char * text)
  9115. {
  9116. if (!text)
  9117. return NULL;
  9118. //skip utf8 BOM, probably excessive
  9119. if (memcmp(text, UTF8_BOM, 3) == 0)
  9120. text += 3;
  9121. loop
  9122. {
  9123. if (isspace(*text))
  9124. text++;
  9125. else if (text[0] == '<' && text[1] == '?')
  9126. {
  9127. text += 2;
  9128. loop
  9129. {
  9130. if (!*text) break;
  9131. if (text[0] == '?' && text[1] == '>')
  9132. {
  9133. text += 2;
  9134. break;
  9135. }
  9136. text++;
  9137. }
  9138. }
  9139. else if (text[0] == '<' && text[1] == '!' && text[2] == '-' && text[3] == '-')
  9140. {
  9141. text += 4;
  9142. loop
  9143. {
  9144. if (!*text) break;
  9145. if (text[0] == '-' && text[1] == '-' && text[2] == '>')
  9146. {
  9147. text += 3;
  9148. break;
  9149. }
  9150. text++;
  9151. }
  9152. }
  9153. else
  9154. break;
  9155. }
  9156. return text;
  9157. }
  9158. extern WORKUNIT_API bool isArchiveQuery(const char * text)
  9159. {
  9160. text = skipLeadingXml(text);
  9161. if (!text)
  9162. return false;
  9163. const char * archivePrefix = "<Archive";
  9164. return memicmp(text, archivePrefix, strlen(archivePrefix)) == 0;
  9165. }
  9166. extern WORKUNIT_API bool isQueryManifest(const char * text)
  9167. {
  9168. text = skipLeadingXml(text);
  9169. if (!text)
  9170. return false;
  9171. const char * manifestPrefix = "<Manifest";
  9172. return memicmp(text, manifestPrefix, strlen(manifestPrefix)) == 0;
  9173. }
  9174. //------------------------------------------------------------------------------
  9175. // Named Alias helper function
  9176. static IPropertyTree * resolveQueryByDll(IPropertyTree * queryRegistry, const char * dll)
  9177. {
  9178. StringBuffer xpath;
  9179. xpath.append("Query[@dll=\"").append(dll).append("\"]");
  9180. return queryRegistry->getPropTree(xpath);
  9181. }
  9182. static IPropertyTree * resolveQueryByWuid(IPropertyTree * queryRegistry, const char * wuid)
  9183. {
  9184. StringBuffer xpath;
  9185. xpath.append("Query[@wuid=\"").append(wuid).append("\"]");
  9186. return queryRegistry->getPropTree(xpath);
  9187. }
  9188. static void clearAliases(IPropertyTree * queryRegistry, const char * id)
  9189. {
  9190. StringBuffer lcId(id);
  9191. lcId.toLowerCase();
  9192. StringBuffer xpath;
  9193. xpath.append("Alias[@id=\"").append(lcId).append("\"]");
  9194. Owned<IPropertyTreeIterator> iter = queryRegistry->getElements(xpath);
  9195. ForEach(*iter)
  9196. {
  9197. queryRegistry->removeProp(xpath.str());
  9198. }
  9199. }
  9200. IPropertyTree * addNamedQuery(IPropertyTree * queryRegistry, const char * name, const char * wuid, const char * dll, bool library, const char *userid, const char *snapshot)
  9201. {
  9202. StringBuffer lcName(name);
  9203. lcName.toLowerCase();
  9204. StringBuffer xpath;
  9205. xpath.append("Query[@name=\"").append(lcName.str()).append("\"]");
  9206. Owned<IPropertyTreeIterator> iter = queryRegistry->getElements(xpath);
  9207. unsigned seq = 1;
  9208. ForEach(*iter)
  9209. {
  9210. IPropertyTree &item = iter->query();
  9211. const char *thisWuid = item.queryProp("@wuid");
  9212. if (strieq(wuid, thisWuid))
  9213. return &item;
  9214. unsigned thisSeq = item.getPropInt("@seq");
  9215. if (thisSeq >= seq)
  9216. seq = thisSeq + 1;
  9217. }
  9218. StringBuffer id;
  9219. id.append(lcName).append(".").append(seq);
  9220. IPropertyTree * newEntry = createPTree("Query", ipt_caseInsensitive);
  9221. newEntry->setProp("@name", lcName);
  9222. newEntry->setProp("@wuid", wuid);
  9223. newEntry->setProp("@dll", dll);
  9224. newEntry->setProp("@id", id);
  9225. newEntry->setPropInt("@seq", seq);
  9226. if (library)
  9227. newEntry->setPropBool("@isLibrary", true);
  9228. if (userid && *userid)
  9229. newEntry->setProp("@publishedBy", userid);
  9230. if (snapshot && *snapshot)
  9231. newEntry->setProp("@snapshot", snapshot);
  9232. return queryRegistry->addPropTree("Query", newEntry);
  9233. }
  9234. void removeNamedQuery(IPropertyTree * queryRegistry, const char * id)
  9235. {
  9236. StringBuffer lcId(id);
  9237. lcId.toLowerCase();
  9238. clearAliases(queryRegistry, lcId);
  9239. StringBuffer xpath;
  9240. xpath.append("Query[@id=\"").append(lcId).append("\"]");
  9241. queryRegistry->removeProp(xpath);
  9242. }
  9243. void removeDllFromNamedQueries(IPropertyTree * queryRegistry, const char * dll)
  9244. {
  9245. Owned<IPropertyTree> match = resolveQueryByDll(queryRegistry, dll);
  9246. if (!match)
  9247. return;
  9248. clearAliases(queryRegistry, match->queryProp("@id"));
  9249. queryRegistry->removeTree(match);
  9250. }
  9251. void removeWuidFromNamedQueries(IPropertyTree * queryRegistry, const char * wuid)
  9252. {
  9253. Owned<IPropertyTree> match = resolveQueryByWuid(queryRegistry, wuid);
  9254. if (!match)
  9255. return;
  9256. clearAliases(queryRegistry, match->queryProp("@id"));
  9257. queryRegistry->removeTree(match);
  9258. }
  9259. void removeAliasesFromNamedQuery(IPropertyTree * queryRegistry, const char * id)
  9260. {
  9261. clearAliases(queryRegistry, id);
  9262. }
  9263. void setQueryAlias(IPropertyTree * queryRegistry, const char * name, const char * value)
  9264. {
  9265. StringBuffer lcName(name);
  9266. lcName.toLowerCase();
  9267. StringBuffer xpath;
  9268. xpath.append("Alias[@name=\"").append(lcName).append("\"]");
  9269. IPropertyTree * match = queryRegistry->queryPropTree(xpath);
  9270. if (!match)
  9271. {
  9272. IPropertyTree * newEntry = createPTree("Alias");
  9273. newEntry->setProp("@name", lcName);
  9274. match = queryRegistry->addPropTree("Alias", newEntry);
  9275. }
  9276. match->setProp("@id", value);
  9277. }
  9278. extern WORKUNIT_API IPropertyTree * getQueryById(IPropertyTree * queryRegistry, const char *queryid)
  9279. {
  9280. if (!queryRegistry || !queryid)
  9281. return NULL;
  9282. StringBuffer xpath;
  9283. xpath.append("Query[@id=\"").append(queryid).append("\"]");
  9284. return queryRegistry->getPropTree(xpath);
  9285. }
  9286. extern WORKUNIT_API IPropertyTree * getQueryById(const char *queryset, const char *queryid, bool readonly)
  9287. {
  9288. Owned<IPropertyTree> queryRegistry = getQueryRegistry(queryset, readonly);
  9289. return getQueryById(queryRegistry, queryid);
  9290. }
  9291. extern WORKUNIT_API IPropertyTree * resolveQueryAlias(IPropertyTree * queryRegistry, const char * alias)
  9292. {
  9293. if (!queryRegistry || !alias)
  9294. return NULL;
  9295. StringBuffer xpath;
  9296. unsigned cnt = 0;
  9297. StringBuffer lc(alias);
  9298. const char * search = lc.toLowerCase().str();
  9299. loop
  9300. {
  9301. xpath.set("Alias[@name='").append(search).append("']/@id");
  9302. const char * queryId = queryRegistry->queryProp(xpath);
  9303. if (!queryId)
  9304. break;
  9305. //Check for too many alias indirections.
  9306. if (cnt++ > 10)
  9307. return NULL;
  9308. search = lc.set(queryId).toLowerCase().str();
  9309. }
  9310. return getQueryById(queryRegistry, search);
  9311. }
  9312. extern WORKUNIT_API IPropertyTree * resolveQueryAlias(const char *queryset, const char *alias, bool readonly)
  9313. {
  9314. Owned<IPropertyTree> queryRegistry = getQueryRegistry(queryset, readonly);
  9315. return resolveQueryAlias(queryRegistry, alias);
  9316. }
  9317. void setQuerySuspendedState(IPropertyTree * queryRegistry, const char *id, bool suspend, const char *userid)
  9318. {
  9319. StringBuffer lcId(id);
  9320. lcId.toLowerCase();
  9321. StringBuffer xpath;
  9322. xpath.append("Query[@id=\"").append(lcId).append("\"]");
  9323. IPropertyTree *tree = queryRegistry->queryPropTree(xpath);
  9324. if (tree)
  9325. {
  9326. if (tree->getPropBool("@suspended", false) == suspend)
  9327. return;
  9328. if (suspend)
  9329. {
  9330. tree->addPropBool("@suspended", true);
  9331. if (userid && *userid)
  9332. tree->addProp("@suspendedBy", userid);
  9333. }
  9334. else
  9335. {
  9336. tree->removeProp("@suspended");
  9337. tree->removeProp("@suspendedBy");
  9338. }
  9339. }
  9340. else
  9341. throw MakeStringException((suspend)? QUERRREG_SUSPEND : QUERRREG_UNSUSPEND, "Modifying query suspended state failed. Could not find query %s", id);
  9342. }
  9343. void setQueryCommentForNamedQuery(IPropertyTree * queryRegistry, const char *id, const char *queryComment)
  9344. {
  9345. if (queryComment)
  9346. {
  9347. StringBuffer lcId(id);
  9348. lcId.toLowerCase();
  9349. StringBuffer xpath;
  9350. xpath.append("Query[@id=\"").append(lcId).append("\"]");
  9351. IPropertyTree *tree = queryRegistry->queryPropTree(xpath);
  9352. if (tree)
  9353. tree->setProp("@queryComment", queryComment);
  9354. else
  9355. throw MakeStringException(QUERRREG_COMMENT, "Could not find query %s", id);
  9356. }
  9357. }
  9358. extern WORKUNIT_API IPropertyTree * getQueryRegistryRoot()
  9359. {
  9360. Owned<IRemoteConnection> conn = querySDS().connect("/QuerySets", myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT);
  9361. if (conn)
  9362. return conn->getRoot();
  9363. else
  9364. return NULL;
  9365. }
  9366. extern WORKUNIT_API void checkAddLibrariesToQueryEntry(IPropertyTree *queryTree, IConstWULibraryIterator *libraries)
  9367. {
  9368. if (!queryTree || !libraries)
  9369. return;
  9370. if (queryTree->hasProp("@libCount")) //already added
  9371. return;
  9372. unsigned libCount=0;
  9373. ForEach(*libraries)
  9374. {
  9375. IConstWULibrary &library = libraries->query();
  9376. SCMStringBuffer libname;
  9377. if (!library.getName(libname).length())
  9378. continue;
  9379. queryTree->addProp("Library", libname.str());
  9380. libCount++;
  9381. }
  9382. queryTree->setPropInt("@libCount", libCount);
  9383. }
  9384. extern WORKUNIT_API void checkAddLibrariesToQueryEntry(IPropertyTree *queryTree, IConstWorkUnit *cw)
  9385. {
  9386. Owned<IConstWULibraryIterator> libraries = &cw->getLibraries();
  9387. checkAddLibrariesToQueryEntry(queryTree, libraries);
  9388. }
  9389. extern WORKUNIT_API IPropertyTree * getQueryRegistry(const char * wsEclId, bool readonly)
  9390. {
  9391. //Only lock the branch for the target we're interested in.
  9392. StringBuffer xpath;
  9393. xpath.append("/QuerySets/QuerySet[@id=\"").append(wsEclId).append("\"]");
  9394. Owned<IRemoteConnection> conn = querySDS().connect(xpath.str(), myProcessSession(), readonly ? RTM_LOCK_READ : RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
  9395. if (conn)
  9396. return conn->getRoot();
  9397. if (readonly)
  9398. return NULL;
  9399. //Lock the QuerySets in case another thread/client wants to check/add the same QuerySet.
  9400. Owned<IRemoteConnection> globalLock = querySDS().connect("/QuerySets/", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
  9401. //Re-check if the QuerySet has been added between checking the 1st time and gaining the globalLock.
  9402. conn.setown(querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT));
  9403. if (conn)
  9404. return conn->getRoot();
  9405. conn.setown(querySDS().connect("/QuerySets/QuerySet", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_ADD, SDS_LOCK_TIMEOUT));
  9406. if (!conn)
  9407. throwUnexpected();
  9408. IPropertyTree * root = conn->queryRoot();
  9409. root->setProp("@id",wsEclId);
  9410. conn->commit();
  9411. return LINK(root);
  9412. }
  9413. IPropertyTree * addNamedPackageSet(IPropertyTree * packageRegistry, const char * name, IPropertyTree *packageInfo, bool overWrite)
  9414. {
  9415. StringBuffer xpath;
  9416. StringBuffer lcName(name);
  9417. lcName.toLowerCase();
  9418. // see if "name" already exists
  9419. xpath.append("Package[@id='").append(name).append("']");
  9420. IPropertyTree *pkgTree = packageRegistry->queryPropTree(xpath.str());
  9421. if (pkgTree)
  9422. {
  9423. if (overWrite)
  9424. packageRegistry->removeTree(pkgTree);
  9425. else
  9426. throw MakeStringException(WUERR_PackageAlreadyExists, "Package name %s already exists, either delete it or specify overwrite",name);
  9427. }
  9428. IPropertyTree *tree = packageRegistry->addPropTree("Package", packageInfo);
  9429. tree->setProp("@id", lcName);
  9430. return tree;
  9431. }
  9432. void removeNamedPackage(IPropertyTree * packageRegistry, const char * id)
  9433. {
  9434. StringBuffer lcId(id);
  9435. lcId.toLowerCase();
  9436. StringBuffer xpath;
  9437. xpath.append("Package[@id=\"").append(lcId).append("\"]");
  9438. packageRegistry->removeProp(xpath);
  9439. }
  9440. extern WORKUNIT_API IPropertyTree * getPackageSetRegistry(const char * wsEclId, bool readonly)
  9441. {
  9442. //Only lock the branch for the target we're interested in.
  9443. StringBuffer xpath;
  9444. xpath.append("/PackageSets/PackageSet[@id=\"").append(wsEclId).append("\"]");
  9445. Owned<IRemoteConnection> conn = querySDS().connect(xpath.str(), myProcessSession(), readonly ? RTM_LOCK_READ : RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
  9446. if (conn)
  9447. return conn->getRoot();
  9448. if (readonly)
  9449. return NULL;
  9450. //Lock the PackageSets in case another thread/client wants to check/add the same PackageSet.
  9451. Owned<IRemoteConnection> globalLock = querySDS().connect("/PackageSets/", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
  9452. //Re-check if the PackageSet has been added between checking the 1st time and gaining the globalLock.
  9453. conn.setown(querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT));
  9454. if (conn)
  9455. return conn->getRoot();
  9456. conn.setown(querySDS().connect("/PackageSets/PackageSet", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_ADD, SDS_LOCK_TIMEOUT));
  9457. if (!conn)
  9458. throwUnexpected();
  9459. IPropertyTree* root = conn->queryRoot();
  9460. root->setProp("@id",wsEclId);
  9461. conn->commit();
  9462. return LINK(root);
  9463. }
  9464. void addQueryToQuerySet(IWorkUnit *workunit, IPropertyTree *queryRegistry, const char *queryName, WUQueryActivationOptions activateOption, StringBuffer &newQueryId, const char *userid)
  9465. {
  9466. StringBuffer cleanQueryName;
  9467. appendUtf8XmlName(cleanQueryName, strlen(queryName), queryName);
  9468. SCMStringBuffer dllName;
  9469. Owned<IConstWUQuery> q = workunit->getQuery();
  9470. q->getQueryDllName(dllName);
  9471. if (!dllName.length())
  9472. throw MakeStringException(WUERR_InvalidDll, "Cannot deploy query - no associated dll.");
  9473. StringBuffer currentTargetClusterType;
  9474. queryRegistry->getProp("@targetclustertype", currentTargetClusterType);
  9475. SCMStringBuffer targetClusterType;
  9476. workunit->getDebugValue("targetclustertype", targetClusterType);
  9477. SCMStringBuffer snapshot;
  9478. workunit->getSnapshot(snapshot);
  9479. if (currentTargetClusterType.length() < 1)
  9480. {
  9481. queryRegistry->setProp("@targetclustertype", targetClusterType.str());
  9482. }
  9483. else
  9484. {
  9485. if (strcmp(currentTargetClusterType.str(), "roxie") == 0 && strcmp(currentTargetClusterType.str(), targetClusterType.str())!=0)
  9486. {
  9487. throw MakeStringException(WUERR_MismatchClusterType, "TargetClusterTypes of workunit and queryset do not match.");
  9488. }
  9489. }
  9490. IPropertyTree *newEntry = addNamedQuery(queryRegistry, cleanQueryName, workunit->queryWuid(), dllName.str(), isLibrary(workunit), userid, snapshot.str());
  9491. Owned<IConstWULibraryIterator> libraries = &workunit->getLibraries();
  9492. checkAddLibrariesToQueryEntry(newEntry, libraries);
  9493. newQueryId.append(newEntry->queryProp("@id"));
  9494. workunit->setIsQueryService(true); //will check querysets before delete
  9495. workunit->commit();
  9496. activateQuery(queryRegistry, activateOption, queryName, newQueryId, userid);
  9497. }
  9498. void activateQuery(IPropertyTree *queryRegistry, WUQueryActivationOptions activateOption, const char *queryName, const char *queryId, const char *userid)
  9499. {
  9500. StringBuffer cleanQueryName;
  9501. appendUtf8XmlName(cleanQueryName, strlen(queryName), queryName);
  9502. if (activateOption == ACTIVATE_SUSPEND_PREVIOUS|| activateOption == ACTIVATE_DELETE_PREVIOUS)
  9503. {
  9504. Owned<IPropertyTree> prevQuery = resolveQueryAlias(queryRegistry, cleanQueryName);
  9505. setQueryAlias(queryRegistry, cleanQueryName, queryId);
  9506. if (prevQuery && !streq(queryId, prevQuery->queryProp("@id")))
  9507. {
  9508. if (activateOption == ACTIVATE_SUSPEND_PREVIOUS)
  9509. setQuerySuspendedState(queryRegistry, prevQuery->queryProp("@id"), true, userid);
  9510. else
  9511. removeNamedQuery(queryRegistry, prevQuery->queryProp("@id"));
  9512. }
  9513. }
  9514. else if (activateOption == MAKE_ACTIVATE || activateOption == MAKE_ACTIVATE_LOAD_DATA_ONLY)
  9515. setQueryAlias(queryRegistry, cleanQueryName, queryId);
  9516. }
  9517. void addQueryToQuerySet(IWorkUnit *workunit, const char *querySetName, const char *queryName, WUQueryActivationOptions activateOption, StringBuffer &newQueryId, const char *userid)
  9518. {
  9519. Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, false);
  9520. addQueryToQuerySet(workunit, queryRegistry, queryName, activateOption, newQueryId, userid);
  9521. }
  9522. bool removeQuerySetAlias(const char *querySetName, const char *alias)
  9523. {
  9524. Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, true);
  9525. StringBuffer xpath;
  9526. xpath.appendf("Alias[@name='%s']", alias);
  9527. IPropertyTree *t = queryRegistry->queryPropTree(xpath);
  9528. return queryRegistry->removeTree(t);
  9529. }
  9530. void addQuerySetAlias(const char *querySetName, const char *alias, const char *id)
  9531. {
  9532. Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, false);
  9533. setQueryAlias(queryRegistry, alias, id);
  9534. }
  9535. void setSuspendQuerySetQuery(const char *querySetName, const char *id, bool suspend, const char *userid)
  9536. {
  9537. Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, true);
  9538. setQuerySuspendedState(queryRegistry, id, suspend, userid);
  9539. }
  9540. void deleteQuerySetQuery(const char *querySetName, const char *id)
  9541. {
  9542. Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, true);
  9543. removeNamedQuery(queryRegistry, id);
  9544. }
  9545. void removeQuerySetAliasesFromNamedQuery(const char *querySetName, const char * id)
  9546. {
  9547. Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, true);
  9548. clearAliases(queryRegistry, id);
  9549. }
  9550. void setQueryCommentForNamedQuery(const char *querySetName, const char *id, const char *queryComment)
  9551. {
  9552. Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, true);
  9553. setQueryCommentForNamedQuery(queryRegistry, id, queryComment);
  9554. }
  9555. const char *queryIdFromQuerySetWuid(IPropertyTree *queryRegistry, const char *wuid, const char *queryName, IStringVal &id)
  9556. {
  9557. if (!queryRegistry)
  9558. return NULL;
  9559. StringBuffer xpath;
  9560. xpath.appendf("Query[@wuid='%s']", wuid);
  9561. if (queryName && *queryName)
  9562. xpath.appendf("[@name='%s']", queryName);
  9563. IPropertyTree *q = queryRegistry->queryPropTree(xpath.str());
  9564. if (q)
  9565. {
  9566. id.set(q->queryProp("@id"));
  9567. }
  9568. return id.str();
  9569. }
  9570. const char *queryIdFromQuerySetWuid(const char *querySetName, const char *wuid, const char *queryName, IStringVal &id)
  9571. {
  9572. Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, true);
  9573. return queryIdFromQuerySetWuid(queryRegistry, wuid, queryName, id);
  9574. }
  9575. extern WORKUNIT_API void gatherLibraryNames(StringArray &names, StringArray &unresolved, IWorkUnitFactory &workunitFactory, IConstWorkUnit &cw, IPropertyTree *queryset)
  9576. {
  9577. Owned<IConstWULibraryIterator> wulibraries = &cw.getLibraries();
  9578. ForEach(*wulibraries)
  9579. {
  9580. SCMStringBuffer libname;
  9581. IConstWULibrary &wulibrary = wulibraries->query();
  9582. wulibrary.getName(libname);
  9583. if (names.contains(libname.str()) || unresolved.contains(libname.str()))
  9584. continue;
  9585. Owned<IPropertyTree> query = resolveQueryAlias(queryset, libname.str());
  9586. if (query && query->getPropBool("@isLibrary"))
  9587. {
  9588. const char *wuid = query->queryProp("@wuid");
  9589. Owned<IConstWorkUnit> libcw = workunitFactory.openWorkUnit(wuid, false);
  9590. if (libcw)
  9591. {
  9592. names.appendUniq(libname.str());
  9593. gatherLibraryNames(names, unresolved, workunitFactory, *libcw, queryset);
  9594. continue;
  9595. }
  9596. }
  9597. unresolved.appendUniq(libname.str());
  9598. }
  9599. }
  9600. bool looksLikeAWuid(const char * wuid)
  9601. {
  9602. if (!wuid)
  9603. return false;
  9604. if (wuid[0] != 'W')
  9605. return false;
  9606. if (!isdigit(wuid[1]) || !isdigit(wuid[2]) || !isdigit(wuid[3]) || !isdigit(wuid[4]))
  9607. return false;
  9608. if (!isdigit(wuid[5]) || !isdigit(wuid[6]) || !isdigit(wuid[7]) || !isdigit(wuid[8]))
  9609. return false;
  9610. return (wuid[9]=='-');
  9611. }
  9612. IPropertyTree * resolveDefinitionInArchive(IPropertyTree * archive, const char * path)
  9613. {
  9614. IPropertyTree * module = archive;
  9615. const char * dot = strrchr(path, '.');
  9616. StringBuffer xpath;
  9617. if (dot)
  9618. {
  9619. xpath.clear().append("Module[@key='").appendLower(dot-path, path).append("']");
  9620. module = archive->queryPropTree(xpath);
  9621. path = dot+1;
  9622. }
  9623. else
  9624. module = archive->queryPropTree("Module[@key='']");
  9625. if (!module)
  9626. return NULL;
  9627. xpath.clear().append("Attribute[@key='").appendLower(strlen(path), path).append("']");
  9628. return module->queryPropTree(xpath);
  9629. }
  9630. extern WORKUNIT_API void associateLocalFile(IWUQuery * query, WUFileType type, const char * name, const char * description, unsigned crc)
  9631. {
  9632. StringBuffer hostname;
  9633. queryHostIP().getIpText(hostname);
  9634. StringBuffer fullPathname;
  9635. makeAbsolutePath(name, fullPathname);
  9636. query->addAssociatedFile(type, fullPathname, hostname, description, crc);
  9637. }
  9638. extern WORKUNIT_API void descheduleWorkunit(char const * wuid)
  9639. {
  9640. Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
  9641. Owned<IWorkUnit> workunit = factory->updateWorkUnit(wuid);
  9642. if(workunit)
  9643. workunit->deschedule();
  9644. else
  9645. doDescheduleWorkkunit(wuid);
  9646. }
  9647. extern WORKUNIT_API void updateWorkunitTimeStat(IWorkUnit * wu, StatisticScopeType scopeType, const char * scope, StatisticKind kind, const char * description, unsigned __int64 value)
  9648. {
  9649. wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), scopeType, scope, kind, description, value, 1, 0, StatsMergeReplace);
  9650. }
  9651. extern WORKUNIT_API void updateWorkunitTimings(IWorkUnit * wu, ITimeReporter *timer)
  9652. {
  9653. StringBuffer scope;
  9654. for (unsigned i = 0; i < timer->numSections(); i++)
  9655. {
  9656. StatisticScopeType scopeType= timer->getScopeType(i);
  9657. timer->getScope(i, scope.clear());
  9658. StatisticKind kind = timer->getTimerType(i);
  9659. wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), scopeType, scope, kind, NULL, timer->getTime(i), timer->getCount(i), timer->getMaxTime(i), StatsMergeReplace);
  9660. }
  9661. }
  9662. extern WORKUNIT_API void getWorkunitTotalTime(IConstWorkUnit* workunit, const char* creator, unsigned __int64 & totalTimeNs, unsigned __int64 & totalThisTimeNs)
  9663. {
  9664. StatisticsFilter summaryTimeFilter(SCTsummary, creator, SSTglobal, GLOBAL_SCOPE, SMeasureTimeNs, StTimeElapsed);
  9665. Owned<IConstWUStatistic> totalThorTime = getStatistic(workunit, summaryTimeFilter);
  9666. Owned<IConstWUStatistic> totalThisThorTime = workunit->getStatistic(queryStatisticsComponentName(), GLOBAL_SCOPE, StTimeElapsed);
  9667. if (totalThorTime)
  9668. totalTimeNs = totalThorTime->getValue();
  9669. else
  9670. totalTimeNs = 0;
  9671. if (totalThisThorTime)
  9672. totalThisTimeNs = totalThisThorTime->getValue();
  9673. else
  9674. totalThisTimeNs = 0;
  9675. }
  9676. extern WORKUNIT_API void addTimeStamp(IWorkUnit * wu, StatisticScopeType scopeType, const char * scope, StatisticKind kind)
  9677. {
  9678. wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), scopeType, scope, kind, NULL, getTimeStampNowValue(), 1, 0, StatsMergeAppend);
  9679. }
  9680. IConstWUStatistic * getStatistic(IConstWorkUnit * wu, const IStatisticsFilter & filter)
  9681. {
  9682. Owned<IConstWUStatisticIterator> iter = &wu->getStatistics(&filter);
  9683. if (iter->first())
  9684. return &OLINK(iter->query());
  9685. return NULL;
  9686. }
  9687. class GlobalStatisticGatherer : public CInterfaceOf<IStatisticGatherer>
  9688. {
  9689. public:
  9690. GlobalStatisticGatherer(IWorkUnit * _wu) : wu(_wu) {}
  9691. virtual void beginScope(const StatsScopeId & id)
  9692. {
  9693. prevLenStack.append(scope.length());
  9694. if (scope.length())
  9695. scope.append(":");
  9696. id.getScopeText(scope);
  9697. scopeTypeStack.append(id.queryScopeType());
  9698. }
  9699. virtual void beginSubGraphScope(unsigned id)
  9700. {
  9701. StatsScopeId scopeId(SSTsubgraph, id);
  9702. beginScope(scopeId);
  9703. }
  9704. virtual void beginActivityScope(unsigned id)
  9705. {
  9706. StatsScopeId scopeId(SSTactivity, id);
  9707. beginScope(scopeId);
  9708. }
  9709. virtual void beginEdgeScope(unsigned id, unsigned oid)
  9710. {
  9711. StatsScopeId scopeId(SSTedge, id, oid);
  9712. beginScope(scopeId);
  9713. }
  9714. virtual void endScope()
  9715. {
  9716. scope.setLength(prevLenStack.popGet());
  9717. scopeTypeStack.pop();
  9718. }
  9719. virtual void addStatistic(StatisticKind kind, unsigned __int64 value)
  9720. {
  9721. StatisticScopeType scopeType = scopeTypeStack.ordinality() ? (StatisticScopeType)scopeTypeStack.tos() : SSTglobal;
  9722. wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), scopeType, scope, kind, NULL, value, 1, 0, StatsMergeAppend);
  9723. }
  9724. virtual void updateStatistic(StatisticKind kind, unsigned __int64 value, StatsMergeAction mergeAction)
  9725. {
  9726. StatisticScopeType scopeType = scopeTypeStack.ordinality() ? (StatisticScopeType)scopeTypeStack.tos() : SSTglobal;
  9727. wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), scopeType, scope, kind, NULL, value, 1, 0, mergeAction);
  9728. }
  9729. virtual IStatisticCollection * getResult()
  9730. {
  9731. return NULL;
  9732. }
  9733. protected:
  9734. Linked<IWorkUnit> wu;
  9735. StringBuffer scope;
  9736. UnsignedArray prevLenStack;
  9737. UnsignedArray scopeTypeStack;
  9738. };
  9739. IStatisticGatherer * createGlobalStatisticGatherer(IWorkUnit * wu)
  9740. {
  9741. return new GlobalStatisticGatherer(wu);
  9742. }