123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305730673077308730973107311731273137314731573167317731873197320732173227323732473257326732773287329733073317332733373347335733673377338733973407341734273437344734573467347734873497350735173527353735473557356735773587359736073617362736373647365736673677368736973707371737273737374737573767377737873797380738173827383738473857386738773887389739073917392739373947395739673977398739974007401740274037404740574067407740874097410741174127413741474157416741774187419742074217422742374247425742674277428742974307431743274337434743574367437743874397440744174427443744474457446744774487449745074517452745374547455745674577458745974607461746274637464746574667467746874697470747174727473747474757476747774787479748074817482748374847485748674877488748974907491749274937494749574967497749874997500750175027503750475057506750775087509751075117512751375147515751675177518751975207521752275237524752575267527752875297530753175327533753475357536753775387539754075417542754375447545754675477548754975507551755275537554755575567557755875597560756175627563756475657566756775687569757075717572757375747575757675777578757975807581758275837584758575867587758875897590759175927593759475957596759775987599760076017602760376047605760676077608760976107611761276137614761576167617761876197620762176227623762476257626762776287629763076317632763376347635763676377638763976407641764276437644764576467647764876497650765176527653765476557656765776587659766076617662766376647665766676677668766976707671767276737674767576767677767876797680768176827683768476857686768776887689769076917692769376947695769676977698769977007701770277037704770577067707770877097710771177127713771477157716771777187719772077217722772377247725772677277728772977307731773277337734773577367737773877397740774177427743774477457746774777487749775077517752775377547755775677577758775977607761776277637764776577667767776877697770777177727773777477757776777777787779778077817782778377847785778677877788778977907791779277937794779577967797779877997800780178027803780478057806780778087809781078117812781378147815781678177818781978207821782278237824782578267827782878297830783178327833783478357836783778387839784078417842784378447845784678477848784978507851785278537854785578567857785878597860786178627863786478657866786778687869787078717872787378747875787678777878787978807881788278837884788578867887788878897890789178927893789478957896789778987899790079017902790379047905790679077908790979107911791279137914791579167917791879197920792179227923792479257926792779287929793079317932793379347935793679377938793979407941794279437944794579467947794879497950795179527953795479557956795779587959796079617962796379647965796679677968796979707971797279737974797579767977797879797980798179827983798479857986798779887989799079917992799379947995799679977998799980008001800280038004800580068007800880098010801180128013801480158016801780188019802080218022802380248025802680278028802980308031803280338034803580368037803880398040804180428043804480458046804780488049805080518052805380548055805680578058805980608061806280638064806580668067806880698070807180728073807480758076807780788079808080818082808380848085808680878088808980908091809280938094809580968097809880998100810181028103810481058106810781088109811081118112811381148115811681178118811981208121812281238124812581268127812881298130813181328133813481358136813781388139814081418142814381448145814681478148814981508151815281538154815581568157815881598160816181628163816481658166816781688169817081718172817381748175817681778178817981808181818281838184818581868187818881898190819181928193819481958196819781988199820082018202820382048205820682078208820982108211821282138214821582168217821882198220822182228223822482258226822782288229823082318232823382348235823682378238823982408241824282438244824582468247824882498250825182528253825482558256825782588259826082618262826382648265826682678268826982708271827282738274827582768277827882798280828182828283828482858286828782888289829082918292829382948295829682978298829983008301830283038304830583068307830883098310831183128313831483158316831783188319832083218322832383248325832683278328832983308331833283338334833583368337833883398340834183428343834483458346834783488349835083518352835383548355835683578358835983608361836283638364836583668367836883698370837183728373837483758376837783788379838083818382838383848385838683878388838983908391839283938394839583968397839883998400840184028403840484058406840784088409841084118412841384148415841684178418841984208421842284238424842584268427842884298430843184328433843484358436843784388439844084418442844384448445844684478448844984508451845284538454845584568457845884598460846184628463846484658466846784688469847084718472847384748475847684778478847984808481848284838484848584868487848884898490849184928493849484958496849784988499850085018502850385048505850685078508850985108511851285138514851585168517851885198520852185228523852485258526852785288529853085318532853385348535853685378538853985408541854285438544854585468547854885498550855185528553855485558556855785588559856085618562856385648565856685678568856985708571857285738574857585768577857885798580858185828583858485858586858785888589859085918592859385948595859685978598859986008601860286038604860586068607860886098610861186128613861486158616861786188619862086218622862386248625862686278628862986308631863286338634863586368637863886398640864186428643864486458646864786488649865086518652865386548655865686578658865986608661866286638664866586668667866886698670867186728673867486758676867786788679868086818682868386848685868686878688868986908691869286938694869586968697869886998700870187028703870487058706870787088709871087118712871387148715871687178718871987208721872287238724872587268727872887298730873187328733873487358736873787388739874087418742874387448745874687478748874987508751875287538754875587568757875887598760876187628763876487658766876787688769877087718772877387748775877687778778877987808781878287838784878587868787878887898790879187928793879487958796879787988799880088018802880388048805880688078808880988108811881288138814881588168817881888198820882188228823882488258826882788288829883088318832883388348835883688378838883988408841884288438844884588468847884888498850885188528853885488558856885788588859886088618862886388648865886688678868886988708871887288738874887588768877887888798880888188828883888488858886888788888889889088918892889388948895889688978898889989008901890289038904890589068907890889098910891189128913891489158916891789188919892089218922892389248925892689278928892989308931893289338934893589368937893889398940894189428943894489458946894789488949895089518952895389548955895689578958895989608961896289638964896589668967896889698970897189728973897489758976897789788979898089818982898389848985898689878988898989908991899289938994899589968997899889999000900190029003900490059006900790089009901090119012901390149015901690179018901990209021902290239024902590269027902890299030903190329033903490359036903790389039904090419042904390449045904690479048904990509051905290539054905590569057905890599060906190629063906490659066906790689069907090719072907390749075907690779078907990809081908290839084908590869087908890899090909190929093909490959096909790989099910091019102910391049105910691079108910991109111911291139114911591169117911891199120912191229123912491259126912791289129913091319132913391349135913691379138913991409141914291439144914591469147914891499150915191529153915491559156915791589159916091619162916391649165916691679168916991709171917291739174917591769177917891799180918191829183918491859186918791889189919091919192919391949195919691979198919992009201920292039204920592069207920892099210921192129213921492159216921792189219922092219222922392249225922692279228922992309231923292339234923592369237923892399240924192429243924492459246924792489249925092519252925392549255925692579258925992609261926292639264926592669267926892699270927192729273927492759276927792789279928092819282928392849285928692879288928992909291929292939294929592969297929892999300930193029303930493059306930793089309931093119312931393149315931693179318931993209321932293239324932593269327932893299330933193329333933493359336933793389339934093419342934393449345934693479348934993509351935293539354935593569357935893599360936193629363936493659366936793689369937093719372937393749375937693779378937993809381938293839384938593869387938893899390939193929393939493959396939793989399940094019402940394049405940694079408940994109411941294139414941594169417941894199420942194229423942494259426942794289429943094319432943394349435943694379438943994409441944294439444944594469447944894499450945194529453945494559456945794589459946094619462946394649465946694679468946994709471947294739474947594769477947894799480948194829483948494859486948794889489949094919492949394949495949694979498949995009501950295039504950595069507950895099510951195129513951495159516951795189519952095219522952395249525952695279528952995309531953295339534953595369537953895399540954195429543954495459546954795489549955095519552955395549555955695579558955995609561956295639564956595669567956895699570957195729573957495759576957795789579958095819582958395849585958695879588958995909591959295939594959595969597959895999600960196029603960496059606960796089609961096119612961396149615961696179618961996209621962296239624962596269627962896299630963196329633963496359636963796389639964096419642964396449645964696479648964996509651965296539654965596569657965896599660966196629663966496659666966796689669967096719672967396749675967696779678967996809681968296839684968596869687968896899690969196929693969496959696969796989699970097019702970397049705970697079708970997109711971297139714971597169717971897199720972197229723972497259726972797289729973097319732973397349735973697379738973997409741974297439744974597469747974897499750975197529753975497559756975797589759976097619762976397649765976697679768976997709771977297739774977597769777977897799780978197829783978497859786978797889789979097919792979397949795979697979798979998009801980298039804980598069807980898099810981198129813981498159816981798189819982098219822982398249825982698279828982998309831983298339834983598369837983898399840984198429843984498459846984798489849985098519852985398549855985698579858985998609861986298639864986598669867986898699870987198729873987498759876987798789879988098819882988398849885988698879888988998909891989298939894989598969897989898999900990199029903990499059906990799089909991099119912991399149915991699179918991999209921992299239924992599269927992899299930993199329933993499359936993799389939994099419942994399449945994699479948994999509951995299539954995599569957995899599960996199629963996499659966996799689969997099719972997399749975997699779978997999809981998299839984998599869987998899899990999199929993999499959996999799989999100001000110002100031000410005100061000710008100091001010011100121001310014100151001610017100181001910020100211002210023100241002510026100271002810029100301003110032100331003410035100361003710038100391004010041100421004310044100451004610047100481004910050100511005210053100541005510056100571005810059100601006110062100631006410065100661006710068100691007010071100721007310074100751007610077100781007910080100811008210083100841008510086100871008810089100901009110092100931009410095100961009710098100991010010101101021010310104101051010610107101081010910110101111011210113101141011510116101171011810119101201012110122101231012410125101261012710128101291013010131101321013310134101351013610137101381013910140101411014210143101441014510146101471014810149101501015110152101531015410155101561015710158101591016010161101621016310164101651016610167101681016910170101711017210173101741017510176101771017810179101801018110182101831018410185101861018710188101891019010191101921019310194101951019610197101981019910200102011020210203102041020510206102071020810209102101021110212102131021410215102161021710218102191022010221102221022310224102251022610227102281022910230102311023210233102341023510236102371023810239102401024110242102431024410245102461024710248102491025010251102521025310254102551025610257102581025910260102611026210263102641026510266102671026810269102701027110272102731027410275102761027710278102791028010281102821028310284102851028610287102881028910290102911029210293102941029510296102971029810299103001030110302103031030410305103061030710308103091031010311103121031310314103151031610317103181031910320103211032210323103241032510326103271032810329103301033110332103331033410335103361033710338103391034010341103421034310344103451034610347103481034910350103511035210353103541035510356103571035810359103601036110362103631036410365103661036710368103691037010371103721037310374103751037610377103781037910380103811038210383103841038510386103871038810389103901039110392103931039410395103961039710398103991040010401104021040310404104051040610407104081040910410104111041210413104141041510416104171041810419104201042110422104231042410425104261042710428104291043010431104321043310434104351043610437104381043910440104411044210443104441044510446104471044810449104501045110452104531045410455104561045710458104591046010461104621046310464104651046610467104681046910470104711047210473104741047510476104771047810479104801048110482104831048410485104861048710488104891049010491104921049310494104951049610497104981049910500105011050210503105041050510506105071050810509105101051110512105131051410515105161051710518105191052010521105221052310524105251052610527105281052910530105311053210533105341053510536105371053810539105401054110542105431054410545105461054710548105491055010551105521055310554105551055610557105581055910560105611056210563105641056510566105671056810569105701057110572105731057410575105761057710578105791058010581105821058310584105851058610587105881058910590105911059210593105941059510596105971059810599106001060110602106031060410605106061060710608106091061010611106121061310614106151061610617106181061910620106211062210623106241062510626106271062810629106301063110632106331063410635106361063710638106391064010641106421064310644106451064610647106481064910650106511065210653106541065510656106571065810659106601066110662106631066410665106661066710668106691067010671106721067310674106751067610677106781067910680106811068210683106841068510686106871068810689106901069110692106931069410695106961069710698106991070010701107021070310704107051070610707107081070910710107111071210713107141071510716107171071810719107201072110722107231072410725107261072710728107291073010731107321073310734107351073610737107381073910740107411074210743107441074510746107471074810749107501075110752107531075410755107561075710758107591076010761 |
- /*##############################################################################
- HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- ############################################################################## */
- #include "jlib.hpp"
- #include "workunit.hpp"
- #include "jprop.hpp"
- #include "jmisc.hpp"
- #include "jexcept.hpp"
- #include "jiter.ipp"
- #include "jptree.hpp"
- #include "jtime.ipp"
- #include "jencrypt.hpp"
- #include "junicode.hpp"
- #include "jlzw.hpp"
- #include "jregexp.hpp"
- #include "eclrtl.hpp"
- #include "deftype.hpp"
- #include <time.h>
- #include "mpbase.hpp"
- #include "daclient.hpp"
- #include "dadfs.hpp"
- #include "dafdesc.hpp"
- #include "dasds.hpp"
- #include "danqs.hpp"
- #include "dautils.hpp"
- #include "dllserver.hpp"
- #include "thorhelper.hpp"
- #include "workflow.hpp"
- #include "nbcd.hpp"
- #include "seclib.hpp"
- #include "wuerror.hpp"
- #include "wujobq.hpp"
- #include "environment.hpp"
- #define GLOBAL_WORKUNIT "global"
- #define SDS_LOCK_TIMEOUT (5*60*1000) // 5mins, 30s a bit short
- static int workUnitTraceLevel = 1;
- static StringBuffer &getXPath(StringBuffer &wuRoot, const char *wuid)
- {
- // MORE - can fold in the date
- return wuRoot.append("/WorkUnits/").append(wuid);
- }
- //To be called by eclserver, but esp etc. won't know, so we need to store it.
- static StringBuffer & appendLibrarySuffix(StringBuffer & suffix)
- {
- #ifdef _WIN32
- suffix.append("W");
- #else
- suffix.append("L");
- #endif
- #ifdef __64BIT__
- suffix.append("64");
- #else
- suffix.append("32");
- #endif
- return suffix;
- }
- typedef MapStringTo<bool> UniqueScopes;
- static void wuAccessError(const char *username, const char *action, const char *wuscope, const char *wuid, bool excpt, bool log)
- {
- StringBuffer err;
- err.append("Workunit Access Denied - action: ").append(action).append(" user:").append(username ? username : "<Unknown>");
- if (wuid)
- err.append(" workunit:").append(wuid);
- if (wuscope)
- err.append(" scope:").append(wuscope);
- //MORE - we would need more information passed in from outside if we want to make the audit message format the same as from higher level ESP calls
- SYSLOG(AUDIT_TYPE_ACCESS_FAILURE, err.str());
- if (log)
- LOG(MCuserError, "%s", err.str());
- if (excpt)
- throw MakeStringException(WUERR_AccessError, "%s", err.str());
- }
- static bool checkWuScopeSecAccess(const char *wuscope, ISecManager &secmgr, ISecUser *secuser, int required, const char *action, bool excpt, bool log)
- {
- bool ret=(!secuser) ? true : (secmgr.authorizeEx(RT_WORKUNIT_SCOPE, *secuser, wuscope)>=required);
- if (!ret && (log || excpt))
- wuAccessError(secuser ? secuser->getName() : NULL, action, wuscope, NULL, excpt, log);
- return ret;
- }
- static bool checkWuScopeListSecAccess(const char *wuscope, ISecResourceList *scopes, int required, const char *action, bool excpt, bool log)
- {
- if (!scopes)
- return true;
- bool ret=true;
- if (wuscope)
- {
- Owned<ISecResource> res=scopes->getResource(wuscope);
- if (!res || res->getAccessFlags()<required)
- ret=false;
- }
- else
- {
- for (int seq=0; ret && seq<scopes->count(); seq++)
- {
- ISecResource *res=scopes->queryResource(seq);
- if (res && res->getAccessFlags()<required)
- return false;
- }
- }
- if (!ret && (log || excpt))
- wuAccessError(NULL, action, wuscope, NULL, excpt, log);
- return ret;
- }
- static bool checkWuSecAccess(IConstWorkUnit &cw, ISecManager &secmgr, ISecUser *secuser, int required, const char *action, bool excpt, bool log)
- {
- SCMStringBuffer wuscope;
- bool ret=(!secuser) ? true : (secmgr.authorizeEx(RT_WORKUNIT_SCOPE, *secuser, cw.getWuScope(wuscope).str())>=required);
- if (!ret && (log || excpt))
- {
- wuAccessError(secuser ? secuser->getName() : NULL, action, wuscope.str(), cw.queryWuid(), excpt, log);
- }
- return ret;
- }
- static bool checkWuSecAccess(const char *wuid, ISecManager &secmgr, ISecUser *secuser, int required, const char *action, bool excpt, bool log)
- {
- StringBuffer wuRoot;
- Owned<IRemoteConnection> conn = querySDS().connect(getXPath(wuRoot, wuid).str(), myProcessSession(), 0, SDS_LOCK_TIMEOUT);
- if (conn)
- {
- Owned<IPropertyTree> ptree=conn->getRoot();
- return checkWuScopeSecAccess(ptree->queryProp("@scope"), secmgr, secuser, required, action, excpt, log);
- }
- if (log || excpt)
- wuAccessError(secuser ? secuser->getName() : NULL, action, "Unknown", NULL, excpt, log);
- return false;
- }
- void doDescheduleWorkkunit(char const * wuid)
- {
- StringBuffer xpath;
- xpath.append("*/*/*/");
- ncnameEscape(wuid, xpath);
- Owned<IRemoteConnection> conn = querySDS().connect("/Schedule", myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
- if(!conn) return;
- Owned<IPropertyTree> root = conn->getRoot();
- bool more;
- do more = root->removeProp(xpath.str()); while(more);
- }
- #define PROGRESS_FORMAT_V 2
- class CConstGraphProgress : public CInterface, implements IConstWUGraphProgress
- {
- class CWuGraphStats : public CInterfaceOf<IWUGraphStats>
- {
- public:
- CWuGraphStats(CConstGraphProgress &_parent, StatisticCreatorType _creatorType, const char * _creator, const char * _rootScope, unsigned _id)
- : parent(_parent), creatorType(_creatorType), creator(_creator), id(_id)
- {
- StringBuffer subgraphScopeName;
- subgraphScopeName.append(_rootScope);
- StatisticScopeType scopeType = SSTgraph;
- StatsScopeId rootScopeId;
- verifyex(rootScopeId.setScopeText(_rootScope));
- collector.setown(createStatisticsGatherer(_creatorType, _creator, rootScopeId));
- }
- virtual void beforeDispose()
- {
- Owned<IStatisticCollection> stats = collector->getResult();
- MemoryBuffer compressed;
- {
- MemoryBuffer serialized;
- serializeStatisticCollection(serialized, stats);
- compressToBuffer(compressed, serialized.length(), serialized.toByteArray());
- }
- unsigned minActivity = 0;
- unsigned maxActivity = 0;
- stats->getMinMaxActivity(minActivity, maxActivity);
- parent.setSubgraphStats(creatorType, creator, id, compressed, minActivity, maxActivity);
- }
- virtual IStatisticGatherer & queryStatsBuilder()
- {
- return *collector;
- }
- protected:
- CConstGraphProgress &parent;
- Owned<IStatisticGatherer> collector;
- StringAttr creator;
- StatisticCreatorType creatorType;
- unsigned id;
- };
- class CGraphProgress : public CInterface, implements IWUGraphProgress
- {
- CConstGraphProgress &parent;
- public:
- IMPLEMENT_IINTERFACE;
- CGraphProgress(CConstGraphProgress &_parent) : parent(_parent)
- {
- parent.lockWrite();
- }
- ~CGraphProgress()
- {
- parent.unlock();
- }
- virtual IPropertyTree * getProgressTree() { return parent.getProgressTree(); }
- virtual WUGraphState queryGraphState() { return parent.queryGraphState(); }
- virtual WUGraphState queryNodeState(WUGraphIDType nodeId) { return parent.queryNodeState(nodeId); }
- virtual IWUGraphProgress * update() { throwUnexpected(); }
- virtual IWUGraphStats * update(StatisticCreatorType creatorType, const char * creator, unsigned subgraph) { throwUnexpected(); }
- virtual unsigned queryFormatVersion() { return parent.queryFormatVersion(); }
- virtual void setGraphState(WUGraphState state)
- {
- parent.setGraphState(state);
- }
- virtual void setNodeState(WUGraphIDType nodeId, WUGraphState state)
- {
- parent.setNodeState(nodeId, state);
- }
- };
- void clearConnection()
- {
- conn.clear();
- progress.clear();
- }
- public:
- IMPLEMENT_IINTERFACE;
- static void deleteWuidProgress(const char *wuid)
- {
- StringBuffer path("/GraphProgress/");
- path.append(wuid);
- Owned<IRemoteConnection> conn = querySDS().connect(path.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
- if (conn)
- conn->close(true);
- }
- CConstGraphProgress(const char *_wuid, const char *_graphName) : wuid(_wuid), graphName(_graphName)
- {
- rootPath.append("/GraphProgress/").append(wuid).append('/').append(graphName).append('/');
- connected = connectedWrite = false;
- formatVersion = 0;
- }
- CConstGraphProgress(const char *_wuid, const char *_graphName, IPropertyTree *_progress) : wuid(_wuid), graphName(_graphName), progress(_progress)
- {
- formatVersion = progress->getPropInt("@format");
- connectedWrite = false; // should never be
- connected = true;
- }
- void connect()
- {
- clearConnection();
- packProgress(wuid,false);
- conn.setown(querySDS().connect(rootPath.str(), myProcessSession(), RTM_LOCK_READ|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT));
- progress.set(conn->queryRoot());
- formatVersion = progress->getPropInt("@format");
- connected = true;
- }
- void lockWrite()
- {
- if (connectedWrite) return;
- if (!rootPath.length())
- throw MakeStringException(WUERR_GraphProgressWriteUnsupported, "Writing to graph progress unsupported in this context");
- // JCSMORE - look at using changeMode here.
- if (conn)
- clearConnection();
- else
- packProgress(wuid,false);
- conn.setown(querySDS().connect(rootPath.str(), myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT));
- progress.set(conn->queryRoot());
- if (!progress->hasChildren()) // i.e. blank.
- {
- formatVersion = PROGRESS_FORMAT_V;
- progress->setPropInt("@format", PROGRESS_FORMAT_V);
- }
- else
- formatVersion = progress->getPropInt("@format");
- connected = connectedWrite = true;
- }
- void unlock()
- {
- connected = false;
- connectedWrite = false;
- clearConnection();
- }
- static bool getRunningGraph(const char *wuid, IStringVal &graphName, WUGraphIDType &subId)
- {
- StringBuffer path;
- Owned<IRemoteConnection> conn = querySDS().connect(path.append("/GraphProgress/").append(wuid).str(), myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT);
- if (!conn) return false;
- const char *name = conn->queryRoot()->queryProp("Running/@graph");
- if (name)
- {
- graphName.set(name);
- subId = conn->queryRoot()->getPropInt64("Running/@subId");
- return true;
- }
- else
- return false;
- }
- void setGraphState(WUGraphState state)
- {
- progress->setPropInt("@_state", (unsigned)state);
- }
- void setNodeState(WUGraphIDType nodeId, WUGraphState state)
- {
- if (!connectedWrite) lockWrite();
- StringBuffer path;
- path.append("node[@id=\"").append(nodeId).append("\"]");
- IPropertyTree *node = progress->queryPropTree(path.str());
- if (!node)
- {
- node = progress->addPropTree("node", createPTree());
- node->setPropInt("@id", (int)nodeId);
- }
- node->setPropInt("@_state", (unsigned)state);
-
- switch (state)
- {
- case WUGraphRunning:
- {
- StringBuffer path;
- Owned<IRemoteConnection> conn = querySDS().connect(path.append("/GraphProgress/").append(wuid).str(), myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
- IPropertyTree *running = conn->queryRoot()->setPropTree("Running", createPTree());
- running->setProp("@graph", graphName);
- running->setPropInt64("@subId", nodeId);
- break;
- }
- case WUGraphComplete:
- {
- StringBuffer path;
- Owned<IRemoteConnection> conn = querySDS().connect(path.append("/GraphProgress/").append(wuid).str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
- conn->queryRoot()->removeProp("Running"); // only one thing running at any given time and one thing with lockWrite access
- break;
- }
- }
- }
- virtual IPropertyTree * getProgressTree()
- {
- if (!connected) connect();
- if (progress->getPropBool("@stats"))
- return createProcessTreeFromStats();
- return LINK(progress);
- }
- virtual WUGraphState queryGraphState()
- {
- return (WUGraphState)queryProgressStateTree()->getPropInt("@_state", (unsigned)WUGraphUnknown);
- }
- virtual WUGraphState queryNodeState(WUGraphIDType nodeId)
- {
- StringBuffer path;
- path.append("node[@id=\"").append(nodeId).append("\"]/@_state");
- return (WUGraphState)queryProgressStateTree()->getPropInt(path.str(), (unsigned)WUGraphUnknown);
- }
- virtual IWUGraphProgress * update()
- {
- return new CGraphProgress(*this);
- }
- virtual IWUGraphStats * update(StatisticCreatorType creatorType, const char * creator, unsigned subgraph)
- {
- return new CWuGraphStats(*this, creatorType, creator, graphName, subgraph);
- }
- virtual unsigned queryFormatVersion()
- {
- if (!connected) connect();
- return formatVersion;
- }
- static bool packProgress(const char *wuid,bool pack)
- {
- //MORE: This function could be deleted, because progress information is never actually packed
- StringBuffer path;
- path.append("/GraphProgress/").append(wuid);
- Owned<IRemoteConnection> conn(querySDS().connect(path.str(), myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT));
- if (!conn)
- return false;
- Owned<IPropertyTree> newt;
- MemoryBuffer buf;
- IPropertyTree *root = conn->queryRoot();
- if (root->getPropBin("Packed",buf)) {
- if (pack)
- return true;
- newt.setown(createPTree(buf));
- IPropertyTree *running = root->queryPropTree("Running");
- if (running)
- newt->setPropTree("Running",createPTreeFromIPT(running));
- }
- else {
- if (!pack)
- return true;
- newt.setown(createPTree(wuid));
- IPropertyTree *running = root->queryPropTree("Running");
- if (running) {
- newt->setPropTree("Running",createPTreeFromIPT(running));
- root->removeTree(running);
- }
- root->serialize(buf);
- newt->setPropBin("Packed",buf.length(),buf.bufferBase());
- }
- root->setPropTree(NULL,newt.getClear());
- return true;
- }
- private:
- IPropertyTree * queryProgressStateTree()
- {
- if (!connected) connect();
- return progress;
- }
- static void expandStats(IPropertyTree * target, IStatisticCollection & collection)
- {
- StringBuffer formattedValue;
- unsigned numStats = collection.getNumStatistics();
- for (unsigned i=0; i < numStats; i++)
- {
- StatisticKind kind;
- unsigned __int64 value;
- collection.getStatistic(kind, value, i);
- formatStatistic(formattedValue.clear(), value, kind);
- target->setProp(queryTreeTag(kind), formattedValue);
- }
- }
- void expandProcessTreeFromStats(IPropertyTree * rootTarget, IPropertyTree * target, IStatisticCollection * collection)
- {
- expandStats(target, *collection);
- StringBuffer scopeName;
- Owned<IStatisticCollectionIterator> activityIter = &collection->getScopes(NULL);
- ForEach(*activityIter)
- {
- IStatisticCollection & cur = activityIter->query();
- cur.getScope(scopeName.clear());
- const char * id = scopeName.str();
- const char * tag;
- IPropertyTree * curTarget = target;
- switch (cur.queryScopeType())
- {
- case SSTedge:
- tag = "edge";
- id += strlen(EdgeScopePrefix);
- break;
- case SSTactivity:
- tag = "node";
- id += strlen(ActivityScopePrefix);
- break;
- case SSTsubgraph:
- //All subgraphs are added a root elements in the progress tree
- curTarget = rootTarget;
- tag = "node";
- id += strlen(SubGraphScopePrefix);
- break;
- default:
- throwUnexpected();
- }
- IPropertyTree * next = curTarget->addPropTree(tag, createPTree());
- next->setProp("@id", id);
- expandProcessTreeFromStats(rootTarget, next, &cur);
- }
- }
- IPropertyTree * createProcessTreeFromStats()
- {
- MemoryBuffer compressed;
- MemoryBuffer serialized;
- Owned<IPropertyTree> progressTree = createPTree();
- Owned<IPropertyTreeIterator> iter = progress->getElements("sg*");
- ForEach(*iter)
- {
- IPropertyTree & curSubGraph = iter->query();
- curSubGraph.getPropBin("Stats", compressed.clear());
- //Protect against updates that delete the stats while we are iterating
- if (compressed.length())
- {
- decompressToBuffer(serialized.clear(), compressed);
- Owned<IStatisticCollection> collection = createStatisticCollection(serialized);
- expandProcessTreeFromStats(progressTree, progressTree, collection);
- }
- }
- return progressTree.getClear();
- }
- void setSubgraphStats(StatisticCreatorType creatorType, const char * creator, unsigned id, const MemoryBuffer & compressed, unsigned minActivity, unsigned maxActivity)
- {
- StringBuffer tag;
- tag.append("sg").append(id);
- IPropertyTree * subgraph = createPTree(tag);
- subgraph->setProp("@c", queryCreatorTypeName(creatorType));
- subgraph->setProp("@creator", creator);
- subgraph->setPropInt("@minActivity", minActivity);
- subgraph->setPropInt("@maxActivity", maxActivity);
- //Replace the particular subgraph statistics added by this creator
- tag.append("[@creator='").append(creator).append("']");
- lockWrite();
- subgraph = progress->setPropTree(tag, subgraph);
- subgraph->setPropBin("Stats", compressed.length(), compressed.toByteArray());
- if (!progress->getPropBool("@stats", false))
- progress->setPropBool("@stats", true);
- unlock();
- }
- private:
- Owned<IRemoteConnection> conn;
- Linked<IPropertyTree> progress;
- StringAttr wuid, graphName;
- StringBuffer rootPath;
- bool connected, connectedWrite;
- unsigned formatVersion;
- };
- //--------------------------------------------------------------------------------------------------------------------
- class ExtractedStatistic : public CInterfaceOf<IConstWUStatistic>
- {
- public:
- virtual IStringVal & getDescription(IStringVal & str, bool createDefault) const
- {
- str.set(description);
- return str;
- }
- virtual IStringVal & getCreator(IStringVal & str) const
- {
- str.set(creator);
- return str;
- }
- virtual IStringVal & getScope(IStringVal & str) const
- {
- str.set(scope);
- return str;
- }
- virtual IStringVal & getFormattedValue(IStringVal & str) const
- {
- StringBuffer formatted;
- formatStatistic(formatted, value, measure);
- str.set(formatted);
- return str;
- }
- virtual StatisticMeasure getMeasure() const
- {
- return measure;
- }
- virtual StatisticKind getKind() const
- {
- return kind;
- }
- virtual StatisticCreatorType getCreatorType() const
- {
- return creatorType;
- }
- virtual StatisticScopeType getScopeType() const
- {
- return scopeType;
- }
- virtual unsigned __int64 getValue() const
- {
- return value;
- }
- virtual unsigned __int64 getCount() const
- {
- return count;
- }
- virtual unsigned __int64 getMax() const
- {
- return max;
- }
- virtual unsigned __int64 getTimestamp() const
- {
- return timeStamp;
- }
- virtual bool matches(const IStatisticsFilter * filter) const
- {
- return filter->matches(creatorType, creator, scopeType, scope, measure, kind);
- }
- public:
- StringAttr creator;
- StringAttr description;
- StringBuffer scope;
- StatisticMeasure measure;
- StatisticKind kind;
- StatisticCreatorType creatorType;
- StatisticScopeType scopeType;
- unsigned __int64 value;
- unsigned __int64 count;
- unsigned __int64 max;
- unsigned __int64 timeStamp;
- };
- class CConstGraphProgressStatisticsIterator : public CInterfaceOf<IConstWUStatisticIterator>
- {
- public:
- CConstGraphProgressStatisticsIterator(const char * wuid, const IStatisticsFilter * _filter) : filter(_filter)
- {
- if (filter)
- scopes.appendList(filter->queryScope(), ":");
- const char * searchGraph = "*";
- if (scopes.ordinality())
- searchGraph = scopes.item(0);
- rootPath.append("/GraphProgress/").append(wuid).append('/');
- bool singleGraph = false;
- if (!containsWildcard(searchGraph))
- {
- rootPath.append(searchGraph).append("/");
- singleGraph = true;
- }
- //Don't lock the statistics while we iterate - any partial updates must not cause problems
- conn.setown(querySDS().connect(rootPath.str(), myProcessSession(), RTM_NONE, SDS_LOCK_TIMEOUT));
- if (conn && !singleGraph)
- graphIter.setown(conn->queryRoot()->getElements("*"));
- curStat.setown(new ExtractedStatistic);
- //These are currently constant for all graph statistics instances
- curStat->count = 1;
- curStat->max = 0;
- valid = false;
- }
- virtual IConstWUStatistic & query()
- {
- return *curStat;
- }
- virtual bool first()
- {
- valid = false;
- if (!conn)
- return false;
- if (graphIter && !graphIter->first())
- return false;
- ensureUniqueStatistic();
- if (!firstSubGraph())
- return false;
- valid = true;
- return true;
- }
- virtual bool next()
- {
- ensureUniqueStatistic();
- if (!nextStatistic())
- {
- if (!nextSubGraph())
- {
- if (!nextGraph())
- {
- valid = false;
- return false;
- }
- }
- }
- return true;
- }
- virtual bool isValid()
- {
- return valid;
- }
- protected:
- bool firstSubGraph()
- {
- IPropertyTree & graphNode = graphIter ? graphIter->query() : *conn->queryRoot();
- const char * xpath = "sg*";
- StringBuffer childXpath;
- if (scopes.isItem(1))
- {
- const char * scope1 = scopes.item(1);
- if (strnicmp(scope1, "sg", 2) == 0)
- {
- childXpath.append(scope1);
- xpath = childXpath.str();
- }
- }
- subgraphIter.setown(graphNode.getElements(xpath));
- if (!subgraphIter)
- subgraphIter.setown(graphNode.getElements("sg0"));
- if (!subgraphIter->first())
- return false;
- if (firstStat())
- return true;
- return nextSubGraph();
- }
- bool nextSubGraph()
- {
- loop
- {
- if (!subgraphIter->next())
- return false;
- if (firstStat())
- return true;
- }
- }
- bool nextGraph()
- {
- if (!graphIter)
- return false;
- loop
- {
- if (!graphIter->next())
- return false;
- if (firstSubGraph())
- return true;
- }
- }
- bool firstStat()
- {
- IPropertyTree & curSubGraph = subgraphIter->query();
- if (!checkSubGraph())
- return false;
- curSubGraph.getPropBin("Stats", compressed.clear());
- //Don't crash on old format progress...
- if (compressed.length() == 0)
- return false;
- decompressToBuffer(serialized.clear(), compressed);
- Owned<IStatisticCollection> collection = createStatisticCollection(serialized);
- curStat->timeStamp = collection->queryWhenCreated();
- return beginCollection(*collection);
- }
- bool beginCollection(IStatisticCollection & collection)
- {
- collections.append(OLINK(collection));
- numStats = collection.getNumStatistics();
- curStatIndex = 0;
- if (checkScope())
- {
- if (curStatIndex < numStats)
- {
- if (checkStatistic())
- return true;
- return nextStatistic();
- }
- }
- return nextChildScope();
- }
- bool nextStatistic()
- {
- //Finish iterating the statistics at this level.
- while (++curStatIndex < numStats)
- {
- if (checkStatistic())
- return true;
- }
- return nextChildScope();
- }
- bool nextChildScope()
- {
- loop
- {
- if (collections.ordinality() == 0)
- return false;
- IStatisticCollection * curCollection = &collections.tos();
- if (childIterators.ordinality() < collections.ordinality())
- {
- //Start iterating the children for the current collection
- childIterators.append(curCollection->getScopes(NULL));
- if (!childIterators.tos().first())
- {
- finishCollection();
- continue;
- }
- }
- else if (!childIterators.tos().next())
- {
- finishCollection();
- continue;
- }
- if (beginCollection(childIterators.tos().query()))
- return true;
- }
- }
- void finishCollection()
- {
- collections.pop();
- childIterators.pop();
- }
- bool checkSubGraph()
- {
- if (!filter)
- return true;
- IPropertyTree & curSubGraph = subgraphIter->query();
- curStat->creatorType = queryCreatorType(curSubGraph.queryProp("@c"));
- curStat->creator.set(curSubGraph.queryProp("@creator"));
- return filter->matches(curStat->creatorType, curStat->creator, SSTall, NULL, SMeasureAll, StKindAll);
- }
- bool checkScope()
- {
- if (!filter)
- return true;
- IStatisticCollection * collection = &collections.tos();
- curStat->scopeType = collection->queryScopeType();
- collection->getFullScope(curStat->scope.clear());
- return filter->matches(SCTall, NULL, curStat->scopeType, curStat->scope, SMeasureAll, StKindAll);
- }
- bool checkStatistic()
- {
- IStatisticCollection & collection = collections.tos();
- collection.getStatistic(curStat->kind, curStat->value, curStatIndex);
- curStat->measure = queryMeasure(curStat->kind);
- if (!filter)
- return true;
- if (!filter->matches(SCTall, NULL, SSTall, NULL, curStat->measure, curStat->kind))
- return false;
- return true;
- }
- void ensureUniqueStatistic()
- {
- //If something else has linked this statistic, clone a unique one.
- if (curStat->IsShared())
- curStat.setown(new ExtractedStatistic(*curStat));
- }
- private:
- Owned<IRemoteConnection> conn;
- Owned<ExtractedStatistic> curStat;
- const IStatisticsFilter * filter;
- StringArray scopes;
- StringBuffer rootPath;
- Owned<IPropertyTreeIterator> graphIter;
- Owned<IPropertyTreeIterator> subgraphIter;
- IArrayOf<IStatisticCollection> collections;
- IArrayOf<IStatisticCollectionIterator> childIterators;
- MemoryBuffer compressed;
- MemoryBuffer serialized;
- unsigned numStats;
- unsigned curStatIndex;
- bool valid;
- };
- //--------------------------------------------------------------------------------------------------------------------
- class CLocalWUAppValue : public CInterface, implements IConstWUAppValue
- {
- Owned<IPropertyTree> p;
- StringAttr prop;
- public:
- IMPLEMENT_IINTERFACE;
- CLocalWUAppValue(IPropertyTree *p,unsigned child);
- virtual IStringVal & getApplication(IStringVal & str) const;
- virtual IStringVal & getName(IStringVal & str) const;
- virtual IStringVal & getValue(IStringVal & dt) const;
- };
- class CLocalWUStatistic : public CInterface, implements IConstWUStatistic
- {
- Owned<IPropertyTree> p;
- public:
- IMPLEMENT_IINTERFACE;
- CLocalWUStatistic(IPropertyTree *p);
- virtual IStringVal & getCreator(IStringVal & str) const;
- virtual IStringVal & getDescription(IStringVal & str, bool createDefault) const;
- virtual IStringVal & getFormattedValue(IStringVal & str) const;
- virtual IStringVal & getType(IStringVal & str) const;
- virtual IStringVal & getScope(IStringVal & str) const;
- virtual StatisticMeasure getMeasure() const;
- virtual StatisticCreatorType getCreatorType() const;
- virtual StatisticScopeType getScopeType() const;
- virtual StatisticKind getKind() const;
- virtual unsigned __int64 getValue() const;
- virtual unsigned __int64 getCount() const;
- virtual unsigned __int64 getMax() const;
- virtual unsigned __int64 getTimestamp() const;
- virtual bool matches(const IStatisticsFilter * filter) const;
- };
- class CLocalWULegacyTiming : public CInterface, implements IConstWUStatistic
- {
- Owned<IPropertyTree> p;
- public:
- IMPLEMENT_IINTERFACE;
- CLocalWULegacyTiming(IPropertyTree *p);
- virtual IStringVal & getCreator(IStringVal & str) const;
- virtual IStringVal & getDescription(IStringVal & str, bool createDefault) const;
- virtual IStringVal & getFormattedValue(IStringVal & str) const;
- virtual IStringVal & getType(IStringVal & str) const;
- virtual IStringVal & getScope(IStringVal & str) const;
- virtual StatisticMeasure getMeasure() const;
- virtual StatisticCreatorType getCreatorType() const;
- virtual StatisticScopeType getScopeType() const;
- virtual StatisticKind getKind() const;
- virtual unsigned __int64 getValue() const;
- virtual unsigned __int64 getCount() const;
- virtual unsigned __int64 getMax() const;
- virtual unsigned __int64 getTimestamp() const;
- virtual bool matches(const IStatisticsFilter * filter) const;
- };
- template <typename T, typename IT> struct CachedTags
- {
- CachedTags(): cached(false) {}
- void load(IPropertyTree* p,const char* xpath)
- {
- if (!cached)
- {
- assertex(tags.length() == 0);
- Owned<IPropertyTreeIterator> r = p->getElements(xpath);
- for (r->first(); r->isValid(); r->next())
- {
- IPropertyTree *rp = &r->query();
- rp->Link();
- tags.append(*new T(rp));
- }
- cached = true;
- }
- }
- void append(IPropertyTree * p)
- {
- tags.append(*new T(p));
- }
- operator IArrayOf<IT>&() { return tags; }
- unsigned ordinality() const { return tags.ordinality(); }
- void kill()
- {
- cached = false;
- tags.kill();
- }
- bool cached;
- IArrayOf<IT> tags;
- };
- template <> struct CachedTags<CLocalWUAppValue, IConstWUAppValue>
- {
- CachedTags(): cached(false) {}
- void load(IPropertyTree* p,const char* xpath)
- {
- if (!cached)
- {
- assertex(tags.length() == 0);
- Owned<IPropertyTreeIterator> r = p->getElements(xpath);
- for (r->first(); r->isValid(); r->next())
- {
- IPropertyTree *rp = &r->query();
- Owned<IPropertyTreeIterator> v = rp->getElements("*");
- unsigned pos = 1;
- for (v->first(); v->isValid(); v->next())
- {
- rp->Link();
- tags.append(*new CLocalWUAppValue(rp,pos++));
- }
- }
- cached = true;
- }
- }
- operator IArrayOf<IConstWUAppValue>&() { return tags; }
- void kill()
- {
- cached = false;
- tags.kill();
- }
- bool cached;
- IArrayOf<IConstWUAppValue> tags;
- };
- class CLocalWorkUnit : public CInterface, implements IWorkUnit , implements IExtendedWUInterface
- {
- friend StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool decodeGraphs, bool includeProgress);
- friend void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool decodeGraphs, bool includeProgress);
- protected:
- Owned<IPropertyTree> p;
- mutable CriticalSection crit;
- mutable Owned<IWUQuery> query;
- mutable Owned<IWUWebServicesInfo> webServicesInfo;
- mutable Owned<IWURoxieQueryInfo> roxieQueryInfo;
- mutable Owned<IWorkflowItemIterator> workflowIterator;
- mutable bool workflowIteratorCached;
- mutable bool resultsCached;
- mutable bool temporariesCached;
- mutable bool variablesCached;
- mutable bool exceptionsCached;
- mutable bool pluginsCached;
- mutable bool librariesCached;
- mutable bool activitiesCached;
- mutable bool webServicesInfoCached;
- mutable bool roxieQueryInfoCached;
- mutable IArrayOf<IWUPlugin> plugins;
- mutable IArrayOf<IWULibrary> libraries;
- mutable IArrayOf<IWUException> exceptions;
- mutable IArrayOf<IWUGraph> graphs;
- mutable IArrayOf<IWUResult> results;
- mutable IArrayOf<IWUResult> temporaries;
- mutable IArrayOf<IWUResult> variables;
- mutable CachedTags<CLocalWUAppValue,IConstWUAppValue> appvalues;
- mutable CachedTags<CLocalWUStatistic,IConstWUStatistic> statistics;
- mutable CachedTags<CLocalWULegacyTiming,IConstWUStatistic> legacyTimings;
- mutable Owned<IUserDescriptor> userDesc;
- Mutex locked;
- Owned<ISecManager> secMgr;
- Owned<ISecUser> secUser;
- mutable Owned<IPropertyTree> cachedGraphs;
- public:
- IMPLEMENT_IINTERFACE;
- CLocalWorkUnit(IPropertyTree* root, ISecManager *secmgr, ISecUser *secuser);
- void beforeDispose();
- CLocalWorkUnit(const char *dummyWuid, const char *xml, ISecManager *secmgr, ISecUser *secuser);
- IPropertyTree *getUnpackedTree(bool includeProgress) const;
- ISecManager *querySecMgr(){return secMgr.get();}
- ISecUser *querySecUser(){return secUser.get();}
- void setSecIfcs(ISecManager *mgr, ISecUser*usr){secMgr.set(mgr); secUser.set(usr);}
-
- virtual bool aborting() const;
- virtual void forceReload() {};
- virtual WUAction getAction() const;
- virtual IStringVal& getActionEx(IStringVal & str) const;
- virtual IStringVal & getApplicationValue(const char * application, const char * propname, IStringVal & str) const;
- virtual int getApplicationValueInt(const char * application, const char * propname, int defVal) const;
- virtual IConstWUAppValueIterator & getApplicationValues() const;
- virtual bool hasWorkflow() const;
- virtual unsigned queryEventScheduledCount() const;
- virtual IPropertyTree * queryWorkflowTree() const;
- virtual IConstWorkflowItemIterator * getWorkflowItems() const;
- virtual IWorkflowItemArray * getWorkflowClone() const;
- virtual IConstLocalFileUploadIterator * getLocalFileUploads() const;
- virtual bool requiresLocalFileUpload() const;
- virtual bool getIsQueryService() const;
- virtual const char *queryClusterName() const;
- virtual bool hasDebugValue(const char * propname) const;
- virtual IStringVal & getDebugValue(const char * propname, IStringVal & str) const;
- virtual IStringIterator & getDebugValues() const;
- virtual IStringIterator & getDebugValues(const char *prop) const;
- virtual int getDebugValueInt(const char * propname, int defVal) const;
- virtual __int64 getDebugValueInt64(const char * propname, __int64 defVal) const;
- virtual bool getDebugValueBool(const char * propname, bool defVal) const;
- virtual unsigned getExceptionCount() const;
- virtual IConstWUExceptionIterator & getExceptions() const;
- virtual IConstWUResult * getGlobalByName(const char * name) const;
- virtual unsigned getGraphCount() const;
- virtual unsigned getSourceFileCount() const;
- virtual unsigned getResultCount() const;
- virtual unsigned getVariableCount() const;
- virtual unsigned getApplicationValueCount() const;
- virtual IConstWUGraphIterator & getGraphs(WUGraphType type) const;
- virtual IConstWUGraphMetaIterator & getGraphsMeta(WUGraphType type) const;
- virtual IConstWUGraph * getGraph(const char *name) const;
- virtual IConstWUGraphProgress * getGraphProgress(const char * name) const;
- virtual const char *queryJobName() const;
- virtual IConstWUPlugin * getPluginByName(const char * name) const;
- virtual IConstWUPluginIterator & getPlugins() const;
- virtual IConstWULibraryIterator & getLibraries() const;
- virtual WUPriorityClass getPriority() const;
- virtual int getPriorityLevel() const;
- virtual int getPriorityValue() const;
- virtual IConstWUQuery * getQuery() const;
- virtual bool getRescheduleFlag() const;
- virtual IConstWUResult * getResultByName(const char * name) const;
- virtual IConstWUResult * getResultBySequence(unsigned seq) const;
- virtual unsigned getResultLimit() const;
- virtual IConstWUResultIterator & getResults() const;
- virtual IStringVal & getScope(IStringVal & str) const;
- virtual IStringVal & getSecurityToken(IStringVal & str) const;
- virtual WUState getState() const;
- virtual IStringVal & getStateEx(IStringVal & str) const;
- virtual __int64 getAgentSession() const;
- virtual unsigned getAgentPID() const;
- virtual const char *queryStateDesc() const;
- virtual IConstWUResult * getTemporaryByName(const char * name) const;
- virtual IConstWUResultIterator & getTemporaries() const;
- virtual IConstWUStatisticIterator & getStatistics(const IStatisticsFilter * filter) const;
- virtual IConstWUStatistic * getStatistic(const char * creator, const char * scope, StatisticKind kind) const;
- virtual IConstWUWebServicesInfo * getWebServicesInfo() const;
- virtual IConstWURoxieQueryInfo * getRoxieQueryInfo() const;
- virtual IStringVal & getXmlParams(IStringVal & params) const;
- virtual const IPropertyTree *getXmlParams() const;
- virtual unsigned __int64 getHash() const;
- virtual IStringIterator *getLogs(const char *type, const char *component) const;
- virtual IStringIterator *getProcesses(const char *type) const;
- virtual IPropertyTreeIterator* getProcesses(const char *type, const char *instance) const;
- virtual bool getWuDate(unsigned & year, unsigned & month, unsigned& day);
- virtual IStringVal & getSnapshot(IStringVal & str) const;
- virtual const char *queryUser() const;
- virtual IStringVal & getWuScope(IStringVal & str) const;
- virtual IConstWUResult * getVariableByName(const char * name) const;
- virtual IConstWUResultIterator & getVariables() const;
- virtual const char *queryWuid() const;
- virtual bool getRunningGraph(IStringVal &graphName, WUGraphIDType &subId) const;
- virtual bool isProtected() const;
- virtual bool isPausing() const;
- virtual IWorkUnit& lock();
- virtual void requestAbort();
- virtual void subscribe(WUSubscribeOptions options);
- virtual unsigned calculateHash(unsigned prevHash);
- virtual void copyWorkUnit(IConstWorkUnit *cached, bool all);
- virtual unsigned queryFileUsage(const char *filename) const;
- virtual bool getCloneable() const;
- virtual IUserDescriptor * queryUserDescriptor() const;
- virtual unsigned getCodeVersion() const;
- virtual unsigned getWuidVersion() const;
- virtual void getBuildVersion(IStringVal & buildVersion, IStringVal & eclVersion) const;
- virtual IPropertyTree * getDiskUsageStats();
- virtual IPropertyTreeIterator & getFileIterator() const;
- virtual bool archiveWorkUnit(const char *base,bool del,bool ignoredllerrors,bool deleteOwned);
- virtual void packWorkUnit(bool pack=true);
- virtual IJlibDateTime & getTimeScheduled(IJlibDateTime &val) const;
- virtual IPropertyTreeIterator & getFilesReadIterator() const;
- virtual void protect(bool protectMode);
- virtual IConstWULibrary * getLibraryByName(const char * name) const;
- virtual unsigned getDebugAgentListenerPort() const;
- virtual IStringVal & getDebugAgentListenerIP(IStringVal &ip) const;
- void clearExceptions();
- void commit();
- IWUException *createException();
- void addProcess(const char *type, const char *instance, unsigned pid, const char *log);
- void setAction(WUAction action);
- void setApplicationValue(const char * application, const char * propname, const char * value, bool overwrite);
- void setApplicationValueInt(const char * application, const char * propname, int value, bool overwrite);
- void incEventScheduledCount();
- void setIsQueryService(bool value);
- void setCloneable(bool value);
- void setIsClone(bool value);
- void setClusterName(const char * value);
- void setCodeVersion(unsigned version, const char * buildVersion, const char * eclVersion);
- void setDebugValue(const char * propname, const char * value, bool overwrite);
- void setDebugValueInt(const char * propname, int value, bool overwrite);
- void setJobName(const char * value);
- void setPriority(WUPriorityClass cls);
- void setPriorityLevel(int level);
- void setRescheduleFlag(bool value);
- void setResultLimit(unsigned value);
- void setState(WUState state);
- void setStateEx(const char * text);
- void setAgentSession(__int64 sessionId);
- void setSecurityToken(const char *value);
- void setStatistic(StatisticCreatorType creatorType, const char * creator, StatisticScopeType scopeType, const char * scope, StatisticKind kind, const char * optDescription, unsigned __int64 value, unsigned __int64 count, unsigned __int64 maxValue, StatsMergeAction mergeAction);
- void setTracingValue(const char * propname, const char * value);
- void setTracingValueInt(const char * propname, int value);
- void setUser(const char * value);
- void setWuScope(const char * value);
- void setSnapshot(const char * value);
- void setDebugAgentListenerPort(unsigned port);
- void setDebugAgentListenerIP(const char * ip);
- void setXmlParams(const char *params);
- void setXmlParams(IPropertyTree *tree);
- void setHash(unsigned __int64 hash);
- IWorkflowItem* addWorkflowItem(unsigned wfid, WFType type, WFMode mode, unsigned success, unsigned failure, unsigned recovery, unsigned retriesAllowed, unsigned contingencyFor);
- IWorkflowItemIterator * updateWorkflowItems();
- void syncRuntimeWorkflow(IWorkflowItemArray * array);
- void resetWorkflow();
- void schedule();
- void deschedule();
- unsigned addLocalFileUpload(LocalFileUploadType type, char const * source, char const * destination, char const * eventTag);
- IWUResult * updateGlobalByName(const char * name);
- IWUGraph * createGraph(const char * name, WUGraphType type, IPropertyTree *xgmml);
- IWUGraph * updateGraph(const char * name);
- IWUQuery * updateQuery();
- IWUWebServicesInfo* updateWebServicesInfo(bool create);
- IWURoxieQueryInfo* updateRoxieQueryInfo(const char *wuid, const char *roxieClusterName);
- IWUPlugin * updatePluginByName(const char * name);
- IWULibrary * updateLibraryByName(const char * name);
- IWUResult * updateResultByName(const char * name);
- IWUResult * updateResultBySequence(unsigned seq);
- IWUResult * updateTemporaryByName(const char * name);
- IWUResult * updateVariableByName(const char * name);
- void addFile(const char *fileName, StringArray *clusters, unsigned usageCount, WUFileKind fileKind, const char *graphOwner);
- void noteFileRead(IDistributedFile *file);
- void releaseFile(const char *fileName);
- void clearGraphProgress();
- void resetBeforeGeneration();
- void deleteTempFiles(const char *graph, bool deleteOwned, bool deleteJobOwned);
- void deleteTemporaries();
- void addDiskUsageStats(__int64 avgNodeUsage, unsigned minNode, __int64 minNodeUsage, unsigned maxNode, __int64 maxNodeUsage, __int64 graphId);
- void setTimeScheduled(const IJlibDateTime &val);
- // ILocalWorkUnit - used for debugging etc
- void loadXML(const char *xml);
- void serialize(MemoryBuffer &tgt);
- void deserialize(MemoryBuffer &src);
- IWorkUnit &lockRemote(bool commit);
- void unlockRemote();
- void abort();
- void cleanupAndDelete(bool deldll,bool deleteOwned, const StringArray *deleteExclusions=NULL);
- bool switchThorQueue(const char *cluster, IQueueSwitcher *qs);
- void setAllowedClusters(const char *value);
- IStringVal & getAllowedClusters(IStringVal & str) const;
- void remoteCheckAccess(IUserDescriptor *user, bool writeaccess) const;
- void setAllowAutoQueueSwitch(bool val);
- bool getAllowAutoQueueSwitch() const;
- void setLibraryInformation(const char * name, unsigned interfaceHash, unsigned definitionHash);
- virtual void setResultInt(const char * name, unsigned sequence, __int64 val)
- {
- Owned<IWUResult> r = updateResult(name, sequence);
- if (r)
- {
- r->setResultInt(val);
- r->setResultStatus(ResultStatusCalculated);
- }
- }
- virtual void setResultUInt(const char * name, unsigned sequence, unsigned __int64 val)
- {
- Owned<IWUResult> r = updateResult(name, sequence);
- if (r)
- {
- r->setResultUInt(val);
- r->setResultStatus(ResultStatusCalculated);
- }
- }
- virtual void setResultReal(const char *name, unsigned sequence, double val)
- {
- Owned<IWUResult> r = updateResult(name, sequence);
- if (r)
- {
- r->setResultReal(val);
- r->setResultStatus(ResultStatusCalculated);
- }
- }
- virtual void setResultVarString(const char * stepname, unsigned sequence, const char *val)
- {
- setResultString(stepname, sequence, strlen(val), val);
- }
- virtual void setResultVarUnicode(const char * stepname, unsigned sequence, UChar const *val)
- {
- setResultUnicode(stepname, sequence, rtlUnicodeStrlen(val), val);
- }
- virtual void setResultString(const char * stepname, unsigned sequence, int len, const char *val)
- {
- doSetResultString(type_string, stepname, sequence, len, val);
- }
- virtual void setResultData(const char * stepname, unsigned sequence, int len, const void *val)
- {
- doSetResultString(type_data, stepname, sequence, len, (const char *)val);
- }
- virtual void setResultRaw(const char * name, unsigned sequence, int len, const void *val)
- {
- Owned<IWUResult> r = updateResult(name, sequence);
- if (r)
- {
- r->setResultRaw(len, val, ResultFormatRaw);
- r->setResultStatus(ResultStatusCalculated);
- }
- }
- virtual void setResultSet(const char * name, unsigned sequence, bool isAll, size32_t len, const void *val, ISetToXmlTransformer *)
- {
- Owned<IWUResult> r = updateResult(name, sequence);
- if (r)
- {
- r->setResultIsAll(isAll);
- r->setResultRaw(len, val, ResultFormatRaw);
- r->setResultStatus(ResultStatusCalculated);
- }
- }
- virtual void setResultUnicode(const char * name, unsigned sequence, int len, UChar const * val)
- {
- Owned<IWUResult> r = updateResult(name, sequence);
- if (r)
- {
- r->setResultUnicode((char const *)val, len);
- r->setResultStatus(ResultStatusCalculated);
- }
- }
- virtual void setResultBool(const char *name, unsigned sequence, bool val)
- {
- Owned<IWUResult> r = updateResult(name, sequence);
- if (r)
- {
- r->setResultBool(val);
- r->setResultStatus(ResultStatusCalculated);
- }
- }
- virtual void setResultDecimal(const char *name, unsigned sequence, int len, int precision, bool isSigned, const void *val)
- {
- Owned<IWUResult> r = updateResult(name, sequence);
- if (r)
- {
- r->setResultDecimal(val, len);
- r->setResultStatus(ResultStatusCalculated);
- }
- }
- virtual void setResultDataset(const char * name, unsigned sequence, size32_t len, const void *val, unsigned numRows, bool extend)
- {
- Owned<IWUResult> r = updateResult(name, sequence);
- if (r)
- {
- __int64 totalRows = numRows;
- if (extend)
- {
- totalRows += r->getResultRowCount();
- r->addResultRaw(len, val, ResultFormatRaw);
- }
- else
- r->setResultRaw(len, val, ResultFormatRaw);
- r->setResultStatus(ResultStatusCalculated);
- r->setResultRowCount(totalRows);
- r->setResultTotalRowCount(totalRows);
- }
- }
- protected:
- IWUResult *updateResult(const char *name, unsigned sequence)
- {
- Owned <IWUResult> result = updateWorkUnitResult(this, name, sequence);
- if (result)
- {
- SCMStringBuffer rname;
- if (!result->getResultName(rname).length())
- result->setResultName(name);
- }
- return result.getClear();
- }
- void doSetResultString(type_t type, const char *name, unsigned sequence, int len, const char *val)
- {
- Owned<IWUResult> r = updateResult(name, sequence);
- if (r)
- {
- r->setResultString(val, len);
- r->setResultStatus(ResultStatusCalculated);
- }
- }
- protected:
- void init();
- IWUGraph *createGraph();
- IWUResult *createResult();
- void loadGraphs() const;
- void loadResults() const;
- void loadTemporaries() const;
- void loadVariables() const;
- void loadExceptions() const;
- void loadPlugins() const;
- void loadLibraries() const;
- void loadClusters() const;
- void unsubscribe();
- void checkAgentRunning(WUState & state);
- void ensureGraphsUnpacked ()
- {
- IPropertyTree *t = p->queryPropTree("PackedGraphs");
- MemoryBuffer buf;
- if (t&&t->getPropBin(NULL,buf)) {
- cachedGraphs.clear();
- IPropertyTree *st = createPTree(buf);
- if (st) {
- p->setPropTree("Graphs",st);
- p->removeTree(t);
- }
- }
- }
- // Implemented by derived classes
- virtual void _lockRemote() {};
- virtual void _unlockRemote() {};
- };
- class CDaliWorkUnit : public CLocalWorkUnit
- {
- public:
- IMPLEMENT_IINTERFACE;
- CDaliWorkUnit(IRemoteConnection *_conn, ISecManager *secmgr, ISecUser *secuser)
- : connection(_conn), CLocalWorkUnit(_conn->getRoot(), secmgr, secuser)
- {
- abortDirty = true;
- abortState = false;
- }
- ~CDaliWorkUnit()
- {
- // NOTE - order is important - we need to construct connection before p and (especially) destroy after p
- // We use the beforeDIspose() in base class to help ensure this
- p.clear();
- }
- virtual void forceReload()
- {
- synchronized sync(locked); // protect locked workunits (uncommited writes) from reload
- StringBuffer wuRoot;
- getXPath(wuRoot, p->queryName());
- IRemoteConnection *newconn = querySDS().connect(wuRoot.str(), myProcessSession(), 0, SDS_LOCK_TIMEOUT);
- if (!newconn)
- throw MakeStringException(WUERR_ConnectFailed, "Could not connect to workunit %s (deleted?)",p->queryName());
- CriticalBlock block(crit);
- connection.setown(newconn);
- init();
- abortDirty = true;
- p.setown(connection->getRoot());
- }
- virtual void cleanupAndDelete(bool deldll, bool deleteOwned, const StringArray *deleteExclusions)
- {
- CLocalWorkUnit::cleanupAndDelete(deldll, deleteOwned, deleteExclusions);
- connection->close(true);
- PROGLOG("WUID %s removed",p->queryName());
- connection.clear();
- }
- virtual void commit()
- {
- CLocalWorkUnit::commit();
- if (connection)
- connection->commit();
- }
- virtual void _lockRemote()
- {
- StringBuffer wuRoot;
- getXPath(wuRoot, p->queryName());
- if (connection)
- connection->changeMode(RTM_LOCK_WRITE,SDS_LOCK_TIMEOUT);
- else
- connection.setown(querySDS().connect(wuRoot.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT));
- if (!connection)
- throw MakeStringException(WUERR_LockFailed, "Failed to get connection for xpath %s", wuRoot.str());
- init();
- abortDirty = true;
- p.setown(connection->getRoot());
- }
- virtual void _unlockRemote()
- {
- try
- {
- //MORE: I'm not convinced this is useful...
- setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTglobal, NULL, StWhenWorkunitModified, NULL, getTimeStampNowValue(), 1, 0, StatsMergeReplace);
- try
- {
- connection->commit();
- }
- catch (IException *e)
- {
- EXCLOG(e, "Error during workunit commit");
- connection->rollback();
- connection->changeMode(0, SDS_LOCK_TIMEOUT);
- throw;
- }
- connection->changeMode(0, SDS_LOCK_TIMEOUT);
- }
- catch (IException *E)
- {
- StringBuffer s;
- PrintLog("Failed to release write lock on workunit: %s", E->errorMessage(s).str());
- throw;
- }
- }
- virtual void subscribe(WUSubscribeOptions options)
- {
- CriticalBlock block(crit);
- assertex(options==SubscribeOptionAbort);
- if (!abortWatcher)
- {
- abortWatcher.setown(new CWorkUnitAbortWatcher(this, p->queryName()));
- abortDirty = true;
- }
- }
- virtual void unsubscribe()
- {
- CriticalBlock block(crit);
- if (abortWatcher)
- {
- abortWatcher->unsubscribe();
- abortWatcher.clear();
- }
- }
- virtual bool aborting() const
- {
- CriticalBlock block(crit);
- if (abortDirty)
- {
- StringBuffer apath;
- apath.append("/WorkUnitAborts/").append(p->queryName());
- Owned<IRemoteConnection> acon = querySDS().connect(apath.str(), myProcessSession(), 0, SDS_LOCK_TIMEOUT);
- if (acon)
- abortState = acon->queryRoot()->getPropInt(NULL) != 0;
- else
- abortState = false;
- abortDirty = false;
- }
- return abortState;
- }
- protected:
- class CWorkUnitAbortWatcher : public CInterface, implements ISDSSubscription
- {
- CDaliWorkUnit *parent; // not linked - it links me
- SubscriptionId abort;
- public:
- IMPLEMENT_IINTERFACE;
- CWorkUnitAbortWatcher(CDaliWorkUnit *_parent, const char *wuid) : parent(_parent)
- {
- StringBuffer wuRoot;
- wuRoot.append("/WorkUnitAborts/").append(wuid);
- abort = querySDS().subscribe(wuRoot.str(), *this);
- }
- ~CWorkUnitAbortWatcher()
- {
- assertex(abort==0);
- }
- void unsubscribe()
- {
- querySDS().unsubscribe(abort);
- abort = 0;
- }
- void notify(SubscriptionId id, const char *xpath, SDSNotifyFlags flags, unsigned valueLen, const void *valueData)
- {
- parent->abortDirty = true;
- }
- };
- Owned<IRemoteConnection> connection;
- Owned<CWorkUnitAbortWatcher> abortWatcher;
- mutable bool abortDirty;
- mutable bool abortState;
- };
- class CLockedWorkUnit : public CInterface, implements ILocalWorkUnit, implements IExtendedWUInterface
- {
- public:
- Owned<CLocalWorkUnit> c;
- IMPLEMENT_IINTERFACE;
- CLockedWorkUnit(CLocalWorkUnit *_c) : c(_c) {}
- ~CLockedWorkUnit()
- {
- if (workUnitTraceLevel > 1)
- PrintLog("Releasing locked workunit %s", queryWuid());
- if (c)
- c->unlockRemote();
- }
- void setSecIfcs(ISecManager *mgr, ISecUser*usr){c->setSecIfcs(mgr, usr);}
- virtual IConstWorkUnit * unlock()
- {
- c->unlockRemote();
- return c.getClear();
- }
- virtual bool aborting() const
- { return c->aborting(); }
- virtual void forceReload()
- { UNIMPLEMENTED; }
- virtual WUAction getAction() const
- { return c->getAction(); }
- virtual IStringVal& getActionEx(IStringVal & str) const
- { return c->getActionEx(str); }
- virtual IStringVal & getApplicationValue(const char * application, const char * propname, IStringVal & str) const
- { return c->getApplicationValue(application, propname, str); }
- virtual int getApplicationValueInt(const char * application, const char * propname, int defVal) const
- { return c->getApplicationValueInt(application, propname, defVal); }
- virtual IConstWUAppValueIterator & getApplicationValues() const
- { return c->getApplicationValues(); }
- virtual bool hasWorkflow() const
- { return c->hasWorkflow(); }
- virtual unsigned queryEventScheduledCount() const
- { return c->queryEventScheduledCount(); }
- virtual IPropertyTree * queryWorkflowTree() const
- { return c->queryWorkflowTree(); }
- virtual IConstWorkflowItemIterator * getWorkflowItems() const
- { return c->getWorkflowItems(); }
- virtual IWorkflowItemArray * getWorkflowClone() const
- { return c->getWorkflowClone(); }
- virtual bool requiresLocalFileUpload() const
- { return c->requiresLocalFileUpload(); }
- virtual IConstLocalFileUploadIterator * getLocalFileUploads() const
- { return c->getLocalFileUploads(); }
- virtual bool getIsQueryService() const
- { return c->getIsQueryService(); }
- virtual bool getCloneable() const
- { return c->getCloneable(); }
- virtual IUserDescriptor * queryUserDescriptor() const
- { return c->queryUserDescriptor(); }
- virtual const char *queryClusterName() const
- { return c->queryClusterName(); }
- virtual unsigned getCodeVersion() const
- { return c->getCodeVersion(); }
- virtual unsigned getWuidVersion() const
- { return c->getWuidVersion(); }
- virtual void getBuildVersion(IStringVal & buildVersion, IStringVal & eclVersion) const
- { c->getBuildVersion(buildVersion, eclVersion); }
- virtual bool hasDebugValue(const char * propname) const
- { return c->hasDebugValue(propname); }
- virtual IStringVal & getDebugValue(const char * propname, IStringVal & str) const
- { return c->getDebugValue(propname, str); }
- virtual int getDebugValueInt(const char * propname, int defVal) const
- { return c->getDebugValueInt(propname, defVal); }
- virtual __int64 getDebugValueInt64(const char * propname, __int64 defVal) const
- { return c->getDebugValueInt64(propname, defVal); }
- virtual bool getDebugValueBool(const char * propname, bool defVal) const
- { return c->getDebugValueBool(propname, defVal); }
- virtual IStringIterator & getDebugValues() const
- { return c->getDebugValues(NULL); }
- virtual IStringIterator & getDebugValues(const char *prop) const
- { return c->getDebugValues(prop); }
- virtual unsigned getExceptionCount() const
- { return c->getExceptionCount(); }
- virtual IConstWUExceptionIterator & getExceptions() const
- { return c->getExceptions(); }
- virtual unsigned getGraphCount() const
- { return c->getGraphCount(); }
- virtual unsigned getSourceFileCount() const
- { return c->getSourceFileCount(); }
- virtual unsigned getResultCount() const
- { return c->getResultCount(); }
- virtual unsigned getVariableCount() const
- { return c->getVariableCount(); }
- virtual unsigned getApplicationValueCount() const
- { return c->getApplicationValueCount(); }
- virtual IConstWUGraphIterator & getGraphs(WUGraphType type) const
- { return c->getGraphs(type); }
- virtual IConstWUGraphMetaIterator & getGraphsMeta(WUGraphType type) const
- { return c->getGraphsMeta(type); }
- virtual IConstWUGraph * getGraph(const char *name) const
- { return c->getGraph(name); }
- virtual IConstWUGraphProgress * getGraphProgress(const char * name) const
- { return c->getGraphProgress(name); }
- virtual const char *queryJobName() const
- { return c->queryJobName(); }
- virtual IConstWUPlugin * getPluginByName(const char * name) const
- { return c->getPluginByName(name); }
- virtual IConstWUPluginIterator & getPlugins() const
- { return c->getPlugins(); }
- virtual IConstWULibrary* getLibraryByName(const char *name) const
- { return c->getLibraryByName(name); }
- virtual IConstWULibraryIterator & getLibraries() const
- { return c->getLibraries(); }
- virtual WUPriorityClass getPriority() const
- { return c->getPriority(); }
- virtual int getPriorityLevel() const
- { return c->getPriorityLevel(); }
- virtual int getPriorityValue() const
- { return c->getPriorityValue(); }
- virtual IConstWUQuery * getQuery() const
- { return c->getQuery(); }
- virtual IConstWUWebServicesInfo * getWebServicesInfo() const
- { return c->getWebServicesInfo(); }
- virtual IConstWURoxieQueryInfo* getRoxieQueryInfo() const
- { return c->getRoxieQueryInfo(); }
- virtual bool getRescheduleFlag() const
- { return c->getRescheduleFlag(); }
- virtual IConstWUResult * getResultByName(const char * name) const
- { return c->getResultByName(name); }
- virtual IConstWUResult * getResultBySequence(unsigned seq) const
- { return c->getResultBySequence(seq); }
- virtual unsigned getResultLimit() const
- { return c->getResultLimit(); }
- virtual IConstWUResultIterator & getResults() const
- { return c->getResults(); }
- virtual IStringVal & getScope(IStringVal & str) const
- { return c->getScope(str); }
- virtual IStringVal & getSecurityToken(IStringVal & str) const
- { return c->getSecurityToken(str); }
- virtual WUState getState() const
- { return c->getState(); }
- virtual IStringVal & getStateEx(IStringVal & str) const
- { return c->getStateEx(str); }
- virtual __int64 getAgentSession() const
- { return c->getAgentSession(); }
- virtual unsigned getAgentPID() const
- { return c->getAgentPID(); }
- virtual const char *queryStateDesc() const
- { return c->queryStateDesc(); }
- virtual bool getRunningGraph(IStringVal & graphName, WUGraphIDType & subId) const
- { return c->getRunningGraph(graphName, subId); }
- virtual IConstWUStatisticIterator & getStatistics(const IStatisticsFilter * filter) const
- { return c->getStatistics(filter); }
- virtual IConstWUStatistic * getStatistic(const char * creator, const char * scope, StatisticKind kind) const
- { return c->getStatistic(creator, scope, kind); }
- virtual bool getWuDate(unsigned & year, unsigned & month, unsigned& day)
- { return c->getWuDate(year,month,day);}
- virtual IStringVal & getSnapshot(IStringVal & str) const
- { return c->getSnapshot(str); }
- virtual const char *queryUser() const
- { return c->queryUser(); }
- virtual IStringVal & getWuScope(IStringVal & str) const
- { return c->getWuScope(str); }
- virtual const char *queryWuid() const
- { return c->queryWuid(); }
- virtual IConstWUResult * getGlobalByName(const char * name) const
- { return c->getGlobalByName(name); }
- virtual IConstWUResult * getTemporaryByName(const char * name) const
- { return c->getTemporaryByName(name); }
- virtual IConstWUResultIterator & getTemporaries() const
- { return c->getTemporaries(); }
- virtual IConstWUResult * getVariableByName(const char * name) const
- { return c->getVariableByName(name); }
- virtual IConstWUResultIterator & getVariables() const
- { return c->getVariables(); }
- virtual bool isProtected() const
- { return c->isProtected(); }
- virtual bool isPausing() const
- { return c->isPausing(); }
- virtual IWorkUnit & lock()
- { ((CInterface *)this)->Link(); return (IWorkUnit &) *this; }
- virtual bool reload()
- { UNIMPLEMENTED; }
- virtual void subscribe(WUSubscribeOptions options)
- { c->subscribe(options); }
- virtual void requestAbort()
- { c->requestAbort(); }
- virtual unsigned calculateHash(unsigned prevHash)
- { return queryExtendedWU(c)->calculateHash(prevHash); }
- virtual void copyWorkUnit(IConstWorkUnit *cached, bool all)
- { queryExtendedWU(c)->copyWorkUnit(cached, all); }
- virtual bool archiveWorkUnit(const char *base,bool del,bool deldll,bool deleteOwned)
- { return queryExtendedWU(c)->archiveWorkUnit(base,del,deldll,deleteOwned); }
- virtual void packWorkUnit(bool pack)
- { queryExtendedWU(c)->packWorkUnit(pack); }
- virtual unsigned queryFileUsage(const char *filename) const
- { return c->queryFileUsage(filename); }
- virtual IJlibDateTime & getTimeScheduled(IJlibDateTime &val) const
- { return c->getTimeScheduled(val); }
- virtual unsigned getDebugAgentListenerPort() const
- { return c->getDebugAgentListenerPort(); }
- virtual IStringVal & getDebugAgentListenerIP(IStringVal &ip) const
- { return c->getDebugAgentListenerIP(ip); }
- virtual IStringVal & getXmlParams(IStringVal & params) const
- { return c->getXmlParams(params); }
- virtual const IPropertyTree *getXmlParams() const
- { return c->getXmlParams(); }
- virtual unsigned __int64 getHash() const
- { return c->getHash(); }
- virtual IStringIterator *getLogs(const char *type, const char *instance) const
- { return c->getLogs(type, instance); }
- virtual IStringIterator *getProcesses(const char *type) const
- { return c->getProcesses(type); }
- virtual IPropertyTreeIterator* getProcesses(const char *type, const char *instance) const
- { return c->getProcesses(type, instance); }
- virtual void clearExceptions()
- { c->clearExceptions(); }
- virtual void commit()
- { c->commit(); }
- virtual IWUException * createException()
- { return c->createException(); }
- virtual void addProcess(const char *type, const char *instance, unsigned pid, const char *log)
- { c->addProcess(type, instance, pid, log); }
- virtual void protect(bool protectMode)
- { c->protect(protectMode); }
- virtual void setAction(WUAction action)
- { c->setAction(action); }
- virtual void setApplicationValue(const char * application, const char * propname, const char * value, bool overwrite)
- { c->setApplicationValue(application, propname, value, overwrite); }
- virtual void setApplicationValueInt(const char * application, const char * propname, int value, bool overwrite)
- { c->setApplicationValueInt(application, propname, value, overwrite); }
- virtual void incEventScheduledCount()
- { c->incEventScheduledCount(); }
- virtual void setIsQueryService(bool value)
- { c->setIsQueryService(value); }
- virtual void setCloneable(bool value)
- { c->setCloneable(value); }
- virtual void setIsClone(bool value)
- { c->setIsClone(value); }
- virtual void setClusterName(const char * value)
- { c->setClusterName(value); }
- virtual void setCodeVersion(unsigned version, const char * buildVersion, const char * eclVersion)
- { c->setCodeVersion(version, buildVersion, eclVersion); }
- virtual void setDebugValue(const char * propname, const char * value, bool overwrite)
- { c->setDebugValue(propname, value, overwrite); }
- virtual void setDebugValueInt(const char * propname, int value, bool overwrite)
- { c->setDebugValueInt(propname, value, overwrite); }
- virtual void setJobName(const char * value)
- { c->setJobName(value); }
- virtual void setPriority(WUPriorityClass cls)
- { c->setPriority(cls); }
- virtual void setPriorityLevel(int level)
- { c->setPriorityLevel(level); }
- virtual void setRescheduleFlag(bool value)
- { c->setRescheduleFlag(value); }
- virtual void setResultLimit(unsigned value)
- { c->setResultLimit(value); }
- virtual void setSecurityToken(const char *value)
- { c->setSecurityToken(value); }
- virtual void setState(WUState state)
- { c->setState(state); }
- virtual void setStateEx(const char * text)
- { c->setStateEx(text); }
- virtual void setAgentSession(__int64 sessionId)
- { c->setAgentSession(sessionId); }
- virtual void setStatistic(StatisticCreatorType creatorType, const char * creator, StatisticScopeType scopeType, const char * scope, StatisticKind kind, const char * optDescription, unsigned __int64 value, unsigned __int64 count, unsigned __int64 maxValue, StatsMergeAction mergeAction)
- { c->setStatistic(creatorType, creator, scopeType, scope, kind, optDescription, value, count, maxValue, mergeAction); }
- virtual void setTracingValue(const char * propname, const char * value)
- { c->setTracingValue(propname, value); }
- virtual void setTracingValueInt(const char * propname, int value)
- { c->setTracingValueInt(propname, value); }
- virtual void setUser(const char * value)
- { c->setUser(value); }
- virtual void setWuScope(const char * value)
- {
- if (value && *value)
- {
- ISecManager *secmgr=c->querySecMgr();
- ISecUser *secusr=c->querySecUser();
- if (!secmgr || !secusr)
- throw MakeStringException(WUERR_SecurityNotAvailable, "Trying to change workunit scope without security interfaces available");
- if (checkWuScopeSecAccess(value, *secmgr, secusr, SecAccess_Write, "Change Scope", true, true))
- c->setWuScope(value);
- }
- }
- virtual IWorkflowItem* addWorkflowItem(unsigned wfid, WFType type, WFMode mode, unsigned success, unsigned failure, unsigned recovery, unsigned retriesAllowed, unsigned contingencyFor)
- { return c->addWorkflowItem(wfid, type, mode, success, failure, recovery, retriesAllowed, contingencyFor); }
- virtual void syncRuntimeWorkflow(IWorkflowItemArray * array)
- { c->syncRuntimeWorkflow(array); }
- virtual IWorkflowItemIterator * updateWorkflowItems()
- { return c->updateWorkflowItems(); }
- virtual void resetWorkflow()
- { c->resetWorkflow(); }
- virtual void schedule()
- { c->schedule(); }
- virtual void deschedule()
- { c->deschedule(); }
- virtual unsigned addLocalFileUpload(LocalFileUploadType type, char const * source, char const * destination, char const * eventTag)
- { return c->addLocalFileUpload(type, source, destination, eventTag); }
- virtual IWUResult * updateGlobalByName(const char * name)
- { return c->updateGlobalByName(name); }
- virtual IWUGraph * createGraph(const char * name, WUGraphType type, IPropertyTree *xgmml)
- { return c->createGraph(name, type, xgmml); }
- virtual IWUGraph * updateGraph(const char * name)
- { return c->updateGraph(name); }
- virtual IWUQuery * updateQuery()
- { return c->updateQuery(); }
- virtual IWUWebServicesInfo * updateWebServicesInfo(bool create)
- { return c->updateWebServicesInfo(create); }
- virtual IWURoxieQueryInfo * updateRoxieQueryInfo(const char *wuid, const char *roxieClusterName)
- { return c->updateRoxieQueryInfo(wuid, roxieClusterName); }
- virtual IWUPlugin * updatePluginByName(const char * name)
- { return c->updatePluginByName(name); }
- virtual IWULibrary * updateLibraryByName(const char * name)
- { return c->updateLibraryByName(name); }
- virtual IWUResult * updateResultByName(const char * name)
- { return c->updateResultByName(name); }
- virtual IWUResult * updateResultBySequence(unsigned seq)
- { return c->updateResultBySequence(seq); }
- virtual IWUResult * updateTemporaryByName(const char * name)
- { return c->updateTemporaryByName(name); }
- virtual IWUResult * updateVariableByName(const char * name)
- { return c->updateVariableByName(name); }
- virtual void addFile(const char *fileName, StringArray *clusters, unsigned usageCount, WUFileKind fileKind, const char *graphOwner)
- { c->addFile(fileName, clusters, usageCount, fileKind, graphOwner); }
- virtual void noteFileRead(IDistributedFile *file)
- { c->noteFileRead(file); }
- virtual void releaseFile(const char *fileName)
- { c->releaseFile(fileName); }
- virtual void clearGraphProgress()
- { c->clearGraphProgress(); }
- virtual void resetBeforeGeneration()
- { c->resetBeforeGeneration(); }
- virtual void deleteTempFiles(const char *graph, bool deleteOwned, bool deleteJobOwned)
- { c->deleteTempFiles(graph, deleteOwned, deleteJobOwned); }
- virtual void deleteTemporaries()
- { c->deleteTemporaries(); }
- virtual void addDiskUsageStats(__int64 avgNodeUsage, unsigned minNode, __int64 minNodeUsage, unsigned maxNode, __int64 maxNodeUsage, __int64 graphId)
- { c->addDiskUsageStats(avgNodeUsage, minNode, minNodeUsage, maxNode, maxNodeUsage, graphId); }
- virtual IPropertyTree * getDiskUsageStats()
- { return c->getDiskUsageStats(); }
- virtual IPropertyTreeIterator & getFileIterator() const
- { return c->getFileIterator(); }
- virtual IPropertyTreeIterator & getFilesReadIterator() const
- { return c->getFilesReadIterator(); }
- virtual void setSnapshot(const char * value)
- { c->setSnapshot(value); }
- virtual void setTimeScheduled(const IJlibDateTime &val)
- { c->setTimeScheduled(val); }
- virtual void setDebugAgentListenerPort(unsigned port)
- { c->setDebugAgentListenerPort(port); }
- virtual void setDebugAgentListenerIP(const char * ip)
- { c->setDebugAgentListenerIP(ip); }
- virtual void setXmlParams(const char *params)
- { c->setXmlParams(params); }
- virtual void setXmlParams(IPropertyTree *tree)
- { c->setXmlParams(tree); }
- virtual void setHash(unsigned __int64 hash)
- { c->setHash(hash); }
- // ILocalWorkUnit - used for debugging etc
- virtual void serialize(MemoryBuffer &tgt)
- { c->serialize(tgt); }
- virtual void deserialize(MemoryBuffer &src)
- { c->deserialize(src); }
- virtual bool switchThorQueue(const char *cluster, IQueueSwitcher *qs)
- { return c->switchThorQueue(cluster,qs); }
- virtual void setAllowedClusters(const char *value)
- { c->setAllowedClusters(value); }
- virtual IStringVal& getAllowedClusters(IStringVal &str) const
- { return c->getAllowedClusters(str); }
- virtual void remoteCheckAccess(IUserDescriptor *user, bool writeaccess) const
- { c->remoteCheckAccess(user,writeaccess); }
- virtual void setAllowAutoQueueSwitch(bool val)
- { c->setAllowAutoQueueSwitch(val); }
- virtual bool getAllowAutoQueueSwitch() const
- { return c->getAllowAutoQueueSwitch(); }
- virtual void setLibraryInformation(const char * name, unsigned interfaceHash, unsigned definitionHash)
- { c->setLibraryInformation(name, interfaceHash, definitionHash); }
- virtual void setResultInt(const char * name, unsigned sequence, __int64 val)
- { c->setResultInt(name, sequence, val); }
- virtual void setResultUInt(const char * name, unsigned sequence, unsigned __int64 val)
- { c->setResultUInt(name, sequence, val); }
- virtual void setResultReal(const char *name, unsigned sequence, double val)
- { c->setResultReal(name, sequence, val); }
- virtual void setResultVarString(const char * stepname, unsigned sequence, const char *val)
- { c->setResultVarString(stepname, sequence, val); }
- virtual void setResultVarUnicode(const char * stepname, unsigned sequence, UChar const *val)
- { c->setResultVarUnicode(stepname, sequence, val); }
- virtual void setResultString(const char * stepname, unsigned sequence, int len, const char *val)
- { c->setResultString(stepname, sequence, len, val); }
- virtual void setResultData(const char * stepname, unsigned sequence, int len, const void *val)
- { c->setResultData(stepname, sequence, len, val); }
- virtual void setResultRaw(const char * name, unsigned sequence, int len, const void *val)
- { c->setResultRaw(name, sequence, len, val); }
- virtual void setResultSet(const char * name, unsigned sequence, bool isAll, size32_t len, const void *val, ISetToXmlTransformer *xform)
- { c->setResultSet(name, sequence, isAll, len, val, xform); }
- virtual void setResultUnicode(const char * name, unsigned sequence, int len, UChar const * val)
- { c->setResultUnicode(name, sequence, len, val); }
- virtual void setResultBool(const char *name, unsigned sequence, bool val)
- { c->setResultBool(name, sequence, val); }
- virtual void setResultDecimal(const char *name, unsigned sequence, int len, int precision, bool isSigned, const void *val)
- { c->setResultDecimal(name, sequence, len, precision, isSigned, val); }
- virtual void setResultDataset(const char * name, unsigned sequence, size32_t len, const void *val, unsigned numRows, bool extend)
- { c->setResultDataset(name, sequence, len, val, numRows, extend); }
- };
- class CLocalWUAssociated : public CInterface, implements IConstWUAssociatedFile
- {
- Owned<IPropertyTree> p;
- public:
- IMPLEMENT_IINTERFACE;
- CLocalWUAssociated(IPropertyTree *p);
- virtual WUFileType getType() const;
- virtual IStringVal & getDescription(IStringVal & ret) const;
- virtual IStringVal & getIp(IStringVal & ret) const;
- virtual IStringVal & getName(IStringVal & ret) const;
- virtual IStringVal & getNameTail(IStringVal & ret) const;
- virtual unsigned getCrc() const;
- };
- class CLocalWUQuery : public CInterface, implements IWUQuery
- {
- Owned<IPropertyTree> p;
- mutable IArrayOf<IConstWUAssociatedFile> associated;
- mutable CriticalSection crit;
- mutable bool associatedCached;
- private:
- void addSpecialCaseAssociated(WUFileType type, const char * propname, unsigned crc) const;
- void loadAssociated() const;
- public:
- IMPLEMENT_IINTERFACE;
- CLocalWUQuery(IPropertyTree *p);
- virtual WUQueryType getQueryType() const;
- virtual IStringVal& getQueryText(IStringVal &str) const;
- virtual IStringVal& getQueryShortText(IStringVal &str) const;
- virtual IStringVal& getQueryName(IStringVal &str) const;
- virtual IStringVal & getQueryMainDefinition(IStringVal & str) const;
- virtual IStringVal& getQueryDllName(IStringVal &str) const;
- virtual unsigned getQueryDllCrc() const;
- virtual IStringVal& getQueryCppName(IStringVal &str) const;
- virtual IStringVal& getQueryResTxtName(IStringVal &str) const;
- virtual IConstWUAssociatedFile * getAssociatedFile(WUFileType type, unsigned index) const;
- virtual IConstWUAssociatedFileIterator& getAssociatedFiles() const;
- virtual void setQueryType(WUQueryType qt);
- virtual void setQueryText(const char *pstr);
- virtual void setQueryName(const char *);
- virtual void setQueryMainDefinition(const char * str);
- virtual void addAssociatedFile(WUFileType type, const char * name, const char * ip, const char * desc, unsigned crc);
- virtual void removeAssociatedFiles();
- };
- class CLocalWUWebServicesInfo : public CInterface, implements IWUWebServicesInfo
- {
- Owned<IPropertyTree> p;
- mutable CriticalSection crit;
- private:
-
- public:
- IMPLEMENT_IINTERFACE;
- CLocalWUWebServicesInfo(IPropertyTree *p);
- virtual IStringVal& getModuleName(IStringVal &str) const;
- virtual IStringVal& getAttributeName(IStringVal &str) const;
- virtual IStringVal& getDefaultName(IStringVal &str) const;
- virtual IStringVal& getInfo(const char *name, IStringVal &str) const;
- virtual unsigned getWebServicesCRC() const;
-
- virtual void setModuleName(const char *);
- virtual void setAttributeName(const char *);
- virtual void setDefaultName(const char *);
- virtual void setInfo(const char *name, const char *info);
- virtual void setWebServicesCRC(unsigned);
- };
- class CLocalWURoxieQueryInfo : public CInterface, implements IWURoxieQueryInfo
- {
- Owned<IPropertyTree> p;
- mutable CriticalSection crit;
- private:
-
- public:
- IMPLEMENT_IINTERFACE;
- CLocalWURoxieQueryInfo(IPropertyTree *p);
- virtual IStringVal& getQueryInfo(IStringVal &str) const;
- virtual IStringVal& getDefaultPackageInfo(IStringVal &str) const;
- virtual IStringVal& getRoxieClusterName(IStringVal &str) const;
- virtual IStringVal& getWuid(IStringVal &str) const;
- virtual void setQueryInfo(const char *info);
- virtual void setDefaultPackageInfo(const char *, int len);
- virtual void setRoxieClusterName(const char *str);
- virtual void setWuid(const char *str);
- };
- class CLocalWUResult : public CInterface, implements IWUResult
- {
- friend class CLocalWorkUnit;
- mutable CriticalSection crit;
- Owned<IPropertyTree> p;
- Owned<IProperties> xmlns;
- void getSchema(TypeInfoArray &types, StringAttrArray &names, IStringVal * ecl=NULL) const;
- public:
- IMPLEMENT_IINTERFACE;
- CLocalWUResult(IPropertyTree *props);
- ~CLocalWUResult() { try { p.clear(); } catch (IException *E) {E->Release();}}
- virtual WUResultStatus getResultStatus() const;
- virtual IStringVal& getResultName(IStringVal &str) const;
- virtual int getResultSequence() const;
- virtual bool isResultScalar() const;
- virtual IStringVal& getResultXml(IStringVal &str) const;
- virtual unsigned getResultFetchSize() const;
- virtual __int64 getResultTotalRowCount() const;
- virtual __int64 getResultRowCount() const;
- virtual void getResultDataset(IStringVal & ecl, IStringVal & defs) const;
- virtual IStringVal& getResultLogicalName(IStringVal &ecl) const;
- virtual IStringVal& getResultKeyField(IStringVal& ecl) const;
- virtual unsigned getResultRequestedRows() const;
- virtual __int64 getResultInt() const;
- virtual bool getResultBool() const;
- virtual double getResultReal() const;
- virtual IStringVal& getResultString(IStringVal & str) const;
- virtual IDataVal& getResultRaw(IDataVal & data, IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const;
- virtual IDataVal& getResultUnicode(IDataVal & data) const;
- virtual void getResultDecimal(void * val, unsigned length, unsigned precision, bool isSigned) const;
- virtual IStringVal& getResultEclSchema(IStringVal & str) const;
- virtual __int64 getResultRawSize(IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const;
- virtual IDataVal& getResultRaw(IDataVal & data, __int64 from, __int64 length, IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const;
- virtual IStringVal& getResultRecordSizeEntry(IStringVal & str) const;
- virtual IStringVal& getResultTransformerEntry(IStringVal & str) const;
- virtual __int64 getResultRowLimit() const;
- virtual IStringVal& getResultFilename(IStringVal & str) const;
- virtual WUResultFormat getResultFormat() const;
- virtual unsigned getResultHash() const;
- virtual bool getResultIsAll() const;
- virtual IProperties *queryResultXmlns();
- // interface IWUResult
- virtual void setResultStatus(WUResultStatus status);
- virtual void setResultName(const char *name);
- virtual void setResultSequence(unsigned seq);
- virtual void setResultSchemaRaw(unsigned len, const void *schema);
- virtual void setResultScalar(bool isScalar);
- virtual void setResultRaw(unsigned len, const void *xml, WUResultFormat format);
- virtual void setResultFetchSize(unsigned rows); // 0 means file-loaded
- virtual void setResultTotalRowCount(__int64 rows); // -1 means unknown
- virtual void setResultRowCount(__int64 rows);
- virtual void setResultDataset(const char *ecl, const char *defs);
- virtual void setResultLogicalName(const char *logicalName);
- virtual void setResultKeyField(const char * name);
- virtual void setResultRequestedRows(unsigned req);
- virtual void setResultRecordSizeEntry(const char * val);
- virtual void setResultTransformerEntry(const char * val);
- virtual void setResultInt(__int64 val);
- virtual void setResultReal(double val);
- virtual void setResultBool(bool val);
- virtual void setResultString(const char * val, unsigned length);
- virtual void setResultUnicode(const void * val, unsigned length);
- virtual void setResultData(const void * val, unsigned length);
- virtual void setResultDecimal(const void * val, unsigned length);
- virtual void addResultRaw(unsigned len, const void * data, WUResultFormat format);
- virtual void setResultRowLimit(__int64 value);
- virtual void setResultFilename(const char * name);
- virtual void setResultUInt(unsigned __int64 val);
- virtual void setResultIsAll(bool value);
- virtual void setResultFormat(WUResultFormat format);
- virtual void setResultXML(const char *val);
- virtual void setResultRow(unsigned len, const void * data);
- virtual void setResultXmlns(const char *prefix, const char *uri);
- };
- class CLocalWUPlugin : public CInterface, implements IWUPlugin
- {
- Owned<IPropertyTree> p;
- public:
- IMPLEMENT_IINTERFACE;
- CLocalWUPlugin(IPropertyTree *p);
- virtual IStringVal& getPluginName(IStringVal &str) const;
- virtual IStringVal& getPluginVersion(IStringVal &str) const;
- virtual void setPluginName(const char *str);
- virtual void setPluginVersion(const char *str);
- };
- class CLocalWULibrary : public CInterface, implements IWULibrary
- {
- Owned<IPropertyTree> p;
- public:
- IMPLEMENT_IINTERFACE;
- CLocalWULibrary(IPropertyTree *p);
- virtual IStringVal & getName(IStringVal & str) const;
- virtual void setName(const char * str);
- };
- class CLocalWUGraph : public CInterface, implements IWUGraph
- {
- const CLocalWorkUnit &owner;
- Owned<IPropertyTree> p;
- mutable Owned<IPropertyTree> graph; // cached copy of graph xgmml
- mutable Linked<IConstWUGraphProgress> progress;
- unsigned wuidVersion;
- void mergeProgress(IPropertyTree &tree, IPropertyTree &progressTree, const unsigned &progressV) const;
- public:
- IMPLEMENT_IINTERFACE;
- CLocalWUGraph(const CLocalWorkUnit &owner, IPropertyTree *p);
- virtual IStringVal & getXGMML(IStringVal & ret, bool mergeProgress) const;
- virtual IStringVal & getName(IStringVal & ret) const;
- virtual IStringVal & getLabel(IStringVal & ret) const;
- virtual IStringVal & getTypeName(IStringVal & ret) const;
- virtual WUGraphType getType() const;
- virtual WUGraphState getState() const;
- virtual IPropertyTree * getXGMMLTree(bool mergeProgress) const;
- virtual IPropertyTree * getXGMMLTreeRaw() const;
- virtual void setName(const char *str);
- virtual void setLabel(const char *str);
- virtual void setType(WUGraphType type);
- virtual void setXGMML(const char *str);
- virtual void setXGMMLTree(IPropertyTree * tree, bool compress=true);
- };
- class CLocalWUException : public CInterface, implements IWUException
- {
- Owned<IPropertyTree> p;
- public:
- IMPLEMENT_IINTERFACE;
- CLocalWUException(IPropertyTree *p);
- virtual IStringVal& getExceptionSource(IStringVal &str) const;
- virtual IStringVal& getExceptionMessage(IStringVal &str) const;
- virtual unsigned getExceptionCode() const;
- virtual WUExceptionSeverity getSeverity() const;
- virtual IStringVal & getTimeStamp(IStringVal & dt) const;
- virtual IStringVal & getExceptionFileName(IStringVal & str) const;
- virtual unsigned getExceptionLineNo() const;
- virtual unsigned getExceptionColumn() const;
- virtual void setExceptionSource(const char *str);
- virtual void setExceptionMessage(const char *str);
- virtual void setExceptionCode(unsigned code);
- virtual void setSeverity(WUExceptionSeverity level);
- virtual void setTimeStamp(const char * dt);
- virtual void setExceptionFileName(const char *str);
- virtual void setExceptionLineNo(unsigned r);
- virtual void setExceptionColumn(unsigned c);
- };
- //==========================================================================================
- extern WORKUNIT_API bool isSpecialResultSequence(unsigned sequence)
- {
- switch (sequence)
- {
- case ResultSequenceInternal:
- case ResultSequenceOnce:
- case ResultSequencePersist:
- case ResultSequenceStored:
- return true;
- default:
- assertex(sequence <= INT_MAX);
- return false;
- }
- }
- struct mapEnums { int val; const char *str; };
- const char *getEnumText(int value, mapEnums *map)
- {
- const char *defval = map->str;
- while (map->str)
- {
- if (value==map->val)
- return map->str;
- map++;
- }
- assertex(!"Unexpected value in setEnum");
- return defval;
- }
- void setEnum(IPropertyTree *p, const char *propname, int value, mapEnums *map)
- {
- const char *defval = map->str;
- while (map->str)
- {
- if (value==map->val)
- {
- p->setProp(propname, map->str);
- return;
- }
- map++;
- }
- assertex(!"Unexpected value in setEnum");
- p->setProp(propname, defval);
- }
- static int getEnum(const char *v, mapEnums *map)
- {
- if (v)
- {
- while (map->str)
- {
- if (stricmp(v, map->str)==0)
- return map->val;
- map++;
- }
- assertex(!"Unexpected value in getEnum");
- }
- return 0;
- }
- static int getEnum(const IPropertyTree *p, const char *propname, mapEnums *map)
- {
- return getEnum(p->queryProp(propname),map);
- }
- //==========================================================================================
- class CConstWUArrayIterator : public CInterface, implements IConstWorkUnitIterator
- {
- IArrayOf<IPropertyTree> trees;
- Owned<IConstWorkUnit> cur;
- unsigned curTreeNum;
- Linked<IWorkUnitFactory> factory;
- void setCurrent()
- {
- cur.setown(factory->openWorkUnit(trees.item(curTreeNum).queryName(), false));
- }
- public:
- IMPLEMENT_IINTERFACE;
- CConstWUArrayIterator(IWorkUnitFactory *_factory, IArrayOf<IPropertyTree> &_trees)
- : factory(_factory)
- {
- ForEachItemIn(t, _trees)
- trees.append(*LINK(&_trees.item(t)));
- curTreeNum = 0;
- }
- bool first()
- {
- curTreeNum = 0;
- return next();
- }
- bool isValid()
- {
- return (NULL != cur.get());
- }
- bool next()
- {
- if (curTreeNum >= trees.ordinality())
- {
- cur.clear();
- return false;
- }
- setCurrent();
- ++curTreeNum;
- return true;
- }
- IConstWorkUnit & query() { return *cur; }
- };
- //==========================================================================================
- class CStringArrayIterator : public CInterface, implements IStringIterator
- {
- StringArray strings;
- unsigned idx;
- public:
- IMPLEMENT_IINTERFACE;
- CStringArrayIterator() { idx = 0; };
- void append(const char *str) { strings.append(str); }
- virtual bool first() { idx = 0; return strings.isItem(idx); }
- virtual bool next() { idx ++; return strings.isItem(idx); }
- virtual bool isValid() { return strings.isItem(idx); }
- virtual IStringVal & str(IStringVal &s) { s.set(strings.item(idx)); return s; }
- };
- class CCachedJobNameIterator : public CInterface, implements IStringIterator
- {
- Owned<IPropertyTreeIterator> it;
- public:
- IMPLEMENT_IINTERFACE;
- CCachedJobNameIterator(IPropertyTreeIterator *p) : it(p) {};
- virtual bool first() { return it->first(); }
- virtual bool next() { return it->next(); }
- virtual bool isValid() { return it->isValid(); }
- virtual IStringVal & str(IStringVal &s) { s.set(it->query().queryName()+1); return s; }
- };
- class CEmptyStringIterator : public CInterface, implements IStringIterator
- {
- public:
- IMPLEMENT_IINTERFACE;
- virtual bool first() { return false; }
- virtual bool next() { return false; }
- virtual bool isValid() { return false; }
- virtual IStringVal & str(IStringVal &s) { s.clear(); return s; }
- };
- mapEnums workunitSortFields[] =
- {
- { WUSFuser, "@submitID" },
- { WUSFcluster, "@clusterName" },
- { WUSFjob, "@jobName" },
- { WUSFstate, "@state" },
- { WUSFpriority, "@priorityClass" },
- { WUSFprotected, "@protected" },
- { WUSFwuid, "@" },
- { WUSFecl, "Query/Text" },
- { WUSFfileread, "FilesRead/File/@name" },
- { WUSFroxiecluster, "RoxieQueryInfo/@roxieClusterName" },
- { WUSFtotalthortime, "Statistics/Statistic[@c='summary'][@creator='thor'][@kind='TimeElapsed']/@value|"
- "Statistics/Statistic[@c='summary'][@creator='hthor'][@kind='TimeElapsed']/@value|"
- "Statistics/Statistic[@c='summary'][@creator='roxie'][@kind='TimeElapsed']/@value|"
- "Statistics/Statistic[@desc='Total thor time']/@value|"
- "Timings/Timing[@name='Total thor time']/@duration" //Use Statistics first. If not found, use Timings
- },
- { WUSFterm, NULL }
- };
- mapEnums querySortFields[] =
- {
- { WUQSFId, "@id" },
- { WUQSFwuid, "@wuid" },
- { WUQSFname, "@name" },
- { WUQSFdll, "@dll" },
- { WUQSFmemoryLimit, "@memoryLimit" },
- { WUQSFmemoryLimitHi, "@memoryLimit" },
- { WUQSFtimeLimit, "@timeLimit" },
- { WUQSFtimeLimitHi, "@timeLimit" },
- { WUQSFwarnTimeLimit, "@warnTimeLimit" },
- { WUQSFwarnTimeLimitHi, "@warnTimeLimit" },
- { WUQSFpriority, "@priority" },
- { WUQSFpriorityHi, "@priority" },
- { WUQSFQuerySet, "../@id" },
- { WUQSFActivited, "@activated" },
- { WUQSFSuspendedByUser, "@suspended" },
- { WUQSFLibrary, "Library"},
- { WUQSFterm, NULL }
- };
- class asyncRemoveDllWorkItem: public CInterface, implements IWorkQueueItem // class only used in asyncRemoveDll
- {
- StringAttr name;
- public:
- IMPLEMENT_IINTERFACE;
- asyncRemoveDllWorkItem(const char * _name) : name(_name)
- {
- }
- void execute()
- {
- PROGLOG("WU removeDll %s", name.get());
- queryDllServer().removeDll(name, true, true); // <name>, removeDlls=true, removeDirectory=true
- }
- };
- class asyncRemoveRemoteFileWorkItem: public CInterface, implements IWorkQueueItem // class only used in asyncRemoveFile
- {
- RemoteFilename name;
- public:
- IMPLEMENT_IINTERFACE;
- asyncRemoveRemoteFileWorkItem(const char * _ip, const char * _name)
- {
- SocketEndpoint ep(_ip);
- name.setPath(ep, _name);
- }
- void execute()
- {
- Owned<IFile> file = createIFile(name);
- PROGLOG("WU removeDll %s",file->queryFilename());
- file->remove();
- }
- };
- //==========================================================================================
- class CConstQuerySetQueryIterator : public CInterface, implements IConstQuerySetQueryIterator
- {
- IArrayOf<IPropertyTree> trees;
- unsigned index;
- public:
- IMPLEMENT_IINTERFACE;
- CConstQuerySetQueryIterator(IArrayOf<IPropertyTree> &_trees)
- {
- ForEachItemIn(t, _trees)
- trees.append(*LINK(&_trees.item(t)));
- index = 0;
- }
- ~CConstQuerySetQueryIterator()
- {
- trees.kill();
- }
- bool first()
- {
- index = 0;
- return (trees.ordinality()!=0);
- }
- bool next()
- {
- index++;
- return (index<trees.ordinality());
- }
- bool isValid()
- {
- return (index<trees.ordinality());
- }
- IPropertyTree &query()
- {
- return trees.item(index);
- }
- };
- #define WUID_VERSION 2 // recorded in each wuid created, useful for bkwd compat. checks
- class CWorkUnitFactory : public CInterface, implements IWorkUnitFactory, implements IDaliClientShutdown
- {
- Owned<IWorkQueueThread> deletedllworkq;
- public:
- IMPLEMENT_IINTERFACE;
- CWorkUnitFactory()
- {
- // Assumes dali client configuration has already been done
- sdsManager = &querySDS();
- session = myProcessSession();
- deletedllworkq.setown(createWorkQueueThread());
- addShutdownHook(*this);
- }
- ~CWorkUnitFactory()
- {
- removeShutdownHook(*this);
- // deletepool->joinAll();
- }
- void clientShutdown();
- SessionId startSession()
- {
- // Temporary placeholder until startSession is implemented
- #ifdef _WIN32
- return GetTickCount();
- #else
- struct timeval tm;
- gettimeofday(&tm,NULL);
- return tm.tv_usec;
- #endif
- }
- IWorkUnit* ensureNamedWorkUnit(const char *name)
- {
- if (workUnitTraceLevel > 1)
- PrintLog("ensureNamedWorkUnit created %s", name);
- StringBuffer wuRoot;
- getXPath(wuRoot, name);
- IRemoteConnection* conn = sdsManager->connect(wuRoot.str(), session, RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
- conn->queryRoot()->setProp("@xmlns:xsi", "http://www.w3.org/1999/XMLSchema-instance");
- Owned<CLocalWorkUnit> cw = new CDaliWorkUnit(conn, (ISecManager *)NULL, NULL);
- return &cw->lockRemote(false);
- }
- virtual IWorkUnit* createNamedWorkUnit(const char *wuid, const char *app, const char *user)
- {
- StringBuffer wuRoot;
- getXPath(wuRoot, wuid);
- IRemoteConnection *conn;
- if (queryDaliServerVersion().compare("2.0") >= 0)
- conn = sdsManager->connect(wuRoot.str(), session, RTM_LOCK_WRITE|RTM_CREATE_UNIQUE, SDS_LOCK_TIMEOUT);
- else
- conn = sdsManager->connect(wuRoot.str(), session, RTM_LOCK_WRITE|RTM_CREATE, SDS_LOCK_TIMEOUT);
- conn->queryRoot()->setProp("@xmlns:xsi", "http://www.w3.org/1999/XMLSchema-instance");
- conn->queryRoot()->setPropInt("@wuidVersion", WUID_VERSION);
- Owned<CLocalWorkUnit> cw = new CDaliWorkUnit(conn, (ISecManager*)NULL, NULL);
- IWorkUnit* ret = &cw->lockRemote(false);
- ret->setDebugValue("CREATED_BY", app, true);
- ret->setDebugValue("CREATED_FOR", user, true);
- if (user)
- cw->setWuScope(user);
- return ret;
- }
- virtual IWorkUnit* createWorkUnit(const char *app, const char *user)
- {
- StringBuffer wuid("W");
- char result[32];
- time_t ltime;
- time( <ime );
- tm *today = localtime( <ime ); // MORE - this is not threadsafe. But I probably don't care that much!
- strftime(result, sizeof(result), "%Y%m%d-%H%M%S", today);
- wuid.append(result);
- if (queryDaliServerVersion().compare("2.0") < 0)
- wuid.append('-').append(startSession());
- if (workUnitTraceLevel > 1)
- PrintLog("createWorkUnit created %s", wuid.str());
- IWorkUnit* ret = createNamedWorkUnit(wuid.str(), app, user);
- if (workUnitTraceLevel > 1)
- PrintLog("createWorkUnit created %s", ret->queryWuid());
- addTimeStamp(ret, SSTglobal, NULL, StWhenCreated);
- return ret;
- }
- bool secDeleteWorkUnit(const char * wuid, ISecManager *secmgr, ISecUser *secuser, bool raiseexceptions)
- {
- if (workUnitTraceLevel > 1)
- PrintLog("deleteWorkUnit %s", wuid);
- StringBuffer wuRoot;
- getXPath(wuRoot, wuid);
- IRemoteConnection *conn = sdsManager->connect(wuRoot.str(), session, RTM_LOCK_WRITE|RTM_LOCK_SUB, SDS_LOCK_TIMEOUT);
- if (!conn)
- {
- if (workUnitTraceLevel > 0)
- PrintLog("deleteWorkUnit %s not found", wuid);
- return false;
- }
- Owned<CLocalWorkUnit> cw = new CDaliWorkUnit(conn, secmgr, secuser); // takes ownership of conn
- if (secmgr && !checkWuSecAccess(*cw.get(), *secmgr, secuser, SecAccess_Full, "delete", true, true)) {
- if (raiseexceptions) {
- // perhaps raise exception here?
- }
- return false;
- }
- if (raiseexceptions) {
- try
- {
- cw->cleanupAndDelete(true,true);
- }
- catch (IException *E)
- {
- StringBuffer s;
- LOG(MCexception(E, MSGCLS_warning), E, s.append("Exception during deleteWorkUnit: ").append(wuid).str());
- E->Release();
- return false;
- }
- }
- else
- cw->cleanupAndDelete(true,true);
- removeWorkUnitFromAllQueues(wuid); //known active workunits wouldn't make it this far
- return true;
- }
- virtual bool deleteWorkUnitEx(const char * wuid)
- {
- return secDeleteWorkUnit(wuid,NULL,NULL,true);
- }
- virtual bool deleteWorkUnit(const char * wuid)
- {
- return secDeleteWorkUnit(wuid,NULL,NULL,false);
- }
- virtual IConstWorkUnitIterator * getWorkUnitsByOwner(const char * owner)
- {
- StringBuffer path("*");
- if (owner && *owner)
- path.append("[@submitID=\"").append(owner).append("\"]");
- return getWorkUnitsByXPath(path.str());
- }
- virtual IConstWorkUnitIterator * getWorkUnitsByState(WUState state);
- virtual IConstWorkUnitIterator * getWorkUnitsByECL(const char* ecl)
- {
- StringBuffer path("*");
- if (ecl && *ecl)
- path.append("[Query/Text=~\"*").append(ecl).append("*\"]");
- return getWorkUnitsByXPath(path.str());
- }
- virtual IConstWorkUnitIterator * getWorkUnitsByCluster(const char* cluster)
- {
- StringBuffer path("*");
- if (cluster && *cluster)
- path.append("[@clusterName=\"").append(cluster).append("\"]");
- return getWorkUnitsByXPath(path.str());
- }
- virtual IConstWorkUnitIterator * getChildWorkUnits(const char *parent)
- {
- StringBuffer path("*[@parent=\"");
- path.append(parent).append("\"]");
- return getWorkUnitsByXPath(path.str());
- }
- virtual IConstWorkUnit* secOpenWorkUnit(const char *wuid, bool lock, ISecManager *secmgr=NULL, ISecUser *secuser=NULL)
- {
- StringBuffer wuidStr(wuid);
- wuidStr.trim();
- if (wuidStr.length() && ('w' == wuidStr.charAt(0)))
- wuidStr.setCharAt(0, 'W');
- if (!wuidStr.length() || ('W' != wuidStr.charAt(0)))
- {
- if (workUnitTraceLevel > 1)
- PrintLog("openWorkUnit %s invalid WUID", nullText(wuidStr.str()));
- return NULL;
- }
- if (workUnitTraceLevel > 1)
- PrintLog("openWorkUnit %s", wuidStr.str());
- StringBuffer wuRoot;
- getXPath(wuRoot, wuidStr.str());
- IRemoteConnection* conn = sdsManager->connect(wuRoot.str(), session, lock ? RTM_LOCK_READ|RTM_LOCK_SUB : 0, SDS_LOCK_TIMEOUT);
- if (conn)
- {
- CLocalWorkUnit *wu = new CDaliWorkUnit(conn, secmgr, secuser);
- if (secmgr && wu)
- {
- if (!checkWuSecAccess(*wu, *secmgr, secuser, SecAccess_Read, "opening", true, true))
- {
- delete wu;
- return NULL;
- }
- }
- return wu;
- }
- else
- {
- if (workUnitTraceLevel > 0)
- PrintLog("openWorkUnit %s not found", wuidStr.str());
- return NULL;
- }
- }
- virtual IConstWorkUnit* openWorkUnit(const char *wuid, bool lock)
- {
- return secOpenWorkUnit(wuid, lock);
- }
- virtual IWorkUnit* secUpdateWorkUnit(const char *wuid, ISecManager *secmgr=NULL, ISecUser *secuser=NULL)
- {
- if (workUnitTraceLevel > 1)
- PrintLog("updateWorkUnit %s", wuid);
- StringBuffer wuRoot;
- getXPath(wuRoot, wuid);
- IRemoteConnection* conn = sdsManager->connect(wuRoot.str(), session, RTM_LOCK_WRITE|RTM_LOCK_SUB, SDS_LOCK_TIMEOUT);
- if (conn)
- {
- Owned<CLocalWorkUnit> cw = new CDaliWorkUnit(conn, secmgr, secuser);
- if (secmgr && cw)
- {
- if (!checkWuSecAccess(*cw.get(), *secmgr, secuser, SecAccess_Write, "updating", true, true))
- return NULL;
- }
- return &cw->lockRemote(false);
- }
- else
- {
- if (workUnitTraceLevel > 0)
- PrintLog("updateWorkUnit %s not found", wuid);
- return NULL;
- }
- }
- virtual IWorkUnit* updateWorkUnit(const char *wuid)
- {
- return secUpdateWorkUnit(wuid);
- }
- virtual int setTracingLevel(int newLevel)
- {
- if (newLevel)
- PrintLog("Setting workunit trace level to %d", newLevel);
- int level = workUnitTraceLevel;
- workUnitTraceLevel = newLevel;
- return level;
- }
- IConstWorkUnitIterator * getWorkUnitsByXPath(const char *xpath)
- {
- return getWorkUnitsByXPath(xpath,NULL,NULL);
- }
- void descheduleAllWorkUnits()
- {
- Owned<IRemoteConnection> conn = querySDS().connect("/Schedule", myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
- if(!conn) return;
- Owned<IPropertyTree> root(conn->queryRoot()->getBranch("."));
- KeptAtomTable entries;
- Owned<IPropertyTreeIterator> iter(root->getElements("*/*/*/*"));
- StringBuffer wuid;
- for(iter->first(); iter->isValid(); iter->next())
- {
- char const * entry = iter->query().queryName();
- if(!entries.find(entry))
- {
- entries.addAtom(entry);
- ncnameUnescape(entry, wuid.clear());
- Owned<IWorkUnit> wu = updateWorkUnit(wuid);
- if(wu && (wu->getState() == WUStateWait))
- wu->setState(WUStateCompleted);
- }
- }
- bool more;
- do more = root->removeProp("*"); while(more);
- }
- IConstWorkUnitIterator * getWorkUnitsByXPath(const char *xpath, ISecManager *secmgr, ISecUser *secuser)
- {
- Owned<IRemoteConnection> conn = sdsManager->connect("/WorkUnits", session, 0, SDS_LOCK_TIMEOUT);
- if (conn)
- {
- CDaliVersion serverVersionNeeded("3.2");
- Owned<IPropertyTreeIterator> iter(queryDaliServerVersion().compare(serverVersionNeeded) < 0 ?
- conn->queryRoot()->getElements(xpath) :
- conn->getElements(xpath));
- return new CConstWUIterator(this, iter, secmgr, secuser);
- }
- else
- return NULL;
- }
- IConstWorkUnitIterator* getWorkUnitsSorted( WUSortField *sortorder, // list of fields to sort by (terminated by WUSFterm)
- WUSortField *filters, // NULL or list of fields to folteron (terminated by WUSFterm)
- const void *filterbuf, // (appended) string values for filters
- unsigned startoffset,
- unsigned maxnum,
- const char *queryowner,
- __int64 *cachehint,
- ISecManager *secmgr,
- ISecUser *secuser,
- unsigned *total)
- {
- class CWorkUnitsPager : public CSimpleInterface, implements IElementsPager
- {
- StringAttr xPath;
- StringAttr sortOrder;
- StringAttr nameFilterLo;
- StringAttr nameFilterHi;
- StringArray unknownAttributes;
- public:
- IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
- CWorkUnitsPager(const char* _xPath, const char *_sortOrder, const char* _nameFilterLo, const char* _nameFilterHi, StringArray& _unknownAttributes)
- : xPath(_xPath), sortOrder(_sortOrder), nameFilterLo(_nameFilterLo), nameFilterHi(_nameFilterHi)
- {
- ForEachItemIn(x, _unknownAttributes)
- unknownAttributes.append(_unknownAttributes.item(x));
- }
- virtual IRemoteConnection* getElements(IArrayOf<IPropertyTree> &elements)
- {
- Owned<IRemoteConnection> conn = querySDS().connect("WorkUnits", myProcessSession(), 0, SDS_LOCK_TIMEOUT);
- if (!conn)
- return NULL;
- Owned<IPropertyTreeIterator> iter = conn->getElements(xPath);
- if (!iter)
- return NULL;
- sortElements(iter, sortOrder.get(), nameFilterLo.get(), nameFilterHi.get(), unknownAttributes, elements);
- return conn.getClear();
- }
- };
- class CScopeChecker : public CSimpleInterface, implements ISortedElementsTreeFilter
- {
- UniqueScopes done;
- ISecManager *secmgr;
- ISecUser *secuser;
- CriticalSection crit;
- public:
- IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
- CScopeChecker(ISecManager *_secmgr,ISecUser *_secuser)
- {
- secmgr = _secmgr;
- secuser = _secuser;
- }
- bool isOK(IPropertyTree &tree)
- {
- const char *scopename = tree.queryProp("@scope");
- if (!scopename||!*scopename)
- return true;
- {
- CriticalBlock block(crit);
- const bool *b = done.getValue(scopename);
- if (b)
- return *b;
- }
- bool ret = checkWuScopeSecAccess(scopename,*secmgr,secuser,SecAccess_Read,"iterating",false,false);
- {
- // conceivably could have already been checked and added, but ok.
- CriticalBlock block(crit);
- done.setValue(scopename,ret);
- }
- return ret;
- }
- };
- Owned<ISortedElementsTreeFilter> sc = new CScopeChecker(secmgr,secuser);
- StringBuffer query;
- StringBuffer so;
- StringAttr namefilter("*");
- StringAttr namefilterlo;
- StringAttr namefilterhi;
- StringArray unknownAttributes;
- if (filters) {
- const char *fv = (const char *)filterbuf;
- for (unsigned i=0;filters[i]!=WUSFterm;i++) {
- int fmt = filters[i];
- int subfmt = (fmt&0xff);
- if (subfmt==WUSFwuid)
- namefilterlo.set(fv);
- else if (subfmt==WUSFwuidhigh)
- namefilterhi.set(fv);
- else if (subfmt==WUSFwildwuid)
- namefilter.set(fv);
- else if (subfmt==WUSFcustom)
- query.append("[").append(fv).append("]");
- else if (!fv || !*fv)
- unknownAttributes.append(getEnumText(subfmt,workunitSortFields));
- else {
- query.append('[').append(getEnumText(subfmt,workunitSortFields)).append('=');
- if (fmt&WUSFnocase)
- query.append('?');
- if (fmt&WUSFwild)
- query.append('~');
- query.append('"').append(fv).append("\"]");
- }
- fv = fv + strlen(fv)+1;
- }
- }
- query.insert(0, namefilter.get());
- if (sortorder) {
- for (unsigned i=0;sortorder[i]!=WUSFterm;i++) {
- if (so.length())
- so.append(',');
- int fmt = sortorder[i];
- if (fmt&WUSFreverse)
- so.append('-');
- if (fmt&WUSFnocase)
- so.append('?');
- if (fmt&WUSFnumeric)
- so.append('#');
- so.append(getEnumText(fmt&0xff,workunitSortFields));
- }
- }
- IArrayOf<IPropertyTree> results;
- Owned<IElementsPager> elementsPager = new CWorkUnitsPager(query.str(), so.length()?so.str():NULL, namefilterlo.get(), namefilterhi.get(), unknownAttributes);
- Owned<IRemoteConnection> conn=getElementsPaged(elementsPager,startoffset,maxnum,secmgr?sc:NULL,queryowner,cachehint,results,total);
- return new CConstWUArrayIterator(this, results);
- }
-
- IConstWorkUnitIterator* getWorkUnitsSorted( WUSortField *sortorder, // list of fields to sort by (terminated by WUSFterm)
- WUSortField *filters, // NULL or list of fields to filter on (terminated by WUSFterm)
- const void *filterbuf, // (appended) string values for filters
- unsigned startoffset,
- unsigned maxnum,
- const char *queryowner,
- __int64 *cachehint,
- unsigned *total)
- {
- return getWorkUnitsSorted(sortorder,filters,filterbuf,startoffset,maxnum,queryowner,cachehint, NULL, NULL, total);
- }
- IConstQuerySetQueryIterator* getQuerySetQueriesSorted( WUQuerySortField *sortorder, // list of fields to sort by (terminated by WUSFterm)
- WUQuerySortField *filters, // NULL or list of fields to filter on (terminated by WUSFterm)
- const void *filterbuf, // (appended) string values for filters
- unsigned startoffset,
- unsigned maxnum,
- __int64 *cachehint,
- unsigned *total,
- const MapStringTo<bool> *_subset)
- {
- struct PostFilters
- {
- WUQueryFilterBoolean activatedFilter;
- WUQueryFilterBoolean suspendedByUserFilter;
- PostFilters()
- {
- activatedFilter = WUQFSAll;
- suspendedByUserFilter = WUQFSAll;
- };
- } postFilters;
- class CQuerySetQueriesPager : public CSimpleInterface, implements IElementsPager
- {
- StringAttr querySet;
- StringAttr xPath;
- StringAttr sortOrder;
- PostFilters postFilters;
- StringArray unknownAttributes;
- const MapStringTo<bool> *subset;
- void populateQueryTree(const IPropertyTree* querySetTree, IPropertyTree* queryTree)
- {
- const char* querySetId = querySetTree->queryProp("@id");
- VStringBuffer path("Query%s", xPath.get());
- Owned<IPropertyTreeIterator> iter = querySetTree->getElements(path.str());
- ForEach(*iter)
- {
- IPropertyTree &query = iter->query();
- bool activated = false;
- const char* queryId = query.queryProp("@id");
- if (queryId && *queryId)
- {
- if (subset)
- {
- VStringBuffer match("%s/%s", querySetId, queryId);
- if (!subset->getValue(match))
- continue;
- }
- VStringBuffer aliasXPath("Alias[@id='%s']", queryId);
- IPropertyTree *alias = querySetTree->queryPropTree(aliasXPath.str());
- if (alias)
- activated = true;
- }
- if (activated && (postFilters.activatedFilter == WUQFSNo))
- continue;
- if (!activated && (postFilters.activatedFilter == WUQFSYes))
- continue;
- if ((postFilters.suspendedByUserFilter == WUQFSNo) && query.hasProp(getEnumText(WUQSFSuspendedByUser,querySortFields)))
- continue;
- if ((postFilters.suspendedByUserFilter == WUQFSYes) && !query.hasProp(getEnumText(WUQSFSuspendedByUser,querySortFields)))
- continue;
- IPropertyTree *queryWithSetId = queryTree->addPropTree("Query", createPTreeFromIPT(&query));
- queryWithSetId->setProp("@querySetId", querySetId);
- queryWithSetId->setPropBool("@activated", activated);
- }
- }
- IRemoteConnection* populateQueryTree(IPropertyTree* queryTree)
- {
- StringBuffer querySetXPath("QuerySets");
- if (!querySet.isEmpty())
- querySetXPath.appendf("/QuerySet[@id=\"%s\"]", querySet.get());
- Owned<IRemoteConnection> conn = querySDS().connect(querySetXPath.str(), myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT);
- if (!conn)
- return NULL;
- if (querySet.isEmpty())
- {
- Owned<IPropertyTreeIterator> querySetIter = conn->queryRoot()->getElements("*");
- ForEach(*querySetIter)
- populateQueryTree(&querySetIter->query(), queryTree);
- }
- else
- populateQueryTree(conn->queryRoot(), queryTree);
- return conn.getClear();
- }
- public:
- IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
- CQuerySetQueriesPager(const char* _querySet, const char* _xPath, const char *_sortOrder, PostFilters& _postFilters, StringArray& _unknownAttributes, const MapStringTo<bool> *_subset)
- : querySet(_querySet), xPath(_xPath), sortOrder(_sortOrder), subset(_subset)
- {
- postFilters.activatedFilter = _postFilters.activatedFilter;
- postFilters.suspendedByUserFilter = _postFilters.suspendedByUserFilter;
- ForEachItemIn(x, _unknownAttributes)
- unknownAttributes.append(_unknownAttributes.item(x));
- }
- virtual IRemoteConnection* getElements(IArrayOf<IPropertyTree> &elements)
- {
- Owned<IPropertyTree> elementTree = createPTree("Queries");
- Owned<IRemoteConnection> conn = populateQueryTree(elementTree);
- if (!conn)
- return NULL;
- Owned<IPropertyTreeIterator> iter = elementTree->getElements("*");
- if (!iter)
- return NULL;
- sortElements(iter, sortOrder.get(), NULL, NULL, unknownAttributes, elements);
- return conn.getClear();
- }
- };
- StringAttr querySet;
- StringBuffer xPath;
- StringBuffer so;
- StringArray unknownAttributes;
- if (filters)
- {
- const char *fv = (const char *)filterbuf;
- for (unsigned i=0;filters[i]!=WUQSFterm;i++) {
- int fmt = filters[i];
- int subfmt = (fmt&0xff);
- if (subfmt==WUQSFQuerySet)
- querySet.set(fv);
- else if ((subfmt==WUQSFmemoryLimit) || (subfmt==WUQSFtimeLimit) || (subfmt==WUQSFwarnTimeLimit) || (subfmt==WUQSFpriority))
- xPath.append('[').append(getEnumText(subfmt,querySortFields)).append(">=").append(fv).append("]");
- else if ((subfmt==WUQSFmemoryLimitHi) || (subfmt==WUQSFtimeLimitHi) || (subfmt==WUQSFwarnTimeLimitHi) || (subfmt==WUQSFpriorityHi))
- xPath.append('[').append(getEnumText(subfmt,querySortFields)).append("<=").append(fv).append("]");
- else if (subfmt==WUQSFActivited)
- postFilters.activatedFilter = (WUQueryFilterBoolean) atoi(fv);
- else if (subfmt==WUQSFSuspendedByUser)
- postFilters.suspendedByUserFilter = (WUQueryFilterBoolean) atoi(fv);
- else if (!fv || !*fv)
- unknownAttributes.append(getEnumText(subfmt,querySortFields));
- else {
- xPath.append('[').append(getEnumText(subfmt,querySortFields)).append('=');
- if (fmt&WUQSFnocase)
- xPath.append('?');
- if (fmt&WUQSFnumeric)
- xPath.append('#');
- if (fmt&WUQSFwild)
- xPath.append('~');
- xPath.append('"').append(fv).append("\"]");
- }
- fv = fv + strlen(fv)+1;
- }
- }
- if (sortorder) {
- for (unsigned i=0;sortorder[i]!=WUQSFterm;i++) {
- if (so.length())
- so.append(',');
- int fmt = sortorder[i];
- if (fmt&WUQSFreverse)
- so.append('-');
- if (fmt&WUQSFnocase)
- so.append('?');
- if (fmt&WUQSFnumeric)
- so.append('#');
- so.append(getEnumText(fmt&0xff,querySortFields));
- }
- }
- IArrayOf<IPropertyTree> results;
- Owned<IElementsPager> elementsPager = new CQuerySetQueriesPager(querySet.get(), xPath.str(), so.length()?so.str():NULL, postFilters, unknownAttributes, _subset);
- Owned<IRemoteConnection> conn=getElementsPaged(elementsPager,startoffset,maxnum,NULL,"",cachehint,results,total);
- return new CConstQuerySetQueryIterator(results);
- }
- virtual unsigned numWorkUnits()
- {
- Owned<IRemoteConnection> conn = sdsManager->connect("/WorkUnits", session, 0, SDS_LOCK_TIMEOUT);
- if (!conn)
- return 0;
- IPropertyTree *root = conn->queryRoot();
- return root->numChildren();
- }
- virtual unsigned numWorkUnitsFiltered(WUSortField *filters,
- const void *filterbuf,
- ISecManager *secmgr,
- ISecUser *secuser)
- {
- unsigned total;
- Owned<IConstWorkUnitIterator> iter = getWorkUnitsSorted( NULL,filters,filterbuf,0,0x7fffffff,NULL,NULL,secmgr,secuser,&total);
- return total;
- }
- virtual unsigned numWorkUnitsFiltered(WUSortField *filters,const void *filterbuf)
- {
- if (!filters)
- return numWorkUnits();
- return numWorkUnitsFiltered(filters,filterbuf,NULL,NULL);
- }
- void asyncRemoveDll(const char * name)
- {
- deletedllworkq->post(new asyncRemoveDllWorkItem(name));
- }
- void asyncRemoveFile(const char * ip, const char * name)
- {
- deletedllworkq->post(new asyncRemoveRemoteFileWorkItem(ip, name));
- }
- ISDSManager *sdsManager;
- SessionId session;
- ISecManager *secMgr;
- private:
- void deleteChildren(IPropertyTree *root, const char *wuid)
- {
- StringBuffer kids("*[@parent=\"");
- kids.append(wuid).append("\"]");
- Owned<IPropertyTreeIterator> it = root->getElements(kids.str());
- ForEach (*it)
- {
- deleteChildren(root, it->query().queryName());
- }
- root->removeProp(wuid);
- }
- class CConstWUIterator : public CInterface, implements IConstWorkUnitIterator
- {
- Owned<IConstWorkUnit> cur;
- Linked<IWorkUnitFactory> factory;
- Linked<IPropertyTreeIterator> ptreeIter;
- Owned<ISecResourceList> scopes;
- void setCurrent()
- {
- cur.setown(factory->openWorkUnit(ptreeIter->query().queryName(), false));
- }
- bool getNext() // scan for a workunit with permissions
- {
- if (!scopes)
- {
- setCurrent();
- return true;
- }
- do
- {
- const char *scopeName = ptreeIter->query().queryProp("@scope");
- if (!scopeName || !*scopeName || checkWuScopeListSecAccess(scopeName, scopes, SecAccess_Read, "iterating", false, false))
- {
- setCurrent();
- return true;
- }
- }
- while (ptreeIter->next());
- cur.clear();
- return false;
- }
- public:
- IMPLEMENT_IINTERFACE;
- CConstWUIterator(IWorkUnitFactory *_factory, IPropertyTreeIterator *_ptreeIter, ISecManager *secmgr=NULL, ISecUser *secuser=NULL)
- : factory(_factory), ptreeIter(_ptreeIter)
- {
- UniqueScopes us;
- if (secmgr /* && secmgr->authTypeRequired(RT_WORKUNIT_SCOPE) tbd */)
- {
- scopes.setown(secmgr->createResourceList("wuscopes"));
- ForEach(*ptreeIter)
- {
- const char *scopeName = ptreeIter->query().queryProp("@scope");
- if (scopeName && *scopeName && !us.getValue(scopeName))
- {
- scopes->addResource(scopeName);
- us.setValue(scopeName, true);
- }
- }
- if (scopes->count())
- secmgr->authorizeEx(RT_WORKUNIT_SCOPE, *secuser, scopes);
- else
- scopes.clear();
- }
- }
- bool first()
- {
- if (!ptreeIter->first())
- {
- cur.clear();
- return false;
- }
- return getNext();
- }
- bool isValid()
- {
- return (NULL != cur.get());
- }
- bool next()
- {
- if (!ptreeIter->next())
- {
- cur.clear();
- return false;
- }
- return getNext();
- }
- IConstWorkUnit & query() { return *cur; }
- };
- IRemoteConnection* connect(const char *xpath, unsigned flags)
- {
- return sdsManager->connect(xpath, session, flags, SDS_LOCK_TIMEOUT);
- }
- };
- static Owned<CWorkUnitFactory> factory;
- void CWorkUnitFactory::clientShutdown()
- {
- factory.clear();
- }
- void clientShutdownWorkUnit()
- {
- factory.clear();
- }
- extern WORKUNIT_API IWorkUnitFactory * getWorkUnitFactory()
- {
- if (!factory)
- factory.setown(new CWorkUnitFactory());
- return factory.getLink();
- }
- class CSecureWorkUnitFactory : public CInterface, implements IWorkUnitFactory
- {
- public:
- IMPLEMENT_IINTERFACE;
- CSecureWorkUnitFactory(ISecManager &secmgr, ISecUser &secuser)
- {
- if (!factory)
- factory.setown(new CWorkUnitFactory());
- secMgr.set(&secmgr);
- secUser.set(&secuser);
- }
- virtual IWorkUnit* createNamedWorkUnit(const char *wuid, const char *app, const char *user)
- {
- checkWuScopeSecAccess(user, *secMgr.get(), secUser.get(), SecAccess_Write, "Create", true, true);
- IWorkUnit *wu=factory->createNamedWorkUnit(wuid, app, user);
- if (wu)
- {
- CLockedWorkUnit* lw = dynamic_cast<CLockedWorkUnit*>(wu);
- if (lw)
- lw->setSecIfcs(secMgr.get(), secUser.get());
- }
- return wu;
- }
- virtual IWorkUnit* createWorkUnit(const char *app, const char *user)
- {
- checkWuScopeSecAccess(user, *secMgr.get(), secUser.get(), SecAccess_Write, "Create", true, true);
- IWorkUnit *wu=factory->createWorkUnit(app, user);
- if (wu)
- {
- CLockedWorkUnit* lw = dynamic_cast<CLockedWorkUnit*>(wu);
- if (lw)
- lw->setSecIfcs(secMgr.get(), secUser.get());
- }
- return wu;
- }
- virtual bool deleteWorkUnitEx(const char * wuid)
- {
- return factory->secDeleteWorkUnit(wuid, secMgr.get(), secUser.get(), true);
- }
- virtual bool deleteWorkUnit(const char * wuid)
- {
- return factory->secDeleteWorkUnit(wuid, secMgr.get(), secUser.get(), false);
- }
- virtual IConstWorkUnit* openWorkUnit(const char *wuid, bool lock)
- {
- return factory->secOpenWorkUnit(wuid, lock, secMgr.get(), secUser.get());
- }
- virtual IWorkUnit* updateWorkUnit(const char *wuid)
- {
- return factory->secUpdateWorkUnit(wuid, secMgr.get(), secUser.get());
- }
- //make cached workunits a non secure pass through for now.
- virtual IConstWorkUnitIterator * getWorkUnitsByOwner(const char * owner)
- {
- StringBuffer path("*");
- if (owner && *owner)
- path.append("[@submitID=\"").append(owner).append("\"]");
- return factory->getWorkUnitsByXPath(path.str(), secMgr.get(), secUser.get());
- }
- virtual IConstWorkUnitIterator * getWorkUnitsByState(WUState state);
- virtual IConstWorkUnitIterator * getWorkUnitsByECL(const char* ecl)
- {
- return factory->getWorkUnitsByECL(ecl);
- }
- virtual IConstWorkUnitIterator * getWorkUnitsByCluster(const char* cluster)
- {
- return factory->getWorkUnitsByCluster(cluster);
- }
- virtual IConstWorkUnitIterator * getWorkUnitsByXPath(const char * xpath)
- {
- return factory->getWorkUnitsByXPath(xpath, secMgr.get(), secUser.get());
- }
- virtual void descheduleAllWorkUnits()
- {
- factory->descheduleAllWorkUnits();
- }
- virtual IConstWorkUnitIterator * getChildWorkUnits(const char *parent)
- {
- StringBuffer path("*[@parent=\"");
- path.append(parent).append("\"]");
- return factory->getWorkUnitsByXPath(path.str(), secMgr.get(), secUser.get());
- }
- virtual int setTracingLevel(int newLevel)
- {
- return factory->setTracingLevel(newLevel);
- }
- virtual IConstWorkUnitIterator* getWorkUnitsSorted( WUSortField *sortorder, // list of fields to sort by (terminated by WUSFterm)
- WUSortField *filters, // NULL or list of fields to filter on (terminated by WUSFterm)
- const void *filterbuf, // (appended) string values for filters
- unsigned startoffset,
- unsigned maxnum,
- const char *queryowner,
- __int64 *cachehint,
- unsigned *total)
- {
- return factory->getWorkUnitsSorted(sortorder,filters,filterbuf,startoffset,maxnum,queryowner,cachehint, secMgr.get(), secUser.get(), total);
- }
- virtual IConstQuerySetQueryIterator* getQuerySetQueriesSorted( WUQuerySortField *sortorder,
- WUQuerySortField *filters,
- const void *filterbuf,
- unsigned startoffset,
- unsigned maxnum,
- __int64 *cachehint,
- unsigned *total,
- const MapStringTo<bool> *subset)
- {
- return factory->getQuerySetQueriesSorted(sortorder,filters,filterbuf,startoffset,maxnum,cachehint,total, subset);
- }
- virtual unsigned numWorkUnits()
- {
- return factory->numWorkUnits();
- }
- virtual unsigned numWorkUnitsFiltered(WUSortField *filters,
- const void *filterbuf)
- {
- return factory->numWorkUnitsFiltered(filters,filterbuf,secMgr.get(), secUser.get());
- }
- private:
- Owned<CWorkUnitFactory> base_factory;
- Owned<ISecManager> secMgr;
- Owned<ISecUser> secUser;
- };
- extern WORKUNIT_API IWorkUnitFactory * getSecWorkUnitFactory(ISecManager &secmgr, ISecUser &secuser)
- {
- return new CSecureWorkUnitFactory(secmgr, secuser);
- }
- extern WORKUNIT_API IWorkUnitFactory * getWorkUnitFactory(ISecManager *secmgr, ISecUser *secuser)
- {
- if (secmgr && secuser)
- return getSecWorkUnitFactory(*secmgr, *secuser);
- else
- return getWorkUnitFactory();
- }
- //==========================================================================================
- class CStringPTreeIterator : public CInterface, implements IStringIterator
- {
- Owned<IPropertyTreeIterator> it;
- public:
- IMPLEMENT_IINTERFACE;
- CStringPTreeIterator(IPropertyTreeIterator *p) : it(p) {};
- virtual bool first() { return it->first(); }
- virtual bool next() { return it->next(); }
- virtual bool isValid() { return it->isValid(); }
- virtual IStringVal & str(IStringVal &s) { s.set(it->query().queryProp(NULL)); return s; }
- };
- class CStringPTreeTagIterator : public CInterface, implements IStringIterator
- {
- Owned<IPropertyTreeIterator> it;
- public:
- IMPLEMENT_IINTERFACE;
- CStringPTreeTagIterator(IPropertyTreeIterator *p) : it(p) {};
- virtual bool first() { return it->first(); }
- virtual bool next() { return it->next(); }
- virtual bool isValid() { return it->isValid(); }
- virtual IStringVal & str(IStringVal &s) { s.set(it->query().queryName()); return s; }
- };
- class CStringPTreeAttrIterator : public CInterface, implements IStringIterator
- {
- Owned<IPropertyTreeIterator> it;
- StringAttr name;
- public:
- IMPLEMENT_IINTERFACE;
- CStringPTreeAttrIterator(IPropertyTreeIterator *p, const char *_name) : it(p), name(_name) {};
- virtual bool first() { return it->first(); }
- virtual bool next() { return it->next(); }
- virtual bool isValid() { return it->isValid(); }
- virtual IStringVal & str(IStringVal &s) { s.set(it->query().queryProp(name)); return s; }
- };
- //==========================================================================================
- CLocalWorkUnit::CLocalWorkUnit(IPropertyTree* root, ISecManager *secmgr, ISecUser *secuser)
- {
- p.setown(root);
- secMgr.set(secmgr);
- secUser.set(secuser);
- }
- void CLocalWorkUnit::init()
- {
- p.clear();
- cachedGraphs.clear();
- workflowIterator.clear();
- query.clear();
- graphs.kill();
- results.kill();
- variables.kill();
- plugins.kill();
- libraries.kill();
- exceptions.kill();
- temporaries.kill();
- roxieQueryInfo.clear();
- webServicesInfo.clear();
- workflowIteratorCached = false;
- resultsCached = false;
- temporariesCached = false;
- variablesCached = false;
- exceptionsCached = false;
- pluginsCached = false;
- librariesCached = false;
- activitiesCached = false;
- webServicesInfoCached = false;
- roxieQueryInfoCached = false;
- }
- // Dummy workunit support
- CLocalWorkUnit::CLocalWorkUnit(const char *_wuid, const char *xml, ISecManager *secmgr, ISecUser *secuser)
- {
- init();
- if (xml)
- p.setown(createPTreeFromXMLString(xml));
- else
- {
- p.setown(createPTree(_wuid));
- p->setProp("@xmlns:xsi", "http://www.w3.org/1999/XMLSchema-instance");
- }
- secMgr.set(secmgr);
- secUser.set(secuser);
- }
- void CLocalWorkUnit::beforeDispose()
- {
- try
- {
- unsubscribe();
- query.clear();
- webServicesInfo.clear();
- roxieQueryInfo.clear();
- workflowIterator.clear();
- plugins.kill();
- libraries.kill();
- exceptions.kill();
- graphs.kill();
- results.kill();
- temporaries.kill();
- variables.kill();
- appvalues.kill();
- statistics.kill();
- userDesc.clear();
- secMgr.clear();
- secUser.clear();
- cachedGraphs.clear();
- p.clear();
- }
- catch (IException *E) { LOG(MCexception(E, MSGCLS_warning), E, "Exception during ~CLocalWorkUnit"); E->Release(); }
- }
- void CLocalWorkUnit::cleanupAndDelete(bool deldll, bool deleteOwned, const StringArray *deleteExclusions)
- {
- TIME_SECTION("WUDELETE cleanupAndDelete total");
- // Delete any related things in SDS etc that might otherwise be forgotten
- if (p->getPropBool("@protected", false))
- throw MakeStringException(WUERR_WorkunitProtected, "%s: Workunit is protected",p->queryName());
- switch (getState())
- {
- case WUStateAborted:
- case WUStateCompleted:
- case WUStateFailed:
- case WUStateArchived:
- break;
- case WUStateCompiled:
- if (getAction()==WUActionRun || getAction()==WUActionUnknown)
- throw MakeStringException(WUERR_WorkunitActive, "%s: Workunit is active",p->queryName());
- break;
- case WUStateWait:
- throw MakeStringException(WUERR_WorkunitScheduled, "%s: Workunit is scheduled",p->queryName());
- default:
- throw MakeStringException(WUERR_WorkunitActive, "%s: Workunit is active",p->queryName());
- break;
- }
- if (getIsQueryService())
- {
- Owned<IPropertyTree> registry = getQueryRegistryRoot();
- if (registry)
- {
- VStringBuffer xpath("QuerySet/Query[@wuid='%s']", p->queryName());
- if (registry->hasProp(xpath.str()))
- throw MakeStringException(WUERR_WorkunitPublished, "%s: Workunit is published",p->queryName());
- }
- }
- try
- {
- if (deldll && !p->getPropBool("@isClone", false))
- {
- Owned<IConstWUQuery> q = getQuery();
- if (q)
- {
- Owned<IConstWUAssociatedFileIterator> iter = &q->getAssociatedFiles();
- SCMStringBuffer name;
- ForEach(*iter)
- {
- IConstWUAssociatedFile & cur = iter->query();
- cur.getNameTail(name);
- if (!deleteExclusions || (NotFound == deleteExclusions->find(name.str())))
- {
- Owned<IDllEntry> entry = queryDllServer().getEntry(name.str());
- if (entry.get())
- factory->asyncRemoveDll(name.str());
- else
- {
- SCMStringBuffer ip, localPath;
- cur.getName(localPath);
- cur.getIp(ip);
- factory->asyncRemoveFile(ip.str(), localPath.str());
- }
- }
- }
- }
- }
- StringBuffer apath;
- // PROGLOG("wuid dll files removed");
- {
- apath.append("/WorkUnitAborts/").append(p->queryName());
- Owned<IRemoteConnection> acon = factory->sdsManager->connect(apath.str(), factory->session, RTM_LOCK_WRITE | RTM_DELETE_ON_DISCONNECT, SDS_LOCK_TIMEOUT);
- acon.clear();
- }
- // PROGLOG("wuid WorkUnitAborts entry removed");
- deleteTempFiles(NULL, deleteOwned, true); // all, any remaining.
- }
- catch(IException *E)
- {
- StringBuffer s;
- LOG(MCexception(E, MSGCLS_warning), E, s.append("Exception during cleanupAndDelete: ").append(p->queryName()).str());
- E->Release();
- }
- catch (...)
- {
- WARNLOG("Unknown exception during cleanupAndDelete: %s", p->queryName());
- }
- CConstGraphProgress::deleteWuidProgress(p->queryName());
- }
- void CLocalWorkUnit::setTimeScheduled(const IJlibDateTime &val)
- {
- SCMStringBuffer strval;
- val.getGmtString(strval);
- p->setProp("@timescheduled",strval.str());
- }
- IJlibDateTime & CLocalWorkUnit::getTimeScheduled(IJlibDateTime &val) const
- {
- StringBuffer str;
- p->getProp("@timescheduled",str);
- if(str.length())
- val.setGmtString(str.str());
- return val;
- }
- bool modifyAndWriteWorkUnitXML(char const * wuid, StringBuffer & buf, StringBuffer & extra, IFileIO * fileio)
- {
- // kludge in extra chunks of XML such as GraphProgress and GeneratedDlls
- if(extra.length())
- {
- size32_t l = (size32_t)strlen(wuid);
- size32_t p = buf.length()-l-4; // bit of a kludge
- assertex(memcmp(buf.str()+p+2,wuid,l)==0);
- StringAttr tail(buf.str()+p);
- buf.setLength(p);
- buf.append(extra);
- buf.append(tail);
- }
- return (fileio->write(0,buf.length(),buf.str()) == buf.length());
- }
- bool CLocalWorkUnit::archiveWorkUnit(const char *base,bool del,bool ignoredllerrors,bool deleteOwned)
- {
- CriticalBlock block(crit);
- StringBuffer path(base);
- if (!p)
- return false;
- const char *wuid = p->queryName();
- if (!wuid||!*wuid)
- return false;
- addPathSepChar(path).append(wuid).append(".xml");
- Owned<IFile> file = createIFile(path.str());
- if (!file)
- return false;
- Owned<IFileIO> fileio = file->open(IFOcreate);
- if (!fileio)
- return false;
- StringBuffer buf;
- exportWorkUnitToXML(this, buf, false, false);
- StringBuffer extraWorkUnitXML;
- StringBuffer xpath("/GraphProgress/");
- xpath.append(wuid);
- Owned<IRemoteConnection> conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
- if (conn)
- {
- Owned<IPropertyTree> tmp = createPTree("GraphProgress");
- mergePTree(tmp,conn->queryRoot());
- toXML(tmp,extraWorkUnitXML,1,XML_Format);
- conn->close();
- }
- Owned<IConstWUQuery> q = getQuery();
- if (!q)
- {
- if (!modifyAndWriteWorkUnitXML(wuid, buf, extraWorkUnitXML, fileio))
- return false;
- if (del)
- {
- if (getState()==WUStateUnknown)
- setState(WUStateArchived); // to allow delete
- cleanupAndDelete(false,deleteOwned); // no query, may as well delete
- }
- return false;
- }
- StringArray deleteExclusions; // associated files not to delete, added if failure to copy
- Owned<IConstWUAssociatedFileIterator> iter = &q->getAssociatedFiles();
- Owned<IPropertyTree> generatedDlls = createPTree("GeneratedDlls");
- ForEach(*iter)
- {
- IConstWUAssociatedFile & cur = iter->query();
- SCMStringBuffer name;
- cur.getNameTail(name);
- if (name.length())
- {
- Owned<IDllEntry> entry = queryDllServer().getEntry(name.str());
- SCMStringBuffer curPath, curIp;
- cur.getName(curPath);
- cur.getIp(curIp);
- SocketEndpoint curEp(curIp.str());
- RemoteFilename curRfn;
- curRfn.setPath(curEp, curPath.str());
- StringBuffer dst(base);
- addPathSepChar(dst);
- curRfn.getTail(dst);
- Owned<IFile> dstFile = createIFile(dst.str());
- if (entry.get())
- {
- Owned<IException> exception;
- Owned<IDllLocation> loc;
- Owned<IPropertyTree> generatedDllBranch = createPTree();
- generatedDllBranch->setProp("@name", entry->queryName());
- generatedDllBranch->setProp("@kind", entry->queryKind());
- try
- {
- loc.setown(entry->getBestLocation()); //throws exception if no readable locations
- }
- catch(IException * e)
- {
- exception.setown(e);
- loc.setown(entry->getBestLocationCandidate()); //this will be closest of the unreadable locations
- }
- RemoteFilename filename;
- loc->getDllFilename(filename);
- if (!exception)
- {
- Owned<IFile> srcfile = createIFile(filename);
- try
- {
- if (dstFile->exists())
- {
- if (streq(srcfile->queryFilename(), dstFile->queryFilename()))
- deleteExclusions.append(name.str()); // restored workunit, referencing archive location for query dll (no longer true post HPCC-11191 fix)
- // still want to delete if already archived but there are source file copies
- }
- else
- copyFile(dstFile, srcfile);
- }
- catch(IException * e)
- {
- exception.setown(e);
- }
- }
- if (exception)
- {
- if (ignoredllerrors)
- {
- EXCLOG(exception.get(), "archiveWorkUnit (copying associated file)");
- //copy failed, so don't delete the registred dll files
- deleteExclusions.append(name.str());
- }
- else
- throw exception.getClear();
- }
- // Record Associated path to restore back to
- StringBuffer restorePath;
- curRfn.getRemotePath(restorePath);
- generatedDllBranch->setProp("@location", restorePath.str());
- generatedDlls->addPropTree("GeneratedDll", generatedDllBranch.getClear());
- }
- else // no generated dll entry
- {
- Owned<IFile> srcFile = createIFile(curRfn);
- try
- {
- copyFile(dstFile, srcFile);
- }
- catch (IException *e)
- {
- VStringBuffer msg("Failed to archive associated file '%s' to destination '%s'", srcFile->queryFilename(), dstFile->queryFilename());
- EXCLOG(e, msg.str());
- e->Release();
- deleteExclusions.append(name.str());
- }
- }
- }
- }
- iter.clear();
- if (generatedDlls->numChildren())
- toXML(generatedDlls, extraWorkUnitXML, 1, XML_Format);
- if (!modifyAndWriteWorkUnitXML(wuid, buf, extraWorkUnitXML, fileio))
- return false;
- if (del)
- {
- //setState(WUStateArchived); // this isn't useful as about to delete it!
- q.clear();
- cleanupAndDelete(true, deleteOwned, &deleteExclusions);
- }
- return true;
- }
- void CLocalWorkUnit::packWorkUnit(bool pack)
- {
- // only packs Graph info currently
- CriticalBlock block(crit);
- if (!p)
- return;
- const char *wuid = p->queryName();
- if (!wuid||!*wuid)
- return;
- if (pack) {
- if (!p->hasProp("PackedGraphs")) {
- cachedGraphs.clear();
- IPropertyTree *t = p->queryPropTree("Graphs");
- if (t) {
- MemoryBuffer buf;
- t->serialize(buf);
- p->setPropBin("PackedGraphs",buf.length(),buf.bufferBase());
- p->removeTree(t);
- }
- }
- }
- else {
- ensureGraphsUnpacked();
- }
- CConstGraphProgress::packProgress(wuid,pack);
- }
- IPropertyTree * pruneBranch(IPropertyTree * from, char const * xpath)
- {
- Owned<IPropertyTree> ret;
- IPropertyTree * branch = from->queryPropTree(xpath);
- if(branch) {
- ret.setown(createPTreeFromIPT(branch));
- from->removeTree(branch);
- }
- return ret.getClear();
- }
- bool restoreWorkUnit(const char *base,const char *wuid)
- {
- StringBuffer path(base);
- if (!wuid||!*wuid)
- return false;
- addPathSepChar(path).append(wuid).append(".xml");
- Owned<IFile> file = createIFile(path.str());
- if (!file)
- return false;
- Owned<IFileIO> fileio = file->open(IFOread);
- if (!fileio)
- return false;
- Owned<IPropertyTree> pt = createPTree(*fileio);
- if (!pt)
- return false;
- CDateTime dt;
- dt.setNow();
- StringBuffer dts;
- dt.getString(dts);
- pt->setProp("@restoredDate", dts.str());
- VStringBuffer xpath("/WorkUnits/%s", wuid);
- Owned<IRemoteConnection> conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
- if (!conn)
- {
- ERRLOG("restoreWorkUnit could not create to %s", xpath.str());
- return false;
- }
- IPropertyTree *root = conn->queryRoot();
- if (root->hasChildren())
- {
- ERRLOG("restoreWorkUnit WUID %s already exists", wuid);
- return false;
- }
- Owned<IPropertyTree> gprogress = pruneBranch(pt, "GraphProgress[1]");
- Owned<IPropertyTree> generatedDlls = pruneBranch(pt, "GeneratedDlls[1]");
- Owned<IPropertyTree> associatedFiles;
- IPropertyTree *srcAssociated = pt->queryPropTree("Query/Associated");
- if (srcAssociated)
- associatedFiles.setown(createPTreeFromIPT(srcAssociated));
- root->setPropTree(NULL, pt.getClear());
- conn.clear();
- // now kludge back GraphProgress and GeneratedDlls
- if (gprogress)
- {
- VStringBuffer xpath("/GraphProgress/%s", wuid);
- conn.setown(querySDS().connect(xpath, myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT));
- if (conn)
- {
- IPropertyTree *groot = conn->queryRoot();
- if (groot->hasChildren())
- WARNLOG("restoreWorkUnit WUID %s graphprogress already exists, replacing",wuid);
- groot->setPropTree(NULL, gprogress.getClear());
- }
- }
- if (generatedDlls)
- {
- Owned<IPropertyTreeIterator> dlls = generatedDlls->getElements("GeneratedDll");
- for(dlls->first(); dlls->isValid(); dlls->next())
- {
- IPropertyTree & dll = dlls->query();
- char const * name = dll.queryProp("@name");
- char const * kind = dll.queryProp("@kind");
- char const * location = dll.queryProp("@location");
- Owned<IDllEntry> got = queryDllServer().getEntry(name);
- if (!got)
- {
- RemoteFilename dstRfn;
- dstRfn.setRemotePath(location);
- StringBuffer srcPath(base);
- addPathSepChar(srcPath);
- dstRfn.getTail(srcPath);
- OwnedIFile srcFile = createIFile(srcPath);
- OwnedIFile dstFile = createIFile(dstRfn);
- copyFile(dstFile, srcFile);
- queryDllServer().registerDll(name, kind, location);
- }
- }
- }
- if (associatedFiles)
- {
- Owned<IPropertyTreeIterator> associated = associatedFiles->getElements("*");
- ForEach(*associated)
- {
- IPropertyTree &file = associated->query();
- const char *filename = file.queryProp("@filename");
- SocketEndpoint ep(file.queryProp("@ip"));
- RemoteFilename rfn;
- rfn.setPath(ep, filename);
- OwnedIFile dstFile = createIFile(rfn);
- StringBuffer srcPath(base), name;
- addPathSepChar(srcPath);
- rfn.getTail(name);
- srcPath.append(name);
- if (generatedDlls)
- {
- VStringBuffer gDllPath("GeneratedDll[@name=\"%s\"]", name.str());
- if (generatedDlls->hasProp(gDllPath))
- continue; // generated dlls handled separately - see above
- }
- OwnedIFile srcFile = createIFile(srcPath);
- if (srcFile->exists())
- {
- try
- {
- copyFile(dstFile, srcFile);
- }
- catch (IException *e)
- {
- VStringBuffer msg("Failed to restore associated file '%s' to destination '%s'", srcFile->queryFilename(), dstFile->queryFilename());
- EXCLOG(e, msg.str());
- e->Release();
- }
- }
- }
- }
- return true;
- }
- void CLocalWorkUnit::loadXML(const char *xml)
- {
- CriticalBlock block(crit);
- init();
- assertex(xml);
- p.setown(createPTreeFromXMLString(xml));
- }
- void CLocalWorkUnit::serialize(MemoryBuffer &tgt)
- {
- CriticalBlock block(crit);
- StringBuffer x;
- tgt.append(exportWorkUnitToXML(this, x, false, false).str());
- }
- void CLocalWorkUnit::deserialize(MemoryBuffer &src)
- {
- CriticalBlock block(crit);
- StringAttr value;
- src.read(value);
- loadXML(value);
- }
- void CLocalWorkUnit::requestAbort()
- {
- CriticalBlock block(crit);
- abortWorkUnit(p->queryName());
- }
- void CLocalWorkUnit::subscribe(WUSubscribeOptions options)
- {
- }
- void CLocalWorkUnit::unsubscribe()
- {
- }
- void CLocalWorkUnit::unlockRemote()
- {
- CriticalBlock block(crit);
- locked.unlock();
- if (IsShared()) // Is this right? Doesn't feel right! Commit on last unlock would seem smarter
- {
- _unlockRemote();
- }
- }
- IWorkUnit &CLocalWorkUnit::lockRemote(bool commit)
- {
- if (secMgr)
- checkWuSecAccess(*this, *secMgr.get(), secUser.get(), SecAccess_Write, "write lock", true, true);
- locked.lock();
- CriticalBlock block(crit);
- if (commit)
- {
- try
- {
- _lockRemote();
- }
- catch (IException *E)
- {
- StringBuffer s;
- PrintLog("Failed to get write lock on workunit: %s", E->errorMessage(s).str());
- locked.unlock();
- throw;
- }
- }
- return *new CLockedWorkUnit(LINK(this));
- }
- void CLocalWorkUnit::commit()
- {
- // Nothing to do if not backed by a persistent store
- }
- IWorkUnit& CLocalWorkUnit::lock()
- {
- return lockRemote(true);
- }
- const char *CLocalWorkUnit::queryWuid() const
- {
- CriticalBlock block(crit);
- return p->queryName();
- }
- unsigned CLocalWorkUnit::getDebugAgentListenerPort() const
- {
- CriticalBlock block(crit);
- return p->getPropInt("@DebugListenerPort", 0);
- }
- void CLocalWorkUnit::setDebugAgentListenerPort(unsigned port)
- {
- CriticalBlock block(crit);
- p->setPropInt("@DebugListenerPort", port);
- }
- IStringVal& CLocalWorkUnit::getDebugAgentListenerIP(IStringVal &ip) const
- {
- CriticalBlock block(crit);
- ip.set(p->queryProp("@DebugListenerIP"));
- return ip;
- }
- void CLocalWorkUnit::setDebugAgentListenerIP(const char * ip)
- {
- CriticalBlock block(crit);
- p->setProp("@DebugListenerIP", ip);
- }
- IStringVal& CLocalWorkUnit::getSecurityToken(IStringVal &str) const
- {
- CriticalBlock block(crit);
- str.set(p->queryProp("@token"));
- return str;
- }
- void CLocalWorkUnit::setSecurityToken(const char *value)
- {
- CriticalBlock block(crit);
- p->setProp("@token", value);
- }
- bool CLocalWorkUnit::getRunningGraph(IStringVal &graphName, WUGraphIDType &subId) const
- {
- return CConstGraphProgress::getRunningGraph(p->queryName(), graphName, subId);
- }
- void CLocalWorkUnit::setJobName(const char *value)
- {
- CriticalBlock block(crit);
- p->setProp("@jobName", value);
- }
- const char *CLocalWorkUnit::queryJobName() const
- {
- CriticalBlock block(crit);
- const char *ret = p->queryProp("@jobName");
- if (!ret)
- ret = "";
- return ret;
- }
- void CLocalWorkUnit::setClusterName(const char *value)
- {
- CriticalBlock block(crit);
- p->setProp("@clusterName", value);
- }
- const char *CLocalWorkUnit::queryClusterName() const
- {
- CriticalBlock block(crit);
- const char *ret = p->queryProp("@clusterName");
- if (!ret)
- ret = "";
- return ret;
- }
- void CLocalWorkUnit::setAllowedClusters(const char *value)
- {
- setDebugValue("allowedclusters",value, true);
- }
- IStringVal& CLocalWorkUnit::getAllowedClusters(IStringVal &str) const
- {
- CriticalBlock block(crit);
- getDebugValue("allowedclusters",str);
- if (str.length()!=0)
- return str;
- str.set(p->queryProp("@clusterName"));
- return str;
- }
- void CLocalWorkUnit::setAllowAutoQueueSwitch(bool val)
- {
- setDebugValueInt("allowautoqueueswitch",val?1:0,true);
- }
-
- bool CLocalWorkUnit::getAllowAutoQueueSwitch() const
- {
- CriticalBlock block(crit);
- return getDebugValueBool("allowautoqueueswitch",false);
- }
- void CLocalWorkUnit::setLibraryInformation(const char * name, unsigned interfaceHash, unsigned definitionHash)
- {
- StringBuffer suffix;
- if (name && *name)
- setApplicationValue("LibraryModule", "name", name, true);
- setApplicationValueInt("LibraryModule", "interfaceHash", interfaceHash, true);
- setApplicationValueInt("LibraryModule", "definitionHash", definitionHash, true);
- setApplicationValue("LibraryModule", "platform", appendLibrarySuffix(suffix).str(), true);
- }
- void CLocalWorkUnit::remoteCheckAccess(IUserDescriptor *user, bool writeaccess) const
- {
- unsigned auditflags = DALI_LDAP_AUDIT_REPORT|DALI_LDAP_READ_WANTED;
- if (writeaccess)
- auditflags |= DALI_LDAP_WRITE_WANTED;
- int perm = 255;
- const char *scopename = p->queryProp("@scope");
- if (scopename&&*scopename) {
- if (!user)
- user = queryUserDescriptor();
- perm = querySessionManager().getPermissionsLDAP("workunit",scopename,user,auditflags);
- if (perm<0) {
- if (perm==-1)
- perm = 255;
- else
- perm = 0;
- }
- }
- if (!HASREADPERMISSION(perm))
- throw MakeStringException(WUERR_WorkunitAccessDenied, "Read access denied for workunit %s", queryWuid());
- if (writeaccess && !HASWRITEPERMISSION(perm))
- throw MakeStringException(WUERR_WorkunitAccessDenied, "Write access denied for workunit %s", queryWuid());
- }
- void CLocalWorkUnit::setUser(const char * value)
- {
- CriticalBlock block(crit);
- p->setProp("@submitID", value);
- }
- const char *CLocalWorkUnit::queryUser() const
- {
- CriticalBlock block(crit);
- const char *ret = p->queryProp("@submitID");
- if (!ret)
- ret = "";
- return ret;
- }
- void CLocalWorkUnit::setWuScope(const char * value)
- {
- CriticalBlock block(crit);
- p->setProp("@scope", value);
- }
- IStringVal& CLocalWorkUnit::getWuScope(IStringVal &str) const
- {
- CriticalBlock block(crit);
- str.set(p->queryProp("@scope"));
- return str;
- }
- mapEnums priorityClasses[] = {
- { PriorityClassUnknown, "unknown" },
- { PriorityClassLow, "low" },
- { PriorityClassNormal, "normal" },
- { PriorityClassHigh, "high" },
- { PriorityClassSize, NULL },
- };
- void CLocalWorkUnit::setPriority(WUPriorityClass cls)
- {
- CriticalBlock block(crit);
- setEnum(p, "@priorityClass", cls, priorityClasses);
- }
- WUPriorityClass CLocalWorkUnit::getPriority() const
- {
- CriticalBlock block(crit);
- return (WUPriorityClass) getEnum(p, "@priorityClass", priorityClasses);
- }
- mapEnums states[] = {
- { WUStateUnknown, "unknown" },
- { WUStateCompiled, "compiled" },
- { WUStateRunning, "running" },
- { WUStateCompleted, "completed" },
- { WUStateFailed, "failed" },
- { WUStateArchived, "archived" },
- { WUStateAborting, "aborting" },
- { WUStateAborted, "aborted" },
- { WUStateBlocked, "blocked" },
- { WUStateSubmitted, "submitted" },
- { WUStateScheduled, "scheduled" },
- { WUStateCompiling, "compiling" },
- { WUStateWait, "wait" },
- { WUStateUploadingFiles, "uploading_files" },
- { WUStateDebugPaused, "debugging" },
- { WUStateDebugRunning, "debug_running" },
- { WUStatePaused, "paused" },
- { WUStateSize, NULL }
- };
- const char * getWorkunitStateStr(WUState state)
- {
- return states[state].str;
- }
- IConstWorkUnitIterator * CWorkUnitFactory::getWorkUnitsByState(WUState state)
- {
- StringBuffer path("*");
- path.append("[@state=\"").append(getEnumText(state, states)).append("\"]");
- return getWorkUnitsByXPath(path.str());
- }
- IConstWorkUnitIterator * CSecureWorkUnitFactory::getWorkUnitsByState(WUState state)
- {
- StringBuffer path("*");
- path.append("[@state=\"").append(getEnumText(state, states)).append("\"]");
- return factory->getWorkUnitsByXPath(path.str(), secMgr.get(), secUser.get());
- }
- void CLocalWorkUnit::setState(WUState value)
- {
- CriticalBlock block(crit);
- if (value==WUStateAborted || value==WUStatePaused || value==WUStateCompleted || value==WUStateFailed || value==WUStateSubmitted || value==WUStateWait)
- {
- unsubscribe();
- StringBuffer apath;
- apath.append("/WorkUnitAborts/").append(p->queryName());
- if(factory)
- {
- Owned<IRemoteConnection> acon = factory->sdsManager->connect(apath.str(), factory->session, RTM_LOCK_WRITE|RTM_LOCK_SUB, SDS_LOCK_TIMEOUT);
- if (acon)
- acon->close(true);
- }
- }
- setEnum(p, "@state", value, states);
- if (getDebugValueBool("monitorWorkunit", false))
- {
- switch(value)
- {
- case WUStateAborted:
- FLLOG(MCoperatorWarning, "Workunit %s aborted", p->queryName());
- break;
- case WUStateCompleted:
- FLLOG(MCoperatorProgress, "Workunit %s completed", p->queryName());
- break;
- case WUStateFailed:
- FLLOG(MCoperatorProgress, "Workunit %s failed", p->queryName());
- break;
- }
- }
- p->removeProp("@stateEx");
- }
- void CLocalWorkUnit::setStateEx(const char * text)
- {
- CriticalBlock block(crit);
- p->setProp("@stateEx", text);
- }
- void CLocalWorkUnit::setAgentSession(__int64 sessionId)
- {
- CriticalBlock block(crit);
- p->setPropInt64("@agentSession", sessionId);
- }
- bool CLocalWorkUnit::aborting() const
- {
- return false;
- }
- bool CLocalWorkUnit::getIsQueryService() const
- {
- CriticalBlock block(crit);
- return p->getPropBool("@isQueryService", false);
- }
- void CLocalWorkUnit::setIsQueryService(bool value)
- {
- CriticalBlock block(crit);
- p->setPropBool("@isQueryService", value);
- }
- void CLocalWorkUnit::checkAgentRunning(WUState & state)
- {
- if (queryDaliServerVersion().compare("2.1")<0)
- return;
- switch(state)
- {
- case WUStateRunning:
- case WUStateDebugPaused:
- case WUStateDebugRunning:
- case WUStateBlocked:
- case WUStateAborting:
- case WUStateCompiling:
- case WUStatePaused:
- {
- SessionId agent = getAgentSession();
- if((agent>0) && querySessionManager().sessionStopped(agent, 0))
- {
- forceReload();
- state = (WUState) getEnum(p, "@state", states);
- bool isecl=state==WUStateCompiling;
- if (aborting())
- state = WUStateAborted;
- else if (state==WUStateRunning || state==WUStatePaused || state==WUStateDebugPaused || state==WUStateDebugRunning || state==WUStateBlocked || state==WUStateCompiling)
- state = WUStateFailed;
- else
- return;
- WARNLOG("checkAgentRunning terminated: %"I64F"d state = %d",(__int64)agent,(int)state);
- Owned<IWorkUnit> w = &lock();
- w->setState(state);
- Owned<IWUException> e = w->createException();
- WUAction action = w->getAction();
- switch (action)
- {
- case WUActionPause:
- case WUActionPauseNow:
- case WUActionResume:
- w->setAction(WUActionUnknown);
- }
- if(isecl)
- {
- e->setExceptionCode(1001);
- e->setExceptionMessage("EclServer terminated unexpectedly");
- }
- else
- {
- e->setExceptionCode(1000);
- e->setExceptionMessage("Workunit terminated unexpectedly");
- }
- }
- }
- }
- }
- WUState CLocalWorkUnit::getState() const
- {
- CriticalBlock block(crit);
- WUState state = (WUState) getEnum(p, "@state", states);
- switch (state)
- {
- case WUStateRunning:
- case WUStateDebugPaused:
- case WUStateDebugRunning:
- case WUStateBlocked:
- case WUStateCompiling:
- if (aborting())
- state = WUStateAborting;
- break;
- case WUStateSubmitted:
- if (aborting())
- state = WUStateAborted;
- break;
- }
- const_cast<CLocalWorkUnit *>(this)->checkAgentRunning(state); //need const_cast as will change state if agent has died
- return state;
- }
- IStringVal& CLocalWorkUnit::getStateEx(IStringVal & str) const
- {
- CriticalBlock block(crit);
- str.set(p->queryProp("@stateEx"));
- return str;
- }
- __int64 CLocalWorkUnit::getAgentSession() const
- {
- CriticalBlock block(crit);
- return p->getPropInt64("@agentSession", -1);
- }
- unsigned CLocalWorkUnit::getAgentPID() const
- {
- CriticalBlock block(crit);
- return p->getPropInt("@agentPID", -1);
- }
- const char * CLocalWorkUnit::queryStateDesc() const
- {
- // MORE - not sure about this - may prefer a separate interface
- CriticalBlock block(crit);
- try
- {
- return getEnumText(getState(), states);
- }
- catch (...)
- {
- return "???";
- }
- }
- mapEnums actions[] = {
- { WUActionUnknown, "unknown" },
- { WUActionCompile, "compile" },
- { WUActionCheck, "check" },
- { WUActionRun, "run" },
- { WUActionExecuteExisting, "execute" },
- { WUActionPause, "pause" },
- { WUActionPauseNow, "pausenow" },
- { WUActionResume, "resume" },
- { WUActionSize, NULL },
- };
- void CLocalWorkUnit::setAction(WUAction value)
- {
- CriticalBlock block(crit);
- setEnum(p, "Action", value, actions);
- }
- WUAction CLocalWorkUnit::getAction() const
- {
- CriticalBlock block(crit);
- return (WUAction) getEnum(p, "Action", actions);
- }
- IStringVal& CLocalWorkUnit::getActionEx(IStringVal & str) const
- {
- CriticalBlock block(crit);
- str.set(p->queryProp("Action"));
- return str;
- }
- IStringVal& CLocalWorkUnit::getApplicationValue(const char *app, const char *propname, IStringVal &str) const
- {
- CriticalBlock block(crit);
- StringBuffer prop("Application/");
- prop.append(app).append('/').append(propname);
- str.set(p->queryProp(prop.str()));
- return str;
- }
- int CLocalWorkUnit::getApplicationValueInt(const char *app, const char *propname, int defVal) const
- {
- CriticalBlock block(crit);
- StringBuffer prop("Application/");
- prop.append(app).append('/').append(propname);
- return p->getPropInt(prop.str(), defVal);
- }
- IConstWUAppValueIterator& CLocalWorkUnit::getApplicationValues() const
- {
- CriticalBlock block(crit);
- appvalues.load(p,"Application/*");
- return *new CArrayIteratorOf<IConstWUAppValue,IConstWUAppValueIterator> (appvalues, 0, (IConstWorkUnit *) this);
- }
- void CLocalWorkUnit::setApplicationValue(const char *app, const char *propname, const char *value, bool overwrite)
- {
- CriticalBlock block(crit);
- StringBuffer prop("Application/");
- prop.append(app).append('/').append(propname);
- if (overwrite || !p->hasProp(prop.str()))
- {
- // MORE - not sure these lines should be needed....
- StringBuffer sp;
- p->setProp(sp.append("Application").str(), "");
- p->setProp(sp.append('/').append(app).str(), "");
- p->setProp(prop.str(), value);
- }
- }
- void CLocalWorkUnit::setApplicationValueInt(const char *app, const char *propname, int value, bool overwrite)
- {
- CriticalBlock block(crit);
- StringBuffer prop("Application/");
- prop.append(app).append('/').append(propname);
- if (overwrite || !p->hasProp(prop.str()))
- {
- // MORE - not sure these lines should be needed....
- StringBuffer sp;
- p->setProp(sp.append("Application").str(), "");
- p->setProp(sp.append('/').append(app).str(), "");
- p->setPropInt(prop.str(), value);
- }
- }
- void CLocalWorkUnit::setPriorityLevel(int level)
- {
- CriticalBlock block(crit);
- p->setPropInt("PriorityFlag", level);
- }
- int CLocalWorkUnit::getPriorityLevel() const
- {
- CriticalBlock block(crit);
- return p->getPropInt("PriorityFlag");
- }
- int calcPriorityValue(const IPropertyTree * p)
- {
- int priority = p->getPropInt("PriorityFlag");
- switch((WUPriorityClass) getEnum(p, "@priorityClass", priorityClasses))
- {
- case PriorityClassLow:
- priority -= 100;
- break;
- case PriorityClassHigh:
- priority += 100;
- break;
- }
- return priority;
- }
- int CLocalWorkUnit::getPriorityValue() const
- {
- CriticalBlock block(crit);
- return calcPriorityValue(p);
- }
- void CLocalWorkUnit::setRescheduleFlag(bool value)
- {
- CriticalBlock block(crit);
- p->setPropInt("RescheduleFlag", (int) value);
- }
- bool CLocalWorkUnit::getRescheduleFlag() const
- {
- CriticalBlock block(crit);
- return p->getPropInt("RescheduleFlag") != 0;
- }
- class NullIStringIterator : public CInterface, extends IStringIterator
- {
- public:
- IMPLEMENT_IINTERFACE;
- bool first() { return false; }
- bool next() { return false; }
- bool isValid() { return false; }
- IStringVal & str(IStringVal & str) { return str; }
- };
- ClusterType getClusterType(const char * platform, ClusterType dft)
- {
- if (stricmp(platform, "thor") == 0)
- return ThorLCRCluster;
- if (stricmp(platform, "thorlcr") == 0)
- return ThorLCRCluster;
- if (stricmp(platform, "hthor") == 0)
- return HThorCluster;
- if (stricmp(platform, "roxie") == 0)
- return RoxieCluster;
- return dft;
- }
- const char *clusterTypeString(ClusterType clusterType, bool lcrSensitive)
- {
- switch (clusterType)
- {
- case ThorLCRCluster:
- if (lcrSensitive)
- return "thorlcr";
- return "thor";
- case RoxieCluster:
- return "roxie";
- case HThorCluster:
- return "hthor";
- }
- throwUnexpected();
- }
- IPropertyTree *queryRoxieProcessTree(IPropertyTree *environment, const char *process)
- {
- if (!process || !*process)
- return NULL;
- VStringBuffer xpath("Software/RoxieCluster[@name=\"%s\"]", process);
- return environment->queryPropTree(xpath.str());
- }
- void getRoxieProcessServers(IPropertyTree *roxie, SocketEndpointArray &endpoints)
- {
- if (!roxie)
- return;
- Owned<IPropertyTreeIterator> servers = roxie->getElements("RoxieServerProcess");
- ForEach(*servers)
- {
- IPropertyTree &server = servers->query();
- const char *netAddress = server.queryProp("@netAddress");
- if (netAddress && *netAddress)
- {
- SocketEndpoint ep(netAddress, server.getPropInt("@port", 9876));
- endpoints.append(ep);
- }
- }
- }
- void getRoxieProcessServers(const char *process, SocketEndpointArray &servers)
- {
- Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
- Owned<IConstEnvironment> env = factory->openEnvironment();
- if (!env)
- return;
- Owned<IPropertyTree> root = &env->getPTree();
- getRoxieProcessServers(queryRoxieProcessTree(root, process), servers);
- }
- class CEnvironmentClusterInfo: public CInterface, implements IConstWUClusterInfo
- {
- StringAttr name;
- StringAttr serverQueue;
- StringAttr agentQueue;
- StringAttr roxieProcess;
- SocketEndpointArray roxieServers;
- StringAttr thorQueue;
- StringArray thorProcesses;
- StringArray primaryThorProcesses;
- StringAttr prefix;
- StringAttr ldapUser;
- StringBuffer ldapPassword;
- ClusterType platform;
- unsigned clusterWidth;
- public:
- IMPLEMENT_IINTERFACE;
- CEnvironmentClusterInfo(const char *_name, const char *_prefix, IPropertyTree *agent, IArrayOf<IPropertyTree> &thors, IPropertyTree *roxie)
- : name(_name), prefix(_prefix)
- {
- StringBuffer queue;
- if (thors.ordinality())
- {
- thorQueue.set(getClusterThorQueueName(queue.clear(), name));
- clusterWidth = 0;
- bool isMultiThor = (thors.length() > 1);
- ForEachItemIn(i,thors)
- {
- IPropertyTree &thor = thors.item(i);
- const char* thorName = thor.queryProp("@name");
- thorProcesses.append(thorName);
- if (!isMultiThor)
- primaryThorProcesses.append(thorName);
- else
- {
- const char *nodeGroup = thor.queryProp("@nodeGroup");
- if (!nodeGroup || strieq(nodeGroup, thorName))
- primaryThorProcesses.append(thorName);
- }
- unsigned nodes = thor.getCount("ThorSlaveProcess");
- if (!nodes)
- throw MakeStringException(WUERR_MismatchClusterSize,"CEnvironmentClusterInfo: Thor cluster can not have 0 slave processes");
- unsigned ts = nodes * thor.getPropInt("@slavesPerNode", 1);
- if (clusterWidth && (ts!=clusterWidth))
- throw MakeStringException(WUERR_MismatchClusterSize,"CEnvironmentClusterInfo: mismatched thor sizes in cluster");
- clusterWidth = ts;
- bool islcr = !thor.getPropBool("@Legacy");
- if (!islcr)
- throw MakeStringException(WUERR_MismatchThorType,"CEnvironmentClusterInfo: Legacy Thor no longer supported");
- }
- platform = ThorLCRCluster;
- }
- else if (roxie)
- {
- roxieProcess.set(roxie->queryProp("@name"));
- platform = RoxieCluster;
- getRoxieProcessServers(roxie, roxieServers);
- clusterWidth = roxieServers.length();
- ldapUser.set(roxie->queryProp("@ldapUser"));
- StringBuffer encPassword = roxie->queryProp("@ldapPassword");
- if (encPassword.length())
- decrypt(ldapPassword, encPassword);
- }
- else
- {
- clusterWidth = 1;
- platform = HThorCluster;
- }
- if (agent)
- {
- assertex(!roxie);
- agentQueue.set(getClusterEclAgentQueueName(queue.clear(), name));
- }
- else if (roxie)
- agentQueue.set(getClusterRoxieQueueName(queue.clear(), name));
- // MORE - does this need to be conditional?
- serverQueue.set(getClusterEclCCServerQueueName(queue.clear(), name));
- }
- IStringVal & getName(IStringVal & str) const
- {
- str.set(name.get());
- return str;
- }
- IStringVal & getScope(IStringVal & str) const
- {
- str.set(prefix.get());
- return str;
- }
- IStringVal & getAgentQueue(IStringVal & str) const
- {
- str.set(agentQueue);
- return str;
- }
- virtual IStringVal & getServerQueue(IStringVal & str) const
- {
- str.set(serverQueue);
- return str;
- }
- IStringVal & getThorQueue(IStringVal & str) const
- {
- str.set(thorQueue);
- return str;
- }
- unsigned getSize() const
- {
- return clusterWidth;
- }
- virtual ClusterType getPlatform() const
- {
- return platform;
- }
- IStringVal & getRoxieProcess(IStringVal & str) const
- {
- str.set(roxieProcess.get());
- return str;
- }
- const StringArray & getThorProcesses() const
- {
- return thorProcesses;
- }
- const StringArray & getPrimaryThorProcesses() const
- {
- return primaryThorProcesses;
- }
- const SocketEndpointArray & getRoxieServers() const
- {
- return roxieServers;
- }
- const char *getLdapUser() const
- {
- return ldapUser.get();
- }
- virtual const char *getLdapPassword() const
- {
- return ldapPassword.str();
- }
- };
- IStringVal &getProcessQueueNames(IStringVal &ret, const char *process, const char *type, const char *suffix)
- {
- if (process)
- {
- Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
- Owned<IConstEnvironment> env = factory->openEnvironment();
- if (env)
- {
- Owned<IPropertyTree> root = &env->getPTree();
- StringBuffer queueNames;
- StringBuffer xpath;
- xpath.appendf("%s[@process=\"%s\"]", type, process);
- Owned<IPropertyTreeIterator> targets = root->getElements("Software/Topology/Cluster");
- ForEach(*targets)
- {
- IPropertyTree &target = targets->query();
- if (target.hasProp(xpath))
- {
- if (queueNames.length())
- queueNames.append(',');
- queueNames.append(target.queryProp("@name")).append(suffix);
- }
- }
- ret.set(queueNames);
- }
- }
- return ret;
- }
- #define ROXIE_QUEUE_EXT ".roxie"
- #define THOR_QUEUE_EXT ".thor"
- #define ECLCCSERVER_QUEUE_EXT ".eclserver"
- #define ECLSERVER_QUEUE_EXT ECLCCSERVER_QUEUE_EXT
- #define ECLSCHEDULER_QUEUE_EXT ".eclscheduler"
- #define ECLAGENT_QUEUE_EXT ".agent"
- extern WORKUNIT_API IStringVal &getEclCCServerQueueNames(IStringVal &ret, const char *process)
- {
- return getProcessQueueNames(ret, process, "EclCCServerProcess", ECLCCSERVER_QUEUE_EXT);
- }
- extern WORKUNIT_API IStringVal &getEclServerQueueNames(IStringVal &ret, const char *process)
- {
- return getProcessQueueNames(ret, process, "EclServerProcess", ECLSERVER_QUEUE_EXT); // shares queue name with EclCCServer
- }
- extern WORKUNIT_API IStringVal &getEclSchedulerQueueNames(IStringVal &ret, const char *process)
- {
- return getProcessQueueNames(ret, process, "EclSchedulerProcess", ECLSCHEDULER_QUEUE_EXT); // Shares deployment/config with EclCCServer
- }
- extern WORKUNIT_API IStringVal &getAgentQueueNames(IStringVal &ret, const char *process)
- {
- return getProcessQueueNames(ret, process, "EclAgentProcess", ECLAGENT_QUEUE_EXT);
- }
- extern WORKUNIT_API IStringVal &getRoxieQueueNames(IStringVal &ret, const char *process)
- {
- return getProcessQueueNames(ret, process, "RoxieCluster", ROXIE_QUEUE_EXT);
- }
- extern WORKUNIT_API IStringVal &getThorQueueNames(IStringVal &ret, const char *process)
- {
- return getProcessQueueNames(ret, process, "ThorCluster", THOR_QUEUE_EXT);
- }
- extern WORKUNIT_API StringBuffer &getClusterThorQueueName(StringBuffer &ret, const char *cluster)
- {
- return ret.append(cluster).append(THOR_QUEUE_EXT);
- }
- extern WORKUNIT_API StringBuffer &getClusterThorGroupName(StringBuffer &ret, const char *cluster)
- {
- Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
- Owned<IConstEnvironment> env = factory->openEnvironment();
- if (env)
- {
- Owned<IPropertyTree> root = &env->getPTree();
- StringBuffer path;
- path.append("Software/ThorCluster[@name=\"").append(cluster).append("\"]");
- IPropertyTree * child = root->queryPropTree(path);
- if (child)
- getClusterGroupName(*child, ret);
- }
- return ret;
- }
- extern WORKUNIT_API StringBuffer &getClusterRoxieQueueName(StringBuffer &ret, const char *cluster)
- {
- return ret.append(cluster).append(ROXIE_QUEUE_EXT);
- }
- extern WORKUNIT_API StringBuffer &getClusterEclCCServerQueueName(StringBuffer &ret, const char *cluster)
- {
- return ret.append(cluster).append(ECLCCSERVER_QUEUE_EXT);
- }
- extern WORKUNIT_API StringBuffer &getClusterEclServerQueueName(StringBuffer &ret, const char *cluster)
- {
- return ret.append(cluster).append(ECLSERVER_QUEUE_EXT);
- }
- extern WORKUNIT_API StringBuffer &getClusterEclAgentQueueName(StringBuffer &ret, const char *cluster)
- {
- return ret.append(cluster).append(ECLAGENT_QUEUE_EXT);
- }
- extern WORKUNIT_API IStringIterator *getTargetClusters(const char *processType, const char *processName)
- {
- Owned<CStringArrayIterator> ret = new CStringArrayIterator;
- Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
- Owned<IConstEnvironment> env = factory->openEnvironment();
- if (env)
- {
- Owned<IPropertyTree> root = &env->getPTree();
- StringBuffer xpath;
- xpath.appendf("%s", processType ? processType : "*");
- if (processName && *processName)
- xpath.appendf("[@process=\"%s\"]", processName);
- Owned<IPropertyTreeIterator> targets = root->getElements("Software/Topology/Cluster");
- ForEach(*targets)
- {
- IPropertyTree &target = targets->query();
- if (target.hasProp(xpath))
- {
- ret->append(target.queryProp("@name"));
- }
- }
- }
- return ret.getClear();
- }
- extern WORKUNIT_API bool isProcessCluster(const char *process)
- {
- if (!process || !*process)
- return false;
- Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
- Owned<IConstEnvironment> env = factory->openEnvironment();
- if (!env)
- return false;
- Owned<IPropertyTree> root = &env->getPTree();
- VStringBuffer xpath("Software/*Cluster[@name=\"%s\"]", process);
- return root->hasProp(xpath.str());
- }
- extern WORKUNIT_API bool isProcessCluster(const char *remoteDali, const char *process)
- {
- if (!remoteDali || !*remoteDali)
- return isProcessCluster(process);
- if (!process || !*process)
- return false;
- Owned<INode> remote = createINode(remoteDali, 7070);
- if (!remote)
- return false;
- //Cannot use getEnvironmentFactory() since it is using a remotedali
- VStringBuffer xpath("Environment/Software/*Cluster[@name=\"%s\"]/@name", process);
- try
- {
- Owned<IPropertyTreeIterator> clusters = querySDS().getElementsRaw(xpath, remote, 1000*60*1);
- return clusters->first();
- }
- catch (IException *E)
- {
- StringBuffer msg;
- E->errorMessage(msg);
- DBGLOG("Exception validating cluster %s/%s: %s", remoteDali, xpath.str(), msg.str());
- E->Release();
- }
- return true;
- }
- IConstWUClusterInfo* getTargetClusterInfo(IPropertyTree *environment, IPropertyTree *cluster)
- {
- const char *clustname = cluster->queryProp("@name");
- // MORE - at the moment configenf specifies eclagent and thor queues by (in effect) placing an 'example' thor or eclagent in the topology
- // that uses the queue that will be used.
- // We should and I hope will change that, at which point the code below gets simpler
- StringBuffer prefix(cluster->queryProp("@prefix"));
- prefix.toLowerCase();
- StringBuffer xpath;
- StringBuffer querySetName;
-
- IPropertyTree *agent = NULL;
- const char *agentName = cluster->queryProp("EclAgentProcess/@process");
- if (agentName)
- {
- xpath.clear().appendf("Software/EclAgentProcess[@name=\"%s\"]", agentName);
- agent = environment->queryPropTree(xpath.str());
- }
- Owned<IPropertyTreeIterator> ti = cluster->getElements("ThorCluster");
- IArrayOf<IPropertyTree> thors;
- ForEach(*ti)
- {
- const char *thorName = ti->query().queryProp("@process");
- if (thorName)
- {
- xpath.clear().appendf("Software/ThorCluster[@name=\"%s\"]", thorName);
- thors.append(*environment->getPropTree(xpath.str()));
- }
- }
- const char *roxieName = cluster->queryProp("RoxieCluster/@process");
- return new CEnvironmentClusterInfo(clustname, prefix, agent, thors, queryRoxieProcessTree(environment, roxieName));
- }
- IPropertyTree* getTopologyCluster(Owned<IPropertyTree> &envRoot, const char *clustname)
- {
- if (!clustname || !*clustname)
- return NULL;
- Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
- Owned<IConstEnvironment> env = factory->openEnvironment();
- if (!env)
- return NULL;
- envRoot.setown(&env->getPTree());
- StringBuffer xpath;
- xpath.appendf("Software/Topology/Cluster[@name=\"%s\"]", clustname);
- return envRoot->getPropTree(xpath.str());
- }
- bool validateTargetClusterName(const char *clustname)
- {
- Owned<IPropertyTree> envRoot;
- Owned<IPropertyTree> cluster = getTopologyCluster(envRoot, clustname);
- return (cluster.get()!=NULL);
- }
- IConstWUClusterInfo* getTargetClusterInfo(const char *clustname)
- {
- Owned<IPropertyTree> envRoot;
- Owned<IPropertyTree> cluster = getTopologyCluster(envRoot, clustname);
- if (!cluster)
- return NULL;
- return getTargetClusterInfo(envRoot, cluster);
- }
- unsigned getEnvironmentClusterInfo(CConstWUClusterInfoArray &clusters)
- {
- Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
- Owned<IConstEnvironment> env = factory->openEnvironment();
- if (!env)
- return 0;
- Owned<IPropertyTree> root = &env->getPTree();
- return getEnvironmentClusterInfo(root, clusters);
- }
- unsigned getEnvironmentClusterInfo(IPropertyTree* environmentRoot, CConstWUClusterInfoArray &clusters)
- {
- if (!environmentRoot)
- return 0;
- Owned<IPropertyTreeIterator> clusterIter = environmentRoot->getElements("Software/Topology/Cluster");
- ForEach(*clusterIter)
- {
- IPropertyTree &node = clusterIter->query();
- Owned<IConstWUClusterInfo> cluster = getTargetClusterInfo(environmentRoot, &node);
- clusters.append(*cluster.getClear());
- }
- return clusters.ordinality();
- }
- const char *getTargetClusterComponentName(const char *clustname, const char *processType, StringBuffer &name)
- {
- if (!clustname)
- return NULL;
- Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
- Owned<IConstEnvironment> env = factory->openEnvironment();
- if (!env)
- return NULL;
- Owned<IPropertyTree> root = &env->getPTree();
- StringBuffer xpath;
- xpath.appendf("Software/Topology/Cluster[@name=\"%s\"]", clustname);
- Owned<IPropertyTree> cluster = root->getPropTree(xpath.str());
- if (!cluster)
- return NULL;
- StringBuffer xpath1;
- xpath1.appendf("%s/@process", processType);
- name.append(cluster->queryProp(xpath1.str()));
- return name.str();
- }
- unsigned getEnvironmentThorClusterNames(StringArray &thorNames, StringArray &groupNames, StringArray &targetNames, StringArray &queueNames)
- {
- Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
- Owned<IConstEnvironment> env = factory->openEnvironment();
- if (!env)
- return 0;
- Owned<IPropertyTree> root = &env->getPTree();
- Owned<IPropertyTreeIterator> allTargets = root->getElements("Software/Topology/Cluster");
- ForEach(*allTargets)
- {
- IPropertyTree &target = allTargets->query();
- const char *targetName = target.queryProp("@name");
- if (targetName && *targetName)
- {
- Owned<IPropertyTreeIterator> thorClusters = target.getElements("ThorCluster");
- ForEach(*thorClusters)
- {
- const char *thorName = thorClusters->query().queryProp("@process");
- VStringBuffer query("Software/ThorCluster[@name=\"%s\"]",thorName);
- IPropertyTree *thorCluster = root->queryPropTree(query.str());
- if (thorCluster)
- {
- const char *groupName = thorCluster->queryProp("@nodeGroup");
- if (!groupName||!*groupName)
- groupName = thorName;
- thorNames.append(thorName);
- groupNames.append(groupName);
- targetNames.append(targetName);
- StringBuffer queueName(targetName);
- queueNames.append(queueName.append(THOR_QUEUE_EXT));
- }
- }
- }
- }
- return thorNames.ordinality();
- }
- unsigned getEnvironmentHThorClusterNames(StringArray &eclAgentNames, StringArray &groupNames, StringArray &targetNames)
- {
- Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
- Owned<IConstEnvironment> env = factory->openEnvironment();
- if (!env)
- return 0;
- Owned<IPropertyTree> root = &env->getPTree();
- Owned<IPropertyTreeIterator> allEclAgents = root->getElements("Software/EclAgentProcess");
- ForEach(*allEclAgents)
- {
- IPropertyTree &eclAgent = allEclAgents->query();
- const char *eclAgentName = eclAgent.queryProp("@name");
- if (eclAgentName && *eclAgentName)
- {
- Owned<IPropertyTreeIterator> allTargets = root->getElements("Software/Topology/Cluster");
- ForEach(*allTargets)
- {
- IPropertyTree &target = allTargets->query();
- const char *targetName = target.queryProp("@name");
- if (targetName && *targetName)
- {
- StringBuffer xpath;
- xpath.appendf("EclAgentProcess[@process=\"%s\"]", eclAgentName);
- if (target.hasProp(xpath) && !target.hasProp("ThorCluster"))
- {
- StringBuffer groupName("hthor__");
- groupName.append(eclAgentName);
- groupNames.append(groupName);
- eclAgentNames.append(eclAgentName);
- targetNames.append(targetName);
- }
- }
- }
- }
- }
- return eclAgentNames.ordinality();
- }
- IStringVal& CLocalWorkUnit::getScope(IStringVal &str) const
- {
- CriticalBlock block(crit);
- if (p->hasProp("Debug/ForceScope"))
- {
- StringBuffer prefix(p->queryProp("Debug/ForceScope"));
- str.set(prefix.toLowerCase().str());
- }
- else
- {
- Owned <IConstWUClusterInfo> ci = getTargetClusterInfo(p->queryProp("@clusterName"));
- if (ci)
- ci->getScope(str);
- else
- str.clear();
- }
- return str;
- }
- //Queries
- void CLocalWorkUnit::setCodeVersion(unsigned codeVersion, const char * buildVersion, const char * eclVersion)
- {
- CriticalBlock block(crit);
- p->setPropInt("@codeVersion", codeVersion);
- p->setProp("@buildVersion", buildVersion);
- p->setProp("@eclVersion", eclVersion);
- }
- unsigned CLocalWorkUnit::getCodeVersion() const
- {
- CriticalBlock block(crit);
- return p->getPropInt("@codeVersion");
- }
- unsigned CLocalWorkUnit::getWuidVersion() const
- {
- CriticalBlock block(crit);
- return p->getPropInt("@wuidVersion");
- }
- void CLocalWorkUnit::getBuildVersion(IStringVal & buildVersion, IStringVal & eclVersion) const
- {
- CriticalBlock block(crit);
- buildVersion.set(p->queryProp("@buildVersion"));
- eclVersion.set(p->queryProp("@eclVersion"));
- }
- void CLocalWorkUnit::setCloneable(bool value)
- {
- CriticalBlock block(crit);
- p->setPropInt("@cloneable", value);
- }
- void CLocalWorkUnit::setIsClone(bool value)
- {
- CriticalBlock block(crit);
- p->setPropInt("@isClone", value);
- }
- bool CLocalWorkUnit::getCloneable() const
- {
- CriticalBlock block(crit);
- return p->getPropBool("@cloneable", false);
- }
- IUserDescriptor *CLocalWorkUnit::queryUserDescriptor() const
- {
- CriticalBlock block(crit);
- if (!userDesc)
- {
- SCMStringBuffer token, user, password;
- getSecurityToken(token);
- extractToken(token.str(), queryWuid(), user, password);
- userDesc.setown(createUserDescriptor());
- userDesc->set(user.str(), password.str());
- }
- return userDesc;
- }
- bool CLocalWorkUnit::isProtected() const
- {
- CriticalBlock block(crit);
- return p->getPropBool("@protected", false);
- }
- bool CLocalWorkUnit::isPausing() const
- {
- CriticalBlock block(crit);
- if (WUActionPause == getAction())
- {
- switch (getState())
- {
- case WUStateRunning:
- case WUStateAborting:
- return true;
- }
- }
- return false;
- }
- void CLocalWorkUnit::protect(bool protectMode)
- {
- CriticalBlock block(crit);
- p->setPropBool("@protected", protectMode);
- }
- void CLocalWorkUnit::setResultLimit(unsigned value)
- {
- CriticalBlock block(crit);
- p->setPropInt("resultLimit", value);
- }
- unsigned CLocalWorkUnit::getResultLimit() const
- {
- CriticalBlock block(crit);
- return p->getPropInt("resultLimit");
- }
- IStringVal & CLocalWorkUnit::getSnapshot(IStringVal & str) const
- {
- CriticalBlock block(crit);
- str.set(p->queryProp("SNAPSHOT"));
- return str;
- }
- void CLocalWorkUnit::setSnapshot(const char * val)
- {
- CriticalBlock block(crit);
- p->setProp("SNAPSHOT", val);
- }
- static int comparePropTrees(IInterface * const *ll, IInterface * const *rr)
- {
- IPropertyTree *l = (IPropertyTree *) *ll;
- IPropertyTree *r = (IPropertyTree *) *rr;
- return stricmp(l->queryName(), r->queryName());
- };
- unsigned CLocalWorkUnit::calculateHash(unsigned crc)
- {
- // Any other values in the WU that could affect generated code should be crc'ed here
- IPropertyTree *tree = p->queryBranch("Debug");
- if (tree)
- {
- Owned<IPropertyTreeIterator> sub = tree->getElements("*");
- ICopyArrayOf<IPropertyTree> subs;
- for(sub->first(); sub->isValid(); sub->next())
- subs.append(sub->query());
- subs.sort(comparePropTrees);
- ForEachItemIn(idx, subs)
- {
- const char *name = subs.item(idx).queryName();
- const char *val = subs.item(idx).queryProp(NULL);
- crc = crc32(name, (size32_t)strlen(name), crc);
- if (val)
- crc = crc32(val, (size32_t)strlen(val), crc);
- }
- }
- Owned<IConstWUPluginIterator> plugins = &getPlugins();
- for (plugins->first();plugins->isValid();plugins->next())
- {
- IConstWUPlugin &thisplugin = plugins->query();
- SCMStringBuffer version;
- thisplugin.getPluginVersion(version);
- crc = crc32(version.str(), version.length(), crc);
- }
- return crc;
- }
- static void updateProp(IPropertyTree * to, const IPropertyTree * from, const char * xpath)
- {
- if (!to->hasProp(xpath) && from->hasProp(xpath))
- to->setProp(xpath, from->queryProp(xpath));
- }
- static void setProp(IPropertyTree * to, const IPropertyTree * from, const char * xpath)
- {
- if (from->hasProp(xpath))
- to->setProp(xpath, from->queryProp(xpath));
- }
- static void copyTree(IPropertyTree * to, const IPropertyTree * from, const char * xpath)
- {
- IPropertyTree * match = from->getBranch(xpath);
- if (match)
- to->setPropTree(xpath, match);
- }
- void CLocalWorkUnit::copyWorkUnit(IConstWorkUnit *cached, bool all)
- {
- CLocalWorkUnit *from = QUERYINTERFACE(cached, CLocalWorkUnit);
- if (!from)
- {
- CLockedWorkUnit *fl = QUERYINTERFACE(cached, CLockedWorkUnit);
- if (!fl)
- throw MakeStringException(WUERR_InternalUnknownImplementation, "Cached workunit not created using workunit dll");
- from = fl->c;
- }
- // Need to copy the query, the results, and the graphs from the cached query.
- // The cache is made before the query is executed so there is no need to clear them.
- if (!cached->getCloneable())
- throw MakeStringException(WUERR_CannotCloneWorkunit, "Source work unit not marked as clonable");
- const IPropertyTree * fromP = from->p;
- IPropertyTree *pt;
- CriticalBlock block(crit);
- query.clear();
- updateProp(p, fromP, "@jobName");
- copyTree(p, fromP, "Query");
- pt = fromP->getBranch("Application/LibraryModule");
- if (pt)
- {
- ensurePTree(p, "Application");
- p->setPropTree("Application/LibraryModule", pt);
- }
- pt = fromP->queryBranch("Debug");
- if (pt)
- {
- IPropertyTree *curDebug = p->queryPropTree("Debug");
- if (curDebug)
- {
- Owned<IPropertyTreeIterator> elems = pt->getElements("*");
- ForEach(*elems)
- {
- IPropertyTree *elem = &elems->query();
- if (!curDebug->hasProp(elem->queryName()))
- curDebug->setPropTree(elem->queryName(),LINK(elem));
- }
- }
- else
- p->setPropTree("Debug", LINK(pt));
- }
- copyTree(p, fromP, "Plugins");
- copyTree(p, fromP, "Libraries");
- copyTree(p, fromP, "Results");
- copyTree(p, fromP, "Graphs");
- copyTree(p, fromP, "Workflow");
- if (all)
- {
- // 'all' mode is used when setting up a dali WU from the embedded wu in a workunit dll
- // Merge timing info from both branches
- pt = fromP->getBranch("Timings");
- if (pt)
- {
- IPropertyTree *tgtTimings = ensurePTree(p, "Timings");
- mergePTree(tgtTimings, pt);
- pt->Release();
- }
- pt = fromP->getBranch("Statistics");
- if (pt)
- {
- IPropertyTree *tgtStatistics = ensurePTree(p, "Statistics");
- mergePTree(tgtStatistics, pt);
- pt->Release();
- }
- }
- updateProp(p, fromP, "@clusterName");
- updateProp(p, fromP, "allowedclusters");
- updateProp(p, fromP, "@submitID");
- updateProp(p, fromP, "SNAPSHOT");
- //MORE: This is very adhoc. All options that should be cloned should really be in a common branch
- if (all)
- {
- setProp(p, fromP, "PriorityFlag");
- setProp(p, fromP, "@priorityClass");
- setProp(p, fromP, "@protected");
- setProp(p, fromP, "@clusterName");
- updateProp(p, fromP, "@scope");
- }
- //Variables may have been set up as parameters to the query - so need to preserve any values that were supplied.
- pt = fromP->getBranch("Variables");
- if (pt)
- {
- IPropertyTree *ptTgtVariables = ensurePTree(p, "Variables");
- Owned<IPropertyTreeIterator> ptiVariable = pt->getElements("Variable");
- for (ptiVariable->first(); ptiVariable->isValid(); ptiVariable->next())
- {
- IPropertyTree *ptSrcVariable = &ptiVariable->query();
- const char *name = ptSrcVariable->queryProp("@name");
- assertex(name);
- StringBuffer xpath;
- xpath.append("Variable[@name='").append(name).append("']");
- IPropertyTree *ptTgtVariable = ptTgtVariables->queryPropTree(xpath.str());
- IPropertyTree *merged = createPTreeFromIPT(ptSrcVariable); // clone entire source info...
- merged->removeProp("Value"); // except value and status
- merged->setProp("@status", "undefined");
- if (!merged->getPropBool("@isScalar"))
- merged->removeProp("totalRowCount");
- merged->removeProp("rowCount");
- // If there are any other fields that get set ONLY by eclagent, strip them out here...
- if (ptTgtVariable)
- {
- // copy status and Value from what is already set in target
- merged->setProp("@status", ptTgtVariable->queryProp("@status"));
- MemoryBuffer value;
- if (ptTgtVariable->getPropBin("Value", value))
- merged->setPropBin("Value", value.length(), value.toByteArray());
- ptTgtVariable->removeProp(xpath.str());
- // If there are any other fields in a variable that get set by ws_ecl before submitting, copy them across here...
- }
- ptTgtVariables->addPropTree("Variable", merged);
- }
- pt->Release();
- }
- p->setProp("@codeVersion", fromP->queryProp("@codeVersion"));
- p->setPropBool("@cloneable", true);
- p->setPropBool("@isClone", true);
- resetWorkflow(); // the source Workflow section may have had some parts already executed...
- // resetResults(); // probably should be resetting the results as well... rather than waiting for the rerun to overwrite them
- }
- bool CLocalWorkUnit::hasDebugValue(const char *propname) const
- {
- StringBuffer lower;
- lower.append(propname).toLowerCase();
- CriticalBlock block(crit);
- StringBuffer prop("Debug/");
- return p->hasProp(prop.append(lower));
- }
- IStringVal& CLocalWorkUnit::getDebugValue(const char *propname, IStringVal &str) const
- {
- StringBuffer lower;
- lower.append(propname).toLowerCase();
- CriticalBlock block(crit);
- StringBuffer prop("Debug/");
- str.set(p->queryProp(prop.append(lower).str()));
- return str;
- }
- IStringIterator& CLocalWorkUnit::getDebugValues() const
- {
- return getDebugValues(NULL);
- }
- IStringIterator& CLocalWorkUnit::getDebugValues(const char *prop) const
- {
- CriticalBlock block(crit);
- StringBuffer path("Debug/");
- if (prop)
- {
- StringBuffer lower;
- lower.append(prop).toLowerCase();
- path.append(lower);
- }
- else
- path.append("*");
- return *new CStringPTreeTagIterator(p->getElements(path.str()));
- }
- int CLocalWorkUnit::getDebugValueInt(const char *propname, int defVal) const
- {
- StringBuffer lower;
- lower.append(propname).toLowerCase();
- CriticalBlock block(crit);
- StringBuffer prop("Debug/");
- prop.append(lower);
- return p->getPropInt(prop.str(), defVal);
- }
- __int64 CLocalWorkUnit::getDebugValueInt64(const char *propname, __int64 defVal) const
- {
- StringBuffer lower;
- lower.append(propname).toLowerCase();
- CriticalBlock block(crit);
- StringBuffer prop("Debug/");
- prop.append(lower);
- return p->getPropInt64(prop.str(), defVal);
- }
- bool CLocalWorkUnit::getDebugValueBool(const char * propname, bool defVal) const
- {
- StringBuffer lower;
- lower.append(propname).toLowerCase();
- CriticalBlock block(crit);
- StringBuffer prop("Debug/");
- prop.append(lower);
- return p->getPropBool(prop.str(), defVal);
- }
- IStringIterator *CLocalWorkUnit::getLogs(const char *type, const char *instance) const
- {
- VStringBuffer xpath("Process/%s/", type);
- if (instance)
- xpath.append(instance);
- else
- xpath.append("*");
- CriticalBlock block(crit);
- if (p->getPropInt("@wuidVersion") < 1) // legacy wuid
- {
- // NB: instance unused
- if (streq("EclAgent", type))
- return new CStringPTreeIterator(p->getElements("Debug/eclagentlog"));
- else if (streq("Thor", type))
- return new CStringPTreeIterator(p->getElements("Debug/thorlog*"));
- VStringBuffer xpath("Debug/%s", type);
- return new CStringPTreeIterator(p->getElements(xpath.str()));
- }
- else
- return new CStringPTreeAttrIterator(p->getElements(xpath.str()), "@log");
- }
- IPropertyTreeIterator* CLocalWorkUnit::getProcesses(const char *type, const char *instance) const
- {
- VStringBuffer xpath("Process/%s/", type);
- if (instance)
- xpath.append(instance);
- else
- xpath.append("*");
- CriticalBlock block(crit);
- return p->getElements(xpath.str());
- }
- IStringIterator *CLocalWorkUnit::getProcesses(const char *type) const
- {
- VStringBuffer xpath("Process/%s/*", type);
- CriticalBlock block(crit);
- return new CStringPTreeTagIterator(p->getElements(xpath.str()));
- }
- void CLocalWorkUnit::addProcess(const char *type, const char *instance, unsigned pid, const char *log)
- {
- VStringBuffer processType("Process/%s", type);
- VStringBuffer xpath("%s/%s", processType.str(), instance);
- if (log)
- xpath.appendf("[@log=\"%s\"]", log);
- CriticalBlock block(crit);
- if (!p->hasProp(xpath))
- {
- IPropertyTree *node = ensurePTree(p, processType.str());
- node = node->addPropTree(instance, createPTree());
- node->setProp("@log", log);
- node->setPropInt("@pid", pid);
- }
- }
- void CLocalWorkUnit::setDebugValue(const char *propname, const char *value, bool overwrite)
- {
- StringBuffer lower;
- lower.append(propname).toLowerCase();
- CriticalBlock block(crit);
- StringBuffer prop("Debug/");
- prop.append(lower);
- if (overwrite || !p->hasProp(prop.str()))
- {
- // MORE - not sure this line should be needed....
- p->setProp("Debug", "");
- p->setProp(prop.str(), value);
- }
- }
- void CLocalWorkUnit::setDebugValueInt(const char *propname, int value, bool overwrite)
- {
- StringBuffer lower;
- lower.append(propname).toLowerCase();
- CriticalBlock block(crit);
- StringBuffer prop("Debug/");
- prop.append(lower);
- if (overwrite || !p->hasProp(prop.str()))
- {
- // MORE - not sure this line should be needed....
- p->setProp("Debug", "");
- p->setPropInt(prop.str(), value);
- }
- }
- void CLocalWorkUnit::setTracingValue(const char *propname, const char *value)
- {
- CriticalBlock block(crit);
- // MORE - not sure this line should be needed....
- p->setProp("Tracing", "");
- StringBuffer prop("Tracing/");
- p->setProp(prop.append(propname).str(), value);
- }
- void CLocalWorkUnit::setTracingValueInt(const char *propname, int value)
- {
- CriticalBlock block(crit);
- StringBuffer prop("Tracing/");
- p->setPropInt(prop.append(propname).str(), value);
- }
- IConstWUQuery* CLocalWorkUnit::getQuery() const
- {
- // For this to be legally called, we must have the read-able interface. So we are already locked for (at least) read.
- CriticalBlock block(crit);
- if (!query)
- {
- IPropertyTree *s = p->getPropTree("Query");
- if (s)
- query.setown(new CLocalWUQuery(s)); // NB takes ownership of 's'
- }
- return query.getLink();
- }
- IWUQuery* CLocalWorkUnit::updateQuery()
- {
- // For this to be legally called, we must have the write-able interface. So we are already locked for write.
- CriticalBlock block(crit);
- if (!query)
- {
- IPropertyTree *s = p->queryPropTree("Query");
- if (!s)
- s = p->addPropTree("Query", createPTreeFromXMLString("<Query fetchEntire='1'/>"));
- s->Link();
- query.setown(new CLocalWUQuery(s));
- }
- return query.getLink();
- }
- void CLocalWorkUnit::loadPlugins() const
- {
- CriticalBlock block(crit);
- if (!pluginsCached)
- {
- assertex(plugins.length() == 0);
- Owned<IPropertyTreeIterator> r = p->getElements("Plugins/Plugin");
- for (r->first(); r->isValid(); r->next())
- {
- IPropertyTree *rp = &r->query();
- rp->Link();
- plugins.append(*new CLocalWUPlugin(rp));
- }
- pluginsCached = true;
- }
- }
- IConstWUPluginIterator& CLocalWorkUnit::getPlugins() const
- {
- CriticalBlock block(crit);
- loadPlugins();
- return *new CArrayIteratorOf<IConstWUPlugin,IConstWUPluginIterator> (plugins, 0, (IConstWorkUnit *) this);
- }
- void CLocalWorkUnit::loadLibraries() const
- {
- CriticalBlock block(crit);
- if (!librariesCached)
- {
- assertex(libraries.length() == 0);
- Owned<IPropertyTreeIterator> r = p->getElements("Libraries/Library");
- ForEach(*r)
- {
- IPropertyTree *rp = &r->query();
- rp->Link();
- libraries.append(*new CLocalWULibrary(rp));
- }
- librariesCached = true;
- }
- }
- IConstWULibraryIterator& CLocalWorkUnit::getLibraries() const
- {
- CriticalBlock block(crit);
- loadLibraries();
- return *new CArrayIteratorOf<IConstWULibrary,IConstWULibraryIterator> (libraries, 0, (IConstWorkUnit *) this);
- }
- IConstWULibrary * CLocalWorkUnit::getLibraryByName(const char * search) const
- {
- CriticalBlock block(crit);
- loadLibraries();
- ForEachItemIn(idx, libraries)
- {
- SCMStringBuffer name;
- IConstWULibrary &cur = libraries.item(idx);
- cur.getName(name);
- if (stricmp(name.str(), search)==0)
- return &OLINK(cur);
- }
- return NULL;
- }
- StringBuffer &formatGraphTimerLabel(StringBuffer &str, const char *graphName, unsigned subGraphNum, unsigned __int64 subId)
- {
- str.append("Graph ").append(graphName);
- if (subGraphNum) str.append(" - ").append(subGraphNum).append(" (").append(subId).append(")");
- else if (subId) str.append(" - id(").append(subId).append(")");
- return str;
- }
- StringBuffer &formatGraphTimerScope(StringBuffer &str, const char *graphName, unsigned subGraphNum, unsigned __int64 subId)
- {
- str.append(graphName);
- if (subId) str.append(":sg").append(subId);
- return str;
- }
- bool parseGraphTimerLabel(const char *label, StringAttr &graphName, unsigned & graphNum, unsigned &subGraphNum, unsigned &subId)
- {
- // expects format: "Graph <graphname>[ - <subgraphnum> (<subgraphid>)]"
- unsigned len = (size32_t)strlen(label);
- if (len < 6 || (0 != memcmp(label, "Graph ", 6)))
- return false;
- graphNum = 0;
- subGraphNum = 0;
- subId = 0;
- const char *finger = label+6;
- const char *finger2 = strchr(finger, '-');
- if (NULL == finger2) // just graphName
- graphName.set(finger);
- else
- {
- graphName.set(finger, (size32_t)((finger2-1)-finger));
- finger = finger2+2; // skip '-' and space
- finger2 = strchr(finger, ' ');
- if (finger2)
- {
- subGraphNum = atoi_l(finger, (size32_t)(finger2-finger));
- finger = finger2+2; // skip space and '('
- finger2 = strchr(finger, ')');
- if (finger2)
- subId = atoi_l(finger, (size32_t)(finger2-finger));
- }
- else if (((len-(finger-label))>3) && 0 == memcmp(finger, "id(", 3)) // subgraph id only, new format.
- {
- finger += 3;
- finger2 = strchr(finger, ')');
- if (finger2)
- subId = atoi_l(finger, (size32_t)(finger2-finger));
- }
- }
- if (graphName && !memicmp(graphName, "graph", 5))
- graphNum = atoi(graphName + 5);
- return true;
- }
- bool parseGraphScope(const char *scope, StringAttr &graphName, unsigned & graphNum, unsigned &subGraphId)
- {
- if (!MATCHES_CONST_PREFIX(scope, GraphScopePrefix))
- return false;
- graphNum = atoi(scope + CONST_STRLEN(GraphScopePrefix));
- const char * colon = strchr(scope, ':');
- if (!colon)
- {
- graphName.set(scope);
- subGraphId = 0;
- return true;
- }
- const char * subgraph = colon+1;
- graphName.set(scope, (size32_t)(colon - scope));
- if (MATCHES_CONST_PREFIX(subgraph, SubGraphScopePrefix))
- subGraphId = atoi(subgraph+CONST_STRLEN(SubGraphScopePrefix));
- return true;
- }
- class WorkUnitStatisticsIterator : public CArrayIteratorOf<IConstWUStatistic,IConstWUStatisticIterator>
- {
- typedef CArrayIteratorOf<IConstWUStatistic,IConstWUStatisticIterator> PARENT;
- public:
- WorkUnitStatisticsIterator(const IArray &a, aindex_t start, IInterface *owner, const IStatisticsFilter * _filter)
- : PARENT(a,start, owner), filter(_filter)
- {
- }
- virtual bool first()
- {
- if (!PARENT::first())
- return false;
- if (matchesFilter())
- return true;
- return next();
- }
- virtual bool next()
- {
- loop
- {
- if (!PARENT::next())
- return false;
- if (matchesFilter())
- return true;
- }
- }
- protected:
- bool matchesFilter()
- {
- if (!filter)
- return true;
- return query().matches(filter);
- }
- protected:
- Linked<const IStatisticsFilter> filter;
- };
- void CLocalWorkUnit::setStatistic(StatisticCreatorType creatorType, const char * creator, StatisticScopeType scopeType, const char * scope, StatisticKind kind, const char * optDescription, unsigned __int64 value, unsigned __int64 count, unsigned __int64 maxValue, StatsMergeAction mergeAction)
- {
- if (!scope || !*scope) scope = GLOBAL_SCOPE;
- const char * kindName = queryStatisticName(kind);
- StatisticMeasure measure = queryMeasure(kind);
- //creator. scope and name must all be present, and must not contain semi colons.
- assertex(creator && scope);
- CriticalBlock block(crit);
- IPropertyTree * stats = p->queryPropTree("Statistics");
- if (!stats)
- stats = p->addPropTree("Statistics", createPTree("Statistics"));
- IPropertyTree * statTree = NULL;
- if (mergeAction != StatsMergeAppend)
- {
- StringBuffer xpath;
- xpath.append("Statistic[@creator='").append(creator).append("'][@scope='").append(scope).append("'][@kind='").append(kindName).append("']");
- statTree = stats->queryPropTree(xpath.str());
- }
- if (!statTree)
- {
- statTree = stats->addPropTree("Statistic", createPTree("Statistic"));
- statTree->setProp("@creator", creator);
- statTree->setProp("@scope", scope);
- statTree->setProp("@kind", kindName);
- //These items are primarily here to facilitate filtering.
- statTree->setProp("@unit", queryMeasureName(measure));
- statTree->setProp("@c", queryCreatorTypeName(creatorType));
- statTree->setProp("@s", queryScopeTypeName(scopeType));
- statTree->setPropInt64("@ts", getTimeStampNowValue());
- if (optDescription)
- statTree->setProp("@desc", optDescription);
- if (statistics.cached)
- statistics.append(LINK(statTree));
- mergeAction = StatsMergeAppend;
- }
- if (mergeAction != StatsMergeAppend)
- {
- unsigned __int64 oldValue = statTree->getPropInt64("@value", 0);
- unsigned __int64 oldCount = statTree->getPropInt64("@count", 0);
- unsigned __int64 oldMax = statTree->getPropInt64("@max", 0);
- if (oldMax < oldValue)
- oldMax = oldValue;
- statTree->setPropInt64("@value", mergeStatisticValue(oldValue, value, mergeAction));
- statTree->setPropInt64("@count", count + oldCount);
- if (maxValue > oldMax)
- statTree->setPropInt64("@max", maxValue);
- }
- else
- {
- statTree->setPropInt64("@value", value);
- statTree->setPropInt64("@count", count);
- if (maxValue)
- statTree->setPropInt64("@max", maxValue);
- else
- statTree->removeProp("@max");
- }
- }
- IConstWUStatisticIterator& CLocalWorkUnit::getStatistics(const IStatisticsFilter * filter) const
- {
- CriticalBlock block(crit);
- //This should be deleted in version 6.0 when support for 4.x is no longer required
- legacyTimings.load(p,"Timings/*");
- if (legacyTimings.ordinality())
- return *new WorkUnitStatisticsIterator(legacyTimings, 0, (IConstWorkUnit *) this, filter);
- statistics.load(p,"Statistics/*");
- Owned<IConstWUStatisticIterator> localStats = new WorkUnitStatisticsIterator(statistics, 0, (IConstWorkUnit *) this, filter);
- if (filter && !filter->queryMergeSources())
- return *localStats.getClear();
- const char * wuid = p->queryName();
- Owned<IConstWUStatisticIterator> graphStats = new CConstGraphProgressStatisticsIterator(wuid, filter);
- return * new CCompoundIteratorOf<IConstWUStatisticIterator, IConstWUStatistic>(localStats, graphStats);
- }
- IConstWUStatistic * CLocalWorkUnit::getStatistic(const char * creator, const char * scope, StatisticKind kind) const
- {
- //MORE: Optimize this....
- StatisticsFilter filter;
- filter.setCreator(creator);
- filter.setScope(scope);
- filter.setKind(kind);
- Owned<IConstWUStatisticIterator> stats = &getStatistics(&filter);
- if (stats->first())
- return LINK(&stats->query());
- return NULL;
- }
- bool CLocalWorkUnit::getWuDate(unsigned & year, unsigned & month, unsigned& day)
- {
- CriticalBlock block(crit);
- if (sscanf(queryWuid(), "W%4u%2u%2u", &year, &month, &day)==3)
- {
- }
-
- return false;
- }
- IWUPlugin* CLocalWorkUnit::updatePluginByName(const char *qname)
- {
- CriticalBlock block(crit);
- IConstWUPlugin *existing = getPluginByName(qname);
- if (existing)
- return (IWUPlugin *) existing;
- if (!plugins.length())
- p->addPropTree("Plugins", createPTree("Plugins"));
- IPropertyTree *pl = p->queryPropTree("Plugins");
- IPropertyTree *s = pl->addPropTree("Plugin", createPTree("Plugin"));
- s->Link();
- IWUPlugin* q = new CLocalWUPlugin(s);
- q->Link();
- plugins.append(*q);
- q->setPluginName(qname);
- return q;
- }
- IConstWUPlugin* CLocalWorkUnit::getPluginByName(const char *qname) const
- {
- CriticalBlock block(crit);
- loadPlugins();
- ForEachItemIn(idx, plugins)
- {
- SCMStringBuffer name;
- IConstWUPlugin &cur = plugins.item(idx);
- cur.getPluginName(name);
- if (stricmp(name.str(), qname)==0)
- {
- cur.Link();
- return &cur;
- }
- }
- return NULL;
- }
- IWULibrary* CLocalWorkUnit::updateLibraryByName(const char *qname)
- {
- CriticalBlock block(crit);
- IConstWULibrary *existing = getLibraryByName(qname);
- if (existing)
- return (IWULibrary *) existing;
- if (!libraries.length())
- p->addPropTree("Libraries", createPTree("Libraries"));
- IPropertyTree *pl = p->queryPropTree("Libraries");
- IPropertyTree *s = pl->addPropTree("Library", createPTree("Library"));
- s->Link();
- IWULibrary* q = new CLocalWULibrary(s);
- q->Link();
- libraries.append(*q);
- q->setName(qname);
- return q;
- }
- void CLocalWorkUnit::loadExceptions() const
- {
- CriticalBlock block(crit);
- if (!exceptionsCached)
- {
-
- assertex(exceptions.length() == 0);
- Owned<IPropertyTreeIterator> r = p->getElements("Exceptions/Exception");
- for (r->first(); r->isValid(); r->next())
- {
- IPropertyTree *rp = &r->query();
- rp->Link();
- exceptions.append(*new CLocalWUException(rp));
- }
- exceptionsCached = true;
- }
- }
- IConstWUExceptionIterator& CLocalWorkUnit::getExceptions() const
- {
- CriticalBlock block(crit);
- loadExceptions();
- return *new CArrayIteratorOf<IConstWUException,IConstWUExceptionIterator> (exceptions, 0, (IConstWorkUnit *) this);
- }
- unsigned CLocalWorkUnit::getExceptionCount() const
- {
- CriticalBlock block(crit);
- loadExceptions();
- return exceptions.length();
- }
- void CLocalWorkUnit::clearExceptions()
- {
- CriticalBlock block(crit);
- // For this to be legally called, we must have the write-able interface. So we are already locked for write.
- exceptions.kill();
- exceptionsCached = true;
- p->removeProp("Exceptions");
- }
- IWUException* CLocalWorkUnit::createException()
- {
- CriticalBlock block(crit);
- // For this to be legally called, we must have the write-able interface. So we are already locked for write.
- loadExceptions();
- if (!exceptions.length())
- p->addPropTree("Exceptions", createPTree("Exceptions"));
- IPropertyTree *r = p->queryPropTree("Exceptions");
- IPropertyTree *s = r->addPropTree("Exception", createPTree("Exception"));
- IWUException* q = new CLocalWUException(LINK(s));
- exceptions.append(*LINK(q));
- Owned<IJlibDateTime> now = createDateTimeNow();
- SCMStringBuffer temp;
- now->getString(temp);
- q->setTimeStamp(temp.str());
- return q;
- }
- IConstWUWebServicesInfo* CLocalWorkUnit::getWebServicesInfo() const
- {
- // For this to be legally called, we must have the read-able interface. So we are already locked for (at least) read.
- CriticalBlock block(crit);
- if (!webServicesInfoCached)
- {
- assertex(!webServicesInfo);
- IPropertyTree *s = p->getPropTree("WebServicesInfo");
- if (s)
- webServicesInfo.setown(new CLocalWUWebServicesInfo(s)); // NB takes ownership of 's'
- webServicesInfoCached = true;
- }
- return webServicesInfo.getLink();
- }
- IWUWebServicesInfo* CLocalWorkUnit::updateWebServicesInfo(bool create)
- {
- // For this to be legally called, we must have the write-able interface. So we are already locked for write.
- CriticalBlock block(crit);
- if (!webServicesInfoCached)
- {
- IPropertyTree *s = p->queryPropTree("WebServicesInfo");
- if (!s)
- {
- if (create)
- s = p->addPropTree("WebServicesInfo", createPTreeFromXMLString("<WebServicesInfo />"));
- else
- return NULL;
- }
- s->Link();
- webServicesInfo.setown(new CLocalWUWebServicesInfo(s));
- webServicesInfoCached = true;
- }
- return webServicesInfo.getLink();
- }
- IConstWURoxieQueryInfo* CLocalWorkUnit::getRoxieQueryInfo() const
- {
- // For this to be legally called, we must have the read-able interface. So we are already locked for (at least) read.
- CriticalBlock block(crit);
- if (!roxieQueryInfoCached)
- {
- assertex(!roxieQueryInfo);
- IPropertyTree *s = p->getPropTree("RoxieQueryInfo");
- if (s)
- roxieQueryInfo.setown(new CLocalWURoxieQueryInfo(s)); // NB takes ownership of 's'
- roxieQueryInfoCached = true;
- }
- return roxieQueryInfo.getLink();
- }
- IWURoxieQueryInfo* CLocalWorkUnit::updateRoxieQueryInfo(const char *wuid, const char *roxieClusterName)
- {
- // For this to be legally called, we must have the write-able interface. So we are already locked for write.
- CriticalBlock block(crit);
- if (!roxieQueryInfoCached)
- {
- IPropertyTree *s = p->queryPropTree("RoxieQueryInfo");
- if (!s)
- s = p->addPropTree("RoxieQueryInfo", createPTreeFromXMLString("<RoxieQueryInfo />"));
- if (wuid && *wuid)
- s->addProp("@wuid", wuid);
- if (roxieClusterName && *roxieClusterName)
- s->addProp("@roxieClusterName", roxieClusterName);
- s->Link();
- roxieQueryInfo.setown(new CLocalWURoxieQueryInfo(s));
- roxieQueryInfoCached = true;
- }
- return roxieQueryInfo.getLink();
- }
- static int compareResults(IInterface * const *ll, IInterface * const *rr)
- {
- CLocalWUResult *l = (CLocalWUResult *) *ll;
- CLocalWUResult *r = (CLocalWUResult *) *rr;
- return l->getResultSequence() - r->getResultSequence();
- }
- void CLocalWorkUnit::loadResults() const
- {
- CriticalBlock block(crit);
- if (!resultsCached)
- {
- assertex(results.length() == 0);
- Owned<IPropertyTreeIterator> r = p->getElements("Results/Result");
- for (r->first(); r->isValid(); r->next())
- {
- IPropertyTree *rp = &r->query();
- rp->Link();
- results.append(*new CLocalWUResult(rp));
- }
- results.sort(compareResults);
- resultsCached = true;
- }
- }
- void CLocalWorkUnit::loadVariables() const
- {
- CriticalBlock block(crit);
- if (!variablesCached)
- {
- assertex(variables.length() == 0);
- Owned<IPropertyTreeIterator> r = p->getElements("Variables/Variable");
- for (r->first(); r->isValid(); r->next())
- {
- IPropertyTree *rp = &r->query();
- rp->Link();
- variables.append(*new CLocalWUResult(rp));
- }
- variablesCached = true;
- }
- }
- void CLocalWorkUnit::loadTemporaries() const
- {
- CriticalBlock block(crit);
- if (!temporariesCached)
- {
- assertex(temporaries.length() == 0);
- Owned<IPropertyTreeIterator> r = p->getElements("Temporaries/Variable");
- for (r->first(); r->isValid(); r->next())
- {
- IPropertyTree *rp = &r->query();
- rp->Link();
- temporaries.append(*new CLocalWUResult(rp));
- }
- temporariesCached = true;
- }
- }
- void CLocalWorkUnit::deleteTemporaries()
- {
- CriticalBlock block(crit);
- if (temporariesCached)
- {
- temporaries.kill();
- temporariesCached = false;
- }
- p->removeProp("Temporaries");
- }
- IWUResult* CLocalWorkUnit::createResult()
- {
- CriticalBlock block(crit);
- // For this to be legally called, we must have the write-able interface. So we are already locked for write.
- loadResults();
- if (!results.length())
- p->addPropTree("Results", createPTree("Results"));
- IPropertyTree *r = p->queryPropTree("Results");
- IPropertyTree *s = r->addPropTree("Result", createPTree());
- s->Link();
- IWUResult* q = new CLocalWUResult(s);
- q->Link();
- results.append(*q);
- return q;
- }
- IWUResult* CLocalWorkUnit::updateResultByName(const char *qname)
- {
- CriticalBlock block(crit);
- IConstWUResult *existing = getResultByName(qname);
- if (existing)
- return (IWUResult *) existing;
- IWUResult* q = createResult();
- q->setResultName(qname);
- return q;
- }
- IWUResult* CLocalWorkUnit::updateResultBySequence(unsigned seq)
- {
- CriticalBlock block(crit);
- IConstWUResult *existing = getResultBySequence(seq);
- if (existing)
- return (IWUResult *) existing;
- IWUResult* q = createResult();
- q->setResultSequence(seq);
- return q;
- }
- IConstWUResultIterator& CLocalWorkUnit::getResults() const
- {
- CriticalBlock block(crit);
- loadResults();
- return *new CArrayIteratorOf<IConstWUResult,IConstWUResultIterator> (results, 0, (IConstWorkUnit *) this);
- }
- IConstWUResult* CLocalWorkUnit::getResultByName(const char *qname) const
- {
- CriticalBlock block(crit);
- loadResults();
- ForEachItemIn(idx, results)
- {
- SCMStringBuffer name;
- IConstWUResult &cur = results.item(idx);
- cur.getResultName(name);
- if (stricmp(name.str(), qname)==0)
- {
- cur.Link();
- return &cur;
- }
- }
- return NULL;
- }
- IConstWUResult* CLocalWorkUnit::getResultBySequence(unsigned seq) const
- {
- CriticalBlock block(crit);
- loadResults();
- ForEachItemIn(idx, results)
- {
- IConstWUResult &cur = results.item(idx);
- if (cur.getResultSequence() == seq)
- {
- cur.Link();
- return &cur;
- }
- }
- return NULL;
- }
- IConstWUResultIterator& CLocalWorkUnit::getVariables() const
- {
- CriticalBlock block(crit);
- loadVariables();
- return *new CArrayIteratorOf<IConstWUResult,IConstWUResultIterator> (variables, 0, (IConstWorkUnit *) this);
- }
- IConstWUResult* CLocalWorkUnit::getGlobalByName(const char *qname) const
- {
- CriticalBlock block(crit);
- if (strcmp(p->queryName(), GLOBAL_WORKUNIT)==0)
- return getVariableByName(qname);
- Owned <IWorkUnit> global = factory->ensureNamedWorkUnit(GLOBAL_WORKUNIT);
- return global->getVariableByName(qname);
- }
- IWUResult* CLocalWorkUnit::updateGlobalByName(const char *qname)
- {
- CriticalBlock block(crit);
- if (strcmp(p->queryName(), GLOBAL_WORKUNIT)==0)
- return updateVariableByName(qname);
- Owned <IWorkUnit> global = factory->ensureNamedWorkUnit(GLOBAL_WORKUNIT);
- return global->updateVariableByName(qname);
- }
- IConstWUResult* CLocalWorkUnit::getVariableByName(const char *qname) const
- {
- CriticalBlock block(crit);
- loadVariables();
- ForEachItemIn(idx, variables)
- {
- SCMStringBuffer name;
- IConstWUResult &cur = variables.item(idx);
- cur.getResultName(name);
- if (stricmp(name.str(), qname)==0)
- {
- cur.Link();
- return &cur;
- }
- }
- return NULL;
- }
- IConstWUResult* CLocalWorkUnit::getTemporaryByName(const char *qname) const
- {
- CriticalBlock block(crit);
- loadTemporaries();
- ForEachItemIn(idx, temporaries)
- {
- SCMStringBuffer name;
- IConstWUResult &cur = temporaries.item(idx);
- cur.getResultName(name);
- if (stricmp(name.str(), qname)==0)
- {
- cur.Link();
- return &cur;
- }
- }
- return NULL;
- }
- IConstWUResultIterator& CLocalWorkUnit::getTemporaries() const
- {
- CriticalBlock block(crit);
- loadTemporaries();
- return *new CArrayIteratorOf<IConstWUResult,IConstWUResultIterator> (temporaries, 0, (IConstWorkUnit *) this);
- }
- IWUResult* CLocalWorkUnit::updateTemporaryByName(const char *qname)
- {
- CriticalBlock block(crit);
- IConstWUResult *existing = getTemporaryByName(qname);
- if (existing)
- return (IWUResult *) existing;
- if (!temporaries.length())
- p->addPropTree("Temporaries", createPTree("Temporaries"));
- IPropertyTree *vars = p->queryPropTree("Temporaries");
- IPropertyTree *s = vars->addPropTree("Variable", createPTree("Variable"));
- s->Link();
- IWUResult* q = new CLocalWUResult(s);
- q->Link();
- temporaries.append(*q);
- q->setResultName(qname);
- return q;
- }
- IWUResult* CLocalWorkUnit::updateVariableByName(const char *qname)
- {
- CriticalBlock block(crit);
- IConstWUResult *existing = getVariableByName(qname);
- if (existing)
- return (IWUResult *) existing;
- if (!variables.length())
- p->addPropTree("Variables", createPTree("Variables"));
- IPropertyTree *vars = p->queryPropTree("Variables");
- IPropertyTree *s = vars->addPropTree("Variable", createPTree("Variable"));
- s->Link();
- IWUResult* q = new CLocalWUResult(s);
- q->Link();
- variables.append(*q);
- q->setResultName(qname);
- return q;
- }
- void CLocalWorkUnit::deleteTempFiles(const char *graph, bool deleteOwned, bool deleteJobOwned)
- {
- CriticalBlock block(crit);
- IPropertyTree *files = p->queryPropTree("Files");
- if (!files) return;
- Owned<IPropertyTreeIterator> iter = files->getElements("File");
- ICopyArrayOf<IPropertyTree> toRemove;
- ForEach (*iter)
- {
- IPropertyTree &file = iter->query();
- WUFileKind fileKind = (WUFileKind) file.getPropInt("@kind", WUFileStandard);
- if(file.getPropBool("@temporary")) fileKind = WUFileTemporary; // @temporary, legacy check
- bool needDelete;
- switch(fileKind)
- {
- case WUFileTemporary:
- if(graph==NULL)
- needDelete = true;
- else
- {
- const char *graphOwner = file.queryProp("@graph");
- needDelete = ((graphOwner==NULL) || (strcmp(graph, graphOwner)==0));
- }
- break;
- case WUFileJobOwned:
- needDelete = ((graph==NULL) && deleteJobOwned);
- break;
- case WUFileOwned:
- needDelete = ((graph==NULL) && deleteOwned);
- break;
- default:
- needDelete = false;
- }
- if(needDelete)
- {
- const char *name = file.queryProp("@name");
- LOG(MCdebugProgress, unknownJob, "Removing workunit file %s from DFS", name);
- queryDistributedFileDirectory().removeEntry(name, queryUserDescriptor());
- toRemove.append(file);
- }
- }
- ForEachItemIn(r, toRemove) files->removeTree(&toRemove.item(r));
- }
- static void _noteFileRead(IDistributedFile *file, IPropertyTree *filesRead)
- {
- IDistributedSuperFile *super = file->querySuperFile();
- StringBuffer fname;
- file->getLogicalName(fname);
- StringBuffer path("File[@name=\"");
- path.append(fname).append("\"]");
- IPropertyTree *fileTree = filesRead->queryPropTree(path.str());
- if (fileTree)
- fileTree->setPropInt("@useCount", fileTree->getPropInt("@useCount")+1);
- else
- {
- StringBuffer cluster;
- file->getClusterName(0,cluster);
- fileTree = createPTree();
- fileTree->setProp("@name", fname.str());
- fileTree->setProp("@cluster", cluster.str());
- fileTree->setPropInt("@useCount", 1);
- fileTree = filesRead->addPropTree("File", fileTree);
- }
-
- if (super)
- {
- Owned<IDistributedFileIterator> iter = super->getSubFileIterator(false);
- ForEach (*iter)
- {
- IDistributedFile &file = iter->query();
- StringBuffer fname;
- file.getLogicalName(fname);
- Owned<IPropertyTree> subfile = createPTree();
- subfile->setProp("@name", fname.str());
- fileTree->addPropTree("Subfile", subfile.getClear());
- _noteFileRead(&file, filesRead);
- }
- }
- }
- void CLocalWorkUnit::noteFileRead(IDistributedFile *file)
- {
- CriticalBlock block(crit);
- IPropertyTree *files = p->queryPropTree("FilesRead");
- if (!files)
- files = p->addPropTree("FilesRead", createPTree());
- _noteFileRead(file, files);
- }
- static void addFile(IPropertyTree *files, const char *fileName, const char *cluster, unsigned usageCount, WUFileKind fileKind, const char *graphOwner)
- {
- StringBuffer path("File[@name=\"");
- path.append(fileName).append("\"]");
- if (cluster)
- path.append("[@cluster=\"").append(cluster).append("\"]");
- IPropertyTree *file = files->queryPropTree(path.str());
- if (file) files->removeTree(file);
- file = createPTree();
- file->setProp("@name", fileName);
- if (cluster)
- file->setProp("@cluster", cluster);
- if (graphOwner)
- file->setProp("@graph", graphOwner);
- file->setPropInt("@kind", (unsigned)fileKind);
- if (WUFileTemporary == fileKind)
- file->setPropInt("@usageCount", usageCount);
- files->addPropTree("File", file);
- }
- void CLocalWorkUnit::addFile(const char *fileName, StringArray *clusters, unsigned usageCount, WUFileKind fileKind, const char *graphOwner)
- {
- CriticalBlock block(crit);
- IPropertyTree *files = p->queryPropTree("Files");
- if (!files)
- files = p->addPropTree("Files", createPTree());
- if (!clusters)
- addFile(fileName, NULL, usageCount, fileKind, graphOwner);
- else
- {
- ForEachItemIn(c, *clusters)
- ::addFile(files, fileName, clusters->item(c), usageCount, fileKind, graphOwner);
- }
- }
- void CLocalWorkUnit::releaseFile(const char *fileName)
- {
- StringBuffer path("File[@name=\"");
- path.append(fileName).append("\"]");
- CriticalBlock block(crit);
- IPropertyTree *files = p->queryPropTree("Files");
- if (!files) return;
- Owned<IPropertyTreeIterator> fiter = files->getElements(path.str());
- ForEach (*fiter)
- {
- IPropertyTree *file = &fiter->query();
- unsigned usageCount = file->getPropInt("@usageCount");
- if (usageCount > 1)
- file->setPropInt("@usageCount", usageCount-1);
- else
- {
- StringAttr name(file->queryProp("@name"));
- files->removeTree(file);
- if (!name.isEmpty()&&(1 == usageCount))
- {
- if (queryDistributedFileDirectory().removeEntry(fileName, queryUserDescriptor()))
- LOG(MCdebugProgress, unknownJob, "Removed (released) file %s from DFS", name.get());
- }
- }
- }
- }
- void CLocalWorkUnit::clearGraphProgress()
- {
- CConstGraphProgress::deleteWuidProgress(p->queryName());
- }
- void CLocalWorkUnit::resetBeforeGeneration()
- {
- CriticalBlock block(crit);
- //Remove all associated files
- Owned<IWUQuery> q = updateQuery();
- q->removeAssociatedFiles();
- //Remove any pre-existing workflow information
- workflowIterator.clear();
- p->removeProp("Workflow");
- }
- unsigned CLocalWorkUnit::queryFileUsage(const char *fileName) const
- {
- StringBuffer path("Files/File[@name=\"");
- path.append(fileName).append("\"]/@usageCount");
- CriticalBlock block(crit);
- return p->getPropInt(path.str());
- }
- IPropertyTree *CLocalWorkUnit::getDiskUsageStats()
- {
- return p->getPropTree("DiskUsageStats");
- }
- void CLocalWorkUnit::addDiskUsageStats(__int64 _avgNodeUsage, unsigned _minNode, __int64 _minNodeUsage, unsigned _maxNode, __int64 _maxNodeUsage, __int64 _graphId)
- {
- IPropertyTree *stats = p->queryPropTree("DiskUsageStats");
- offset_t maxNodeUsage;
- if (stats)
- maxNodeUsage = stats->getPropInt64("@maxNodeUsage");
- else
- {
- stats = p->addPropTree("DiskUsageStats", createPTree());
- maxNodeUsage = 0;
- }
- if ((offset_t)_maxNodeUsage > maxNodeUsage)
- {
- // record all details at time of max node usage.
- stats->setPropInt("@minNode", _minNode);
- stats->setPropInt("@maxNode", _maxNode);
- stats->setPropInt64("@minNodeUsage", _minNodeUsage);
- stats->setPropInt64("@maxNodeUsage", _maxNodeUsage);
- stats->setPropInt64("@graphId", _graphId);
- if (_avgNodeUsage)
- {
- unsigned _skewHi = (unsigned)((100 * (_maxNodeUsage-_avgNodeUsage))/_avgNodeUsage);
- unsigned _skewLo = (unsigned)((100 * (_avgNodeUsage-_minNodeUsage))/_avgNodeUsage);
- stats->setPropInt("@skewHi", _skewHi);
- stats->setPropInt("@skewLo", _skewLo);
- }
- }
- }
- IPropertyTreeIterator & CLocalWorkUnit::getFileIterator() const
- {
- CriticalBlock block(crit);
- return * p->getElements("Files/File");
- }
- IPropertyTreeIterator & CLocalWorkUnit::getFilesReadIterator() const
- {
- CriticalBlock block(crit);
- return * p->getElements("FilesRead/File");
- }
- //=================================================================================================
- bool CLocalWorkUnit::switchThorQueue(const char *cluster, IQueueSwitcher *qs)
- {
- CriticalBlock block(crit);
- if (qs->isAuto()&&!getAllowAutoQueueSwitch())
- return false;
- Owned<IConstWUClusterInfo> newci = getTargetClusterInfo(cluster);
- if (!newci)
- return false;
- StringBuffer currentcluster;
- if (!p->getProp("@clusterName",currentcluster))
- return false;
- Owned<IConstWUClusterInfo> curci = getTargetClusterInfo(currentcluster.str());
- if (!curci)
- return false;
- SCMStringBuffer curqname;
- curci->getThorQueue(curqname);
- const char *wuid = p->queryName();
- void *qi = qs->getQ(curqname.str(),wuid);
- if (!qi)
- return false;
- setClusterName(cluster);
- SCMStringBuffer newqname;
- newci->getThorQueue(newqname);
- qs->putQ(newqname.str(),wuid,qi);
- return true;
- }
- //=================================================================================================
- IPropertyTree *CLocalWorkUnit::getUnpackedTree(bool includeProgress) const
- {
- Owned<IPropertyTree> ret = createPTreeFromIPT(p);
- Owned<IConstWUGraphIterator> graphIter = &getGraphs(GraphTypeAny);
- ForEach(*graphIter)
- {
- IConstWUGraph &graph = graphIter->query();
- Owned<IPropertyTree> graphTree = graph.getXGMMLTree(includeProgress);
- SCMStringBuffer gName;
- graph.getName(gName);
- StringBuffer xpath("Graphs/Graph[@name=\"");
- xpath.append(gName.s).append("\"]/xgmml");
- IPropertyTree *xgmml = ret->queryPropTree(xpath.str());
- if (xgmml) // don't know of any reason it shouldn't exist
- {
- xgmml->removeProp("graphBin");
- xgmml->setPropTree("graph", graphTree.getClear());
- }
- }
- return ret.getClear();
- }
- void CLocalWorkUnit::loadGraphs() const
- {
- CriticalBlock block(crit);
- if (!cachedGraphs.get())
- {
- MemoryBuffer buf;
- IPropertyTree *t = p->queryPropTree("PackedGraphs");
- if (t&&t->getPropBin(NULL,buf)) {
- cachedGraphs.setown(createPTree(buf));
- }
- else
- cachedGraphs.set(p->queryPropTree("Graphs"));
- if (cachedGraphs.get())
- {
- Owned<IPropertyTreeIterator> iter = cachedGraphs->getElements("Graph");
- ForEach(*iter)
- {
- IPropertyTree &graph = iter->query();
- graphs.append(*new CLocalWUGraph(*this, LINK(&graph)));
- }
- }
- }
- }
- mapEnums graphTypes[] = {
- { GraphTypeAny, "unknown" },
- { GraphTypeProgress, "progress" },
- { GraphTypeEcl, "ECL" },
- { GraphTypeActivities, "activities" },
- { GraphTypeSubProgress, "subgraph" },
- { GraphTypeSize, NULL },
- };
- CLocalWUGraph::CLocalWUGraph(const CLocalWorkUnit &_owner, IPropertyTree *props) : p(props), owner(_owner)
- {
- wuidVersion = owner.getWuidVersion();
- }
- IStringVal& CLocalWUGraph::getName(IStringVal &str) const
- {
- str.set(p->queryProp("@name"));
- return str;
- }
- IStringVal& CLocalWUGraph::getLabel(IStringVal &str) const
- {
- if (wuidVersion >= 2)
- {
- str.set(p->queryProp("@label"));
- return str;
- }
- else
- {
- Owned<IPropertyTree> xgmml = getXGMMLTree(false);
- str.set(xgmml->queryProp("@label"));
- return str;
- }
- }
- WUGraphState CLocalWUGraph::getState() const
- {
- Owned<IConstWUGraphProgress> graphProgress = owner.getGraphProgress(p->queryProp("@name"));
- if (!graphProgress)
- return WUGraphUnknown;
- return graphProgress->queryGraphState();
- }
- IStringVal& CLocalWUGraph::getXGMML(IStringVal &str, bool mergeProgress) const
- {
- Owned<IPropertyTree> xgmml = getXGMMLTree(mergeProgress);
- if (xgmml)
- {
- StringBuffer x;
- toXML(xgmml, x);
- str.set(x.str());
- }
- return str;
- }
- unsigned CLocalWorkUnit::getGraphCount() const
- {
- CriticalBlock block(crit);
- if (p->hasProp("Graphs"))
- {
- return p->queryPropTree("Graphs")->numChildren();
- }
- return 0;
- }
- unsigned CLocalWorkUnit::getSourceFileCount() const
- {
- CriticalBlock block(crit);
- if (p->hasProp("FilesRead"))
- {
- return p->queryPropTree("FilesRead")->numChildren();
- }
- return 0;
-
- }
- unsigned CLocalWorkUnit::getResultCount() const
- {
- CriticalBlock block(crit);
- if (p->hasProp("Results"))
- {
- return p->queryPropTree("Results")->numChildren();
- }
- return 0;
-
- }
- unsigned CLocalWorkUnit::getVariableCount() const
- {
- CriticalBlock block(crit);
- if (p->hasProp("Variables"))
- {
- return p->queryPropTree("Variables")->numChildren();
- }
- return 0;
-
- }
- unsigned CLocalWorkUnit::getApplicationValueCount() const
- {
- CriticalBlock block(crit);
- if (p->hasProp("Application"))
- {
- return p->queryPropTree("Application")->numChildren();
- }
- return 0;
-
- }
- IStringVal &CLocalWorkUnit::getXmlParams(IStringVal &str) const
- {
- CriticalBlock block(crit);
- IPropertyTree *paramTree = p->queryPropTree("Parameters");
- if (paramTree)
- {
- StringBuffer temp;
- toXML(paramTree, temp);
- str.set(temp.str());
- }
- return str;
- }
- const IPropertyTree *CLocalWorkUnit::getXmlParams() const
- {
- CriticalBlock block(crit);
- return p->getPropTree("Parameters");
- }
- void CLocalWorkUnit::setXmlParams(const char *params)
- {
- CriticalBlock block(crit);
- p->setPropTree("Parameters", createPTreeFromXMLString(params));
- }
- void CLocalWorkUnit::setXmlParams(IPropertyTree *tree)
- {
- CriticalBlock block(crit);
- p->setPropTree("Parameters", tree);
- }
- unsigned __int64 CLocalWorkUnit::getHash() const
- {
- CriticalBlock block(crit);
- return p->getPropInt64("@hash");
- }
- void CLocalWorkUnit::setHash(unsigned __int64 hash)
- {
- CriticalBlock block(crit);
- p->setPropInt64("@hash", hash);
- }
- IConstWUGraphMetaIterator& CLocalWorkUnit::getGraphsMeta(WUGraphType type) const
- {
- /* NB: this method should be 'cheap', loadGraphs() creates IConstWUGraph interfaces to the graphs
- * it does not actually pull the graph data. We only use IConstWUGraphMeta here, which never probes the xgmml
- * This method also connects to the graph progress (/GraphProgress/<wuid>) using a single connection, in order
- * to get state information, it does not pull all the progress data.
- */
- CriticalBlock block(crit);
- loadGraphs();
- class CConstWUGraphMetaIterator: public CInterface, implements IConstWUGraphMetaIterator, implements IConstWUGraphMeta
- {
- StringAttr wuid;
- WUGraphType type;
- Owned<IConstWUGraphIterator> graphIter;
- IConstWUGraph *curGraph;
- Owned<IConstWUGraphProgress> curGraphProgress;
- Owned<IRemoteConnection> progressConn;
- bool match()
- {
- return (GraphTypeAny == type) || (type == graphIter->query().getType());
- }
- void setCurrent(IConstWUGraph &graph)
- {
- curGraph = &graph;
- SCMStringBuffer graphName;
- curGraph->getName(graphName);
- if (progressConn)
- {
- IPropertyTree *progress = progressConn->queryRoot()->queryPropTree(graphName.str());
- if (progress)
- {
- curGraphProgress.setown(new CConstGraphProgress(wuid, graphName.str(), progress));
- return;
- }
- }
- curGraphProgress.clear();
- }
- public:
- IMPLEMENT_IINTERFACE;
- CConstWUGraphMetaIterator(const char *_wuid, IConstWUGraphIterator *_graphIter, WUGraphType _type)
- : wuid(_wuid), graphIter(_graphIter), type(_type)
- {
- curGraph = NULL;
- StringBuffer progressPath;
- progressPath.append("/GraphProgress/").append(wuid);
- progressConn.setown(querySDS().connect(progressPath.str(), myProcessSession(), RTM_NONE, SDS_LOCK_TIMEOUT));
- }
- virtual bool first()
- {
- curGraph = NULL;
- curGraphProgress.clear();
- if (!graphIter->first())
- return false;
- if (match())
- {
- setCurrent(graphIter->query());
- return true;
- }
- return next();
- }
- virtual bool next()
- {
- while (graphIter->next())
- {
- if (match())
- {
- setCurrent(graphIter->query());
- return true;
- }
- }
- curGraph = NULL;
- curGraphProgress.clear();
- return false;
- }
- virtual bool isValid()
- {
- return NULL != curGraph;
- }
- virtual IConstWUGraphMeta & query()
- {
- return *this;
- }
- // IConstWUGraphMeta
- virtual IStringVal & getName(IStringVal & ret) const
- {
- return curGraph->getName(ret);
- }
- virtual IStringVal & getLabel(IStringVal & ret) const
- {
- return curGraph->getLabel(ret);
- }
- virtual IStringVal & getTypeName(IStringVal & ret) const
- {
- return curGraph->getTypeName(ret);
- }
- virtual WUGraphType getType() const
- {
- return curGraph->getType();
- }
- virtual WUGraphState getState() const
- {
- if (!curGraphProgress)
- return WUGraphUnknown;
- return curGraphProgress->queryGraphState();
- }
- };
- IConstWUGraphIterator *graphIter = new CArrayIteratorOf<IConstWUGraph,IConstWUGraphIterator> (graphs, 0, (IConstWorkUnit *) this);
- return * new CConstWUGraphMetaIterator(p->queryName(), graphIter, type);
- }
- IConstWUGraphIterator& CLocalWorkUnit::getGraphs(WUGraphType type) const
- {
- CriticalBlock block(crit);
- loadGraphs();
- IConstWUGraphIterator *giter = new CArrayIteratorOf<IConstWUGraph,IConstWUGraphIterator> (graphs, 0, (IConstWorkUnit *) this);
- if (type!=GraphTypeAny) {
- class CConstWUGraphIterator: public CInterface, implements IConstWUGraphIterator
- {
- WUGraphType type;
- Owned<IConstWUGraphIterator> base;
- bool match()
- {
- return base->query().getType()==type;
- }
- public:
- IMPLEMENT_IINTERFACE;
- CConstWUGraphIterator(IConstWUGraphIterator *_base,WUGraphType _type)
- : base(_base)
- {
- type = _type;
- }
- bool first()
- {
- if (!base->first())
- return false;
- if (match())
- return true;
- return next();
- }
- bool next()
- {
- while (base->next())
- if (match())
- return true;
- return false;
- }
- virtual bool isValid()
- {
- return base->isValid();
- }
- IConstWUGraph & query()
- {
- return base->query();
- }
- };
- giter = new CConstWUGraphIterator(giter,type);
- }
- return *giter;
- }
- IConstWUGraph* CLocalWorkUnit::getGraph(const char *qname) const
- {
- CriticalBlock block(crit);
- loadGraphs();
- ForEachItemIn(idx, graphs)
- {
- SCMStringBuffer name;
- IConstWUGraph &cur = graphs.item(idx);
- cur.getName(name);
- if (stricmp(name.str(), qname)==0)
- {
- cur.Link();
- return &cur;
- }
- }
- return NULL;
- }
- IWUGraph* CLocalWorkUnit::createGraph()
- {
- // For this to be legally called, we must have the write-able interface. So we are already locked for write.
- CriticalBlock block(crit);
- ensureGraphsUnpacked();
- loadGraphs();
- if (!graphs.ordinality())
- p->addPropTree("Graphs", createPTree("Graphs"));
- IPropertyTree *r = p->queryPropTree("Graphs");
- IPropertyTree *s = r->addPropTree("Graph", createPTree());
- s->Link();
- IWUGraph* q = new CLocalWUGraph(*this, s);
- q->Link();
- graphs.append(*q);
- return q;
- }
- IWUGraph * CLocalWorkUnit::createGraph(const char * name, WUGraphType type, IPropertyTree *xgmml)
- {
- CriticalBlock block(crit);
- ensureGraphsUnpacked();
- Linked<IConstWUGraph> existing = getGraph(name);
- if (existing)
- throwUnexpected();
- if (!graphs.length())
- p->addPropTree("Graphs", createPTree("Graphs"));
- IPropertyTree *r = p->queryPropTree("Graphs");
- IPropertyTree *s = r->addPropTree("Graph", createPTree());
- IWUGraph* q = new CLocalWUGraph(*this, LINK(s));
- graphs.append(*LINK(q));
- q->setName(name);
- q->setXGMMLTree(xgmml);
- q->setType(type);
- return q;
- }
- IWUGraph * CLocalWorkUnit::updateGraph(const char * name)
- {
- CriticalBlock block(crit);
- ensureGraphsUnpacked();
- return (IWUGraph *)getGraph(name);
- }
- IConstWUGraphProgress *CLocalWorkUnit::getGraphProgress(const char *name) const
- {
- CriticalBlock block(crit);
- return new CConstGraphProgress(p->queryName(), name);
- }
- void CLocalWUGraph::setName(const char *str)
- {
- p->setProp("@name", str);
- progress.clear();
- progress.setown(new CConstGraphProgress(owner.queryWuid(), str));
- }
- void CLocalWUGraph::setLabel(const char *str)
- {
- p->setProp("@label", str);
- }
- void CLocalWUGraph::setXGMML(const char *str)
- {
- setXGMMLTree(createPTreeFromXMLString(str));
- }
- void CLocalWUGraph::setXGMMLTree(IPropertyTree *_graph, bool compress)
- {
- assertex(strcmp(_graph->queryName(), "graph")==0);
- IPropertyTree *xgmml = p->setPropTree("xgmml", createPTree());
- if (compress)
- {
- MemoryBuffer mb;
- _graph->serialize(mb);
- xgmml->setPropBin("graphBin", mb.length(), mb.toByteArray());
- graph.setown(_graph);
- }
- else
- xgmml->setPropTree("graph", _graph);
- }
- static void expandAttributes(IPropertyTree & targetNode, IPropertyTree & progressNode)
- {
- Owned<IAttributeIterator> aIter = progressNode.getAttributes();
- ForEach (*aIter)
- {
- const char *aName = aIter->queryName()+1;
- if (0 != stricmp("id", aName)) // "id" reserved.
- {
- IPropertyTree *att = targetNode.addPropTree("att", createPTree());
- att->setProp("@name", aName);
- att->setProp("@value", aIter->queryValue());
- }
- }
- }
- void CLocalWUGraph::mergeProgress(IPropertyTree &rootNode, IPropertyTree &progressTree, const unsigned &progressV) const
- {
- IPropertyTree *graphNode = rootNode.queryPropTree("att/graph");
- if (!graphNode) return;
- unsigned nodeId = rootNode.getPropInt("@id");
- StringBuffer progressNodePath("node[@id=\"");
- progressNodePath.append(nodeId).append("\"]");
- IPropertyTree *progressNode = progressTree.queryPropTree(progressNodePath.str());
- if (progressNode)
- {
- expandAttributes(*graphNode, *progressNode);
- Owned<IPropertyTreeIterator> edges = progressNode->getElements("edge");
- ForEach (*edges)
- {
- IPropertyTree &edge = edges->query();
- StringBuffer edgePath("edge[@id=\"");
- edgePath.append(edge.queryProp("@id")).append("\"]");
- IPropertyTree *graphEdge = graphNode->queryPropTree(edgePath.str());
- if (graphEdge)
- {
- if (progressV < 1)
- mergePTree(graphEdge, &edge);
- else
- { // must translate to XGMML format
- expandAttributes(*graphEdge, edge);
- // This is really only here, so that our progress format can use non-attribute values, which have different efficiency qualifies (e.g. can be external by dali)
- Owned<IPropertyTreeIterator> iter = edge.getElements("*");
- ForEach (*iter)
- {
- IPropertyTree &t = iter->query();
- IPropertyTree *att = graphEdge->addPropTree("att", createPTree());
- att->setProp("@name", t.queryName());
- att->setProp("@value", t.queryProp(NULL));
- }
- }
- }
- }
- Owned<IPropertyTreeIterator> nodes = progressNode->getElements("node");
- ForEach (*nodes)
- {
- IPropertyTree &node = nodes->query();
- StringBuffer nodePath("node[@id=\"");
- nodePath.append(node.queryProp("@id")).append("\"]");
- IPropertyTree *_node = graphNode->queryPropTree(nodePath.str());
- if (_node)
- {
- if (progressV < 1)
- mergePTree(_node, &node);
- else
- { // must translate to XGMML format
- expandAttributes(*_node, node);
- }
- }
- }
- }
- Owned<IPropertyTreeIterator> iter = graphNode->getElements("node");
- ForEach (*iter)
- mergeProgress(iter->query(), progressTree, progressV);
- }
- IPropertyTree * CLocalWUGraph::getXGMMLTreeRaw() const
- {
- return p->getPropTree("xgmml");
- }
- IPropertyTree * CLocalWUGraph::getXGMMLTree(bool doMergeProgress) const
- {
- if (!graph)
- {
- // NB: although graphBin introduced in wuidVersion==2,
- // daliadmin can retrospectively compress existing graphs, so need to check for all versions
- MemoryBuffer mb;
- if (p->getPropBin("xgmml/graphBin", mb))
- graph.setown(createPTree(mb));
- else
- graph.setown(p->getBranch("xgmml/graph"));
- if (!graph)
- return NULL;
- }
- if (!doMergeProgress)
- return graph.getLink();
- else
- {
- Owned<IPropertyTree> copy = createPTreeFromIPT(graph);
- Owned<IConstWUGraphProgress> _progress;
- if (progress) _progress.set(progress);
- else
- _progress.setown(new CConstGraphProgress(owner.queryWuid(), p->queryProp("@name")));
- //MORE: Eventually this should directly access the new stats structure
- unsigned progressV = _progress->queryFormatVersion();
- Owned<IPropertyTree> progressTree = _progress->getProgressTree();
- Owned<IPropertyTreeIterator> nodeIterator = copy->getElements("node");
- ForEach (*nodeIterator)
- mergeProgress(nodeIterator->query(), *progressTree, progressV);
- return copy.getClear();
- }
- }
- WUGraphType CLocalWUGraph::getType() const
- {
- return (WUGraphType) getEnum(p, "@type", graphTypes);
- }
- IStringVal & CLocalWUGraph::getTypeName(IStringVal &str) const
- {
- str.set(p->queryProp("@type"));
- if (!str.length())
- str.set("unknown");
- return str;
- }
- void CLocalWUGraph::setType(WUGraphType _type)
- {
- setEnum(p, "@type", _type, graphTypes);
- }
- //=================================================================================================
- mapEnums queryFileTypes[] = {
- { FileTypeCpp, "cpp" },
- { FileTypeDll, "dll" },
- { FileTypeResText, "res" },
- { FileTypeHintXml, "hint" },
- { FileTypeXml, "xml" },
- { FileTypeSize, NULL },
- };
- CLocalWUAssociated::CLocalWUAssociated(IPropertyTree *props) : p(props)
- {
- }
- WUFileType CLocalWUAssociated::getType() const
- {
- return (WUFileType)getEnum(p, "@type", queryFileTypes);
- }
- IStringVal & CLocalWUAssociated::getDescription(IStringVal & str) const
- {
- str.set(p->queryProp("@desc"));
- return str;
- }
- IStringVal & CLocalWUAssociated::getIp(IStringVal & str) const
- {
- str.set(p->queryProp("@ip"));
- return str;
- }
- IStringVal & CLocalWUAssociated::getName(IStringVal & str) const
- {
- str.set(p->queryProp("@filename"));
- return str;
- }
- IStringVal & CLocalWUAssociated::getNameTail(IStringVal & str) const
- {
- str.set(pathTail(p->queryProp("@filename")));
- return str;
- }
- unsigned CLocalWUAssociated::getCrc() const
- {
- return p->getPropInt("@crc", 0);
- }
- //=================================================================================================
- CLocalWUQuery::CLocalWUQuery(IPropertyTree *props) : p(props)
- {
- associatedCached = false;
- }
- mapEnums queryTypes[] = {
- { QueryTypeUnknown, "unknown" },
- { QueryTypeEcl, "ECL" },
- { QueryTypeSql, "SQL" },
- { QueryTypeXml, "XML" },
- { QueryTypeAttribute, "Attribute" },
- { QueryTypeSize, NULL },
- };
- WUQueryType CLocalWUQuery::getQueryType() const
- {
- return (WUQueryType) getEnum(p, "@type", queryTypes);
- }
- void CLocalWUQuery::setQueryType(WUQueryType qt)
- {
- setEnum(p, "@type", qt, queryTypes);
- }
- IStringVal& CLocalWUQuery::getQueryText(IStringVal &str) const
- {
- str.set(p->queryProp("Text"));
- return str;
- }
- IStringVal& CLocalWUQuery::getQueryShortText(IStringVal &str) const
- {
- const char * text = p->queryProp("Text");
- if (isArchiveQuery(text))
- {
- Owned<IPropertyTree> xml = createPTreeFromXMLString(text, ipt_caseInsensitive);
- const char * path = xml->queryProp("Query/@attributePath");
- if (path)
- {
- IPropertyTree * resolved = resolveDefinitionInArchive(xml, path);
- if (resolved)
- str.set(resolved->queryProp(NULL));
- }
- else
- str.set(xml->queryProp("Query"));
- }
- else
- str.set(text);
- return str;
- }
- IStringVal& CLocalWUQuery::getQueryName(IStringVal &str) const
- {
- str.set(p->queryProp("@name"));
- return str;
- }
- IStringVal & CLocalWUQuery::getQueryMainDefinition(IStringVal & str) const
- {
- str.set(p->queryProp("@main"));
- return str;
- }
- IStringVal& CLocalWUQuery::getQueryDllName(IStringVal &str) const
- {
- Owned<IConstWUAssociatedFile> entry = getAssociatedFile(FileTypeDll, 0);
- if (entry)
- entry->getNameTail(str);
- return str;
- }
- IStringVal& CLocalWUQuery::getQueryCppName(IStringVal &str) const
- {
- Owned<IConstWUAssociatedFile> entry = getAssociatedFile(FileTypeCpp, 0);
- if (entry)
- entry->getName(str);
- return str;
- }
- IStringVal& CLocalWUQuery::getQueryResTxtName(IStringVal &str) const
- {
- Owned<IConstWUAssociatedFile> entry = getAssociatedFile(FileTypeResText, 0);
- if (entry)
- entry->getName(str);
- return str;
- }
- unsigned CLocalWUQuery::getQueryDllCrc() const
- {
- Owned<IConstWUAssociatedFile> entry = getAssociatedFile(FileTypeDll, 0);
- if (entry)
- return entry->getCrc();
- return 0;
- }
- void CLocalWUQuery::setQueryText(const char *text)
- {
- p->setProp("Text", text);
- }
- void CLocalWUQuery::setQueryName(const char *qname)
- {
- p->setProp("@name", qname);
- }
- void CLocalWUQuery::setQueryMainDefinition(const char * str)
- {
- p->setProp("@main", str);
- }
- void CLocalWUQuery::addAssociatedFile(WUFileType type, const char * name, const char * ip, const char * desc, unsigned crc)
- {
- CriticalBlock block(crit);
- loadAssociated();
- if (!associated.length())
- p->addPropTree("Associated", createPTree("Associated"));
- IPropertyTree *pl = p->queryPropTree("Associated");
- IPropertyTree *s = pl->addPropTree("File", createPTree("File"));
- setEnum(s, "@type", type, queryFileTypes);
- s->setProp("@filename", name);
- s->setProp("@ip", ip);
- s->setProp("@desc", desc);
- if (crc)
- s->setPropInt("@crc", crc);
- IConstWUAssociatedFile * q = new CLocalWUAssociated(LINK(s));
- associated.append(*q);
- }
- void CLocalWUQuery::removeAssociatedFiles()
- {
- associatedCached = false;
- associated.kill();
- p->removeProp("Associated");
- }
- IConstWUAssociatedFile * CLocalWUQuery::getAssociatedFile(WUFileType type, unsigned index) const
- {
- CriticalBlock block(crit);
- loadAssociated();
- ForEachItemIn(idx, associated)
- {
- CLocalWUAssociated &cur = static_cast<CLocalWUAssociated &>(associated.item(idx));
- if (cur.getType() == type)
- {
- if (index-- == 0)
- return &OLINK(cur);
- }
- }
- return NULL;
- }
- void CLocalWUQuery::addSpecialCaseAssociated(WUFileType type, const char * propname, unsigned crc) const
- {
- const char * name = p->queryProp(propname);
- if (name)
- {
- IPropertyTree *s = createPTree("File");
- setEnum(s, "@type", type, queryFileTypes);
- s->setProp("@filename", name);
- if (crc)
- s->setPropInt("@crc", crc);
- associated.append(*new CLocalWUAssociated(s));
- }
- }
- void CLocalWUQuery::loadAssociated() const
- {
- CriticalBlock block(crit);
- if (!associatedCached)
- {
- assertex(associated.length() == 0);
- addSpecialCaseAssociated(FileTypeDll, "DllName", p->getPropInt("DllCrc", 0));
- addSpecialCaseAssociated(FileTypeCpp, "CppName", 0);
- addSpecialCaseAssociated(FileTypeResText, "ResTxtName", 0);
- Owned<IPropertyTreeIterator> r = p->getElements("Associated/File");
- for (r->first(); r->isValid(); r->next())
- {
- IPropertyTree *rp = &r->query();
- rp->Link();
- associated.append(*new CLocalWUAssociated(rp));
- }
- associatedCached = true;
- }
- }
- IConstWUAssociatedFileIterator& CLocalWUQuery::getAssociatedFiles() const
- {
- CriticalBlock block(crit);
- loadAssociated();
- return *new CArrayIteratorOf<IConstWUAssociatedFile,IConstWUAssociatedFileIterator> (associated, 0, (IConstWUQuery *) this);
- }
- //========================================================================================
- CLocalWUWebServicesInfo::CLocalWUWebServicesInfo(IPropertyTree *props) : p(props)
- {
- }
- IStringVal& CLocalWUWebServicesInfo::getModuleName(IStringVal &str) const
- {
- str.set(p->queryProp("@module"));
- return str;
- }
- IStringVal& CLocalWUWebServicesInfo::getAttributeName(IStringVal &str) const
- {
- str.set(p->queryProp("@attribute"));
- return str;
- }
- IStringVal& CLocalWUWebServicesInfo::getDefaultName(IStringVal &str) const
- {
- str.set(p->queryProp("@defaultName"));
- return str;
- }
- unsigned CLocalWUWebServicesInfo::getWebServicesCRC() const
- {
- return (unsigned) p->getPropInt("@crc");
- }
- IStringVal& CLocalWUWebServicesInfo::getInfo(const char *name, IStringVal &str) const
- {
- if (!name)
- {
- StringBuffer ws_info;
- ws_info.appendf("<%s ", p->queryName());
- Owned<IAttributeIterator> attrs = p->getAttributes();
- for(attrs->first(); attrs->isValid(); attrs->next())
- {
- const char *name = attrs->queryName()+1;
- const char *value = attrs->queryValue();
- ws_info.appendf("%s='%s' ", name, value);
- }
- ws_info.append("> \n");
- Owned<IPropertyTreeIterator> info = p->getElements("*");
- ForEach(*info)
- {
- IPropertyTree &item = info->query();
- const char *name = item.queryName();
- if (name)
- {
- MemoryBuffer mb;
- bool isbin = p->isBinary(name);
- if (isbin)
- {
- p->getPropBin(name,mb);
- if (mb.length())
- {
- unsigned len = 0;
- mb.read(len);
- StringBuffer encodedString;
- StringBuffer val(len, (const char *) mb.readDirect(len));
- encodeXML(val, encodedString);
- ws_info.appendf("<%s>%s</%s>", name, encodedString.str(), name);
- }
- }
- else
- {
- StringBuffer tmp;
- toXML(&item, tmp);
- ws_info.append(tmp.str());
- }
- }
- }
- ws_info.appendf("</%s>", p->queryName());
- str.setLen(ws_info.str(), ws_info.length());
- }
- else
- {
- MemoryBuffer mb;
- p->getPropBin(name,mb);
- if (mb.length())
- {
- unsigned len;
- mb.read(len);
- str.setLen((const char *) mb.readDirect(len), len);
- }
- }
- return str;
- }
- void CLocalWUWebServicesInfo::setModuleName(const char *mname)
- {
- p->setProp("@module", mname);
- }
- void CLocalWUWebServicesInfo::setAttributeName(const char *aname)
- {
- p->setProp("@attribute", aname);
- }
- void CLocalWUWebServicesInfo::setDefaultName(const char *dname)
- {
- p->setProp("@defaultName", dname);
- }
- void CLocalWUWebServicesInfo::setWebServicesCRC(unsigned crc)
- {
- p->setPropInt("@crc", crc);
- }
- void CLocalWUWebServicesInfo::setInfo(const char *name, const char *info)
- {
- MemoryBuffer m;
- unsigned len = (size32_t)strlen(info);
- serializeLPString(len, info, m);
- p->setPropBin(name, m.length(), m.toByteArray());
- }
- //========================================================================================
- CLocalWURoxieQueryInfo::CLocalWURoxieQueryInfo(IPropertyTree *props) : p(props)
- {
- }
- IStringVal& CLocalWURoxieQueryInfo::getQueryInfo(IStringVal &str) const
- {
- IPropertyTree *queryTree = p->queryPropTree("query");
- if (queryTree)
- {
- StringBuffer temp;
- toXML(queryTree, temp);
- str.set(temp.str());
- }
- return str;
- }
- IStringVal& CLocalWURoxieQueryInfo::getDefaultPackageInfo(IStringVal &str) const
- {
- MemoryBuffer mb;
- p->getPropBin("RoxiePackages",mb);
- if (mb.length())
- {
- unsigned len;
- mb.read(len);
- str.setLen((const char *) mb.readDirect(len), len);
- }
- return str;
- }
- IStringVal& CLocalWURoxieQueryInfo::getRoxieClusterName(IStringVal &str) const
- {
- const char *val = p->queryProp("@roxieClusterName");
- if (val)
- str.set(val);
-
- return str;
- }
- IStringVal& CLocalWURoxieQueryInfo::getWuid(IStringVal &str) const
- {
- const char *val = p->queryProp("@wuid");
- if (val)
- str.set(val);
- return str;
- }
- void CLocalWURoxieQueryInfo::setQueryInfo(const char *info)
- {
- IPropertyTree *queryTree = p->queryPropTree("query");
- if (queryTree)
- p->removeTree(queryTree);
- IPropertyTree * tempTree = p->addPropTree("query", createPTreeFromXMLString(info));
- if (!p->hasProp("@roxieClusterName"))
- {
- const char *roxieClusterName = tempTree->queryProp("@roxieName");
- if (roxieClusterName && *roxieClusterName)
- p->addProp("@roxieClusterName", roxieClusterName);
- }
- if (!p->hasProp("@wuid"))
- {
- const char *wuid = tempTree->queryProp("Query/@wuid");
- if (wuid && *wuid)
- p->addProp("@wuid", wuid);
- }
- }
- void CLocalWURoxieQueryInfo::setDefaultPackageInfo(const char *info, int len)
- {
- MemoryBuffer m;
- serializeLPString(len, info, m);
- p->setPropBin("RoxiePackages", m.length(), m.toByteArray());
- }
- void CLocalWURoxieQueryInfo::setRoxieClusterName(const char *info)
- {
- p->setProp("@roxieClusterName", info);
- }
- void CLocalWURoxieQueryInfo::setWuid(const char *info)
- {
- p->setProp("@wuid", info);
- }
- //========================================================================================
- CLocalWUResult::CLocalWUResult(IPropertyTree *props) : p(props)
- {
- }
- mapEnums resultStatuses[] = {
- { ResultStatusUndefined, "undefined" },
- { ResultStatusCalculated, "calculated" },
- { ResultStatusSupplied, "supplied" },
- { ResultStatusFailed, "failed" },
- { ResultStatusPartial, "partial" },
- { ResultStatusSize, NULL }
- };
- WUResultStatus CLocalWUResult::getResultStatus() const
- {
- return (WUResultStatus ) getEnum(p, "@status", resultStatuses);
- }
- IStringVal& CLocalWUResult::getResultName(IStringVal &str) const
- {
- str.set(p->queryProp("@name"));
- return str;
- }
- int CLocalWUResult::getResultSequence() const
- {
- return p->getPropInt("@sequence", -1);
- }
- bool CLocalWUResult::isResultScalar() const
- {
- return p->getPropInt("@isScalar", 1) != 0;
- }
- bool findSize(int size, IntArray &sizes)
- {
- ForEachItemIn(idx, sizes)
- {
- if (sizes.item(idx)==size)
- return true;
- }
- return false;
- }
- void CLocalWUResult::getSchema(TypeInfoArray &types, StringAttrArray &names, IStringVal * eclText) const
- {
- MemoryBuffer schema;
- p->getPropBin("SchemaRaw", schema);
- if (schema.length())
- {
- for (;;)
- {
- StringAttr name;
- schema.read(name);
- if (*schema.readDirect(0)==type_void)
- break;
- names.append(*new StringAttrItem(name));
- types.append(*deserializeType(schema)); // MORE - nested records!
- }
- schema.skip(1);
- if (schema.length() != schema.getPos())
- {
- unsigned eclLen;
- schema.read(eclLen);
- const char * schemaData = (const char *)schema.readDirect(eclLen);
- if (eclText)
- {
- eclText->setLen(schemaData, eclLen);
- if ((eclLen == 0) && names.ordinality())
- {
- const char * firstName = names.item(0).text;
- StringBuffer temp;
- temp.append("RECORD ");
- types.item(0).getECLType(temp);
- temp.append(" value{NAMED('").append(firstName).append("')}").append("; END;");
- eclText->set(temp.str());
- }
- }
- }
- }
- }
- void readRow(StringBuffer &out, MemoryBuffer &in, TypeInfoArray &types, StringAttrArray &names)
- {
- ForEachItemIn(idx, types)
- {
- StringAttrItem &name = names.item(idx);
- ITypeInfo &type = types.item(idx);
- unsigned size = type.getSize();
- switch(type.getTypeCode())
- {
- case type_data:
- if (size==UNKNOWN_LENGTH)
- {
- if (in.remaining() < sizeof(int))
- throw MakeStringException(WUERR_CorruptResult, "corrupt workunit information");
- in.read(size);
- }
- outputXmlData(size, in.readDirect(size), name.text, out);
- break;
- case type_string:
- if (size==UNKNOWN_LENGTH)
- {
- if (in.remaining() < sizeof(int))
- throw MakeStringException(WUERR_CorruptResult, "corrupt workunit information");
- in.read(size);
- }
- outputXmlString(size, (const char *) in.readDirect(size), name.text, out);
- break;
- case type_varstring:
- {
- if (size == UNKNOWN_LENGTH)
- size = (size32_t)strlen((const char *) in.readDirect(0))+1;
- const char * text = (const char *) in.readDirect(size);
- outputXmlString((size32_t)strlen(text), text, name.text, out);
- break;
- }
- case type_unicode:
- {
- unsigned len = type.getStringLen();
- if (size==UNKNOWN_LENGTH)
- in.read(len);
- outputXmlUnicode(len, (UChar const *) in.readDirect(len*2), name.text, out);
- }
- break;
- case type_utf8:
- {
- unsigned len = type.getStringLen();
- if (size==UNKNOWN_LENGTH)
- {
- in.read(len);
- size = rtlUtf8Size(len, in.readDirect(0));
- }
- outputXmlUtf8(len, (const char *) in.readDirect(size), name.text, out);
- }
- break;
- case type_qstring:
- {
- unsigned len = type.getStringLen();
- if (size==UNKNOWN_LENGTH)
- in.read(len);
- unsigned outlen;
- char *outstr;
- rtlQStrToStrX(outlen, outstr, len, (const char *) in.readDirect(rtlQStrSize(len)));
- outputXmlString(outlen, outstr, name.text, out);
- free(outstr);
- break;
- }
- case type_int:
- case type_swapint:
- if (type.isSigned())
- {
- const unsigned char *raw = (const unsigned char *) in.readDirect(size);
- unsigned __int64 cval8 = 0;
- //MORE: I think this is wrong - swapped doesn't mean little/big/
- if (type.isSwappedEndian())
- {
- unsigned idx = 0;
- if (raw[idx] & 0x80)
- cval8 = (__int64)-1;
- while (size--)
- cval8 = (cval8 << 8) | raw[idx++];
- }
- else
- {
- if (raw[size-1] & 0x80)
- cval8 = (__int64)-1;
- while (size--)
- cval8 = (cval8 << 8) | raw[size];
- }
- outputXmlInt((__int64) cval8, name.text, out);
- }
- else
- {
- const unsigned char *raw = (const unsigned char *) in.readDirect(size);
- unsigned __int64 cval8 = 0;
- if (type.isSwappedEndian())
- {
- unsigned idx = 0;
- while (size--)
- cval8 = (cval8 << 8) | raw[idx++];
- }
- else
- {
- while (size--)
- cval8 = (cval8 << 8) | raw[size];
- }
- outputXmlUInt(cval8, name.text, out);
- }
- break;
- case type_boolean:
- bool cvalb;
- in.read(cvalb);
- outputXmlBool(cvalb, name.text, out);
- break;
- case type_decimal:
- if (type.isSigned())
- outputXmlDecimal(in.readDirect(size), size, type.getPrecision(), name.text, out);
- else
- outputXmlUDecimal(in.readDirect(size), size, type.getPrecision(), name.text, out);
- break;
- case type_real:
- double cvald;
- switch(size)
- {
- case 4:
- float cvalf;
- in.read(cvalf);
- cvald = cvalf;
- break;
- case 8:
- in.read(cvald);
- break;
- }
- outputXmlReal(cvald, name.text, out);
- break;
- default:
- assertex(!"unexpected type in raw record");
- break;
- }
- }
- }
- IStringVal& CLocalWUResult::getResultXml(IStringVal &str) const
- {
- TypeInfoArray types;
- StringAttrArray names;
- getSchema(types, names);
- StringBuffer xml;
- const char * name = p->queryProp("@name");
- if (name)
- xml.appendf("<Dataset name=\'%s\'>\n", name);
- else
- xml.append("<Dataset>\n");
- if (p->hasProp("Value"))
- {
- MemoryBuffer raw;
- p->getPropBin("Value", raw);
- unsigned __int64 numrows = getResultRowCount();
- while (numrows--)
- {
- xml.append(" <Row>");
- readRow(xml, raw, types, names);
- xml.append("</Row>\n");
- }
- }
- else if (p->hasProp("xmlValue"))
- {
- xml.append(" <Row>");
- appendXMLTag(xml, name, p->queryProp("xmlValue"));
- xml.append("</Row>\n");
- }
- xml.append("</Dataset>\n");
- str.set(xml.str());
- return str;
- }
- IProperties *CLocalWUResult::queryResultXmlns()
- {
- CriticalBlock block(crit);
- if (xmlns)
- return xmlns;
- xmlns.setown(createProperties());
- Owned<IAttributeIterator> it = p->getAttributes();
- unsigned prefixLen = strlen("@xmlns");
- ForEach(*it)
- {
- const char *name = it->queryName();
- if (!strncmp("@xmlns", name, prefixLen))
- {
- if (name[prefixLen]==':') //normal case
- xmlns->setProp(name+prefixLen+1, it->queryValue());
- else if (!name[prefixLen]) //special case, unprefixed namespace
- xmlns->setProp("xmlns", it->queryValue());
- }
- }
- return xmlns;
- }
- unsigned CLocalWUResult::getResultFetchSize() const
- {
- return p->getPropInt("fetchSize", 100);
- }
- __int64 CLocalWUResult::getResultTotalRowCount() const
- {
- return p->getPropInt64("totalRowCount", -1);
- }
- __int64 CLocalWUResult::getResultRowCount() const
- {
- return p->getPropInt64("rowCount", 0);
- }
- void CLocalWUResult::getResultDataset(IStringVal & ecl, IStringVal & defs) const
- {
- ecl.set(p->queryProp("datasetEcl"));
- defs.set(p->queryProp("datasetEclDefs"));
- }
- IStringVal& CLocalWUResult::getResultLogicalName(IStringVal & val) const
- {
- val.set(p->queryProp("logicalName"));
- return val;
- }
- IStringVal& CLocalWUResult::getResultKeyField(IStringVal & ecl) const
- {
- ecl.set(p->queryProp("keyField"));
- return ecl;
- }
- unsigned CLocalWUResult::getResultRequestedRows() const
- {
- return p->getPropInt("requestedRows", 1);
- }
- IStringVal& CLocalWUResult::getResultEclSchema(IStringVal & str) const
- {
- TypeInfoArray types;
- StringAttrArray names;
- getSchema(types, names, &str);
- return str;
- }
- IStringVal& CLocalWUResult::getResultRecordSizeEntry(IStringVal & str) const
- {
- str.set(p->queryProp("@recordSizeEntry"));
- return str;
- }
- IStringVal& CLocalWUResult::getResultTransformerEntry(IStringVal & str) const
- {
- str.set(p->queryProp("@transformerEntry"));
- return str;
- }
- __int64 CLocalWUResult::getResultRowLimit() const
- {
- return p->getPropInt64("@rowLimit");
- }
- IStringVal& CLocalWUResult::getResultFilename(IStringVal & str) const
- {
- str.set(p->queryProp("@tempFilename"));
- return str;
- }
- void CLocalWUResult::setResultStatus(WUResultStatus status)
- {
- setEnum(p, "@status", status, resultStatuses);
- if (status==ResultStatusUndefined)
- p->removeProp("Value");
- }
- void CLocalWUResult::setResultName(const char *s)
- {
- p->setProp("@name", s);
- }
- void CLocalWUResult::setResultSequence(unsigned seq)
- {
- p->setPropInt("@sequence", seq);
- }
- void CLocalWUResult::setResultSchemaRaw(unsigned size, const void *schema)
- {
- p->setPropBin("SchemaRaw", size, schema);
- }
- void CLocalWUResult::setResultXmlns(const char *prefix, const char *uri)
- {
- StringBuffer xpath("@xmlns");
- if (prefix && *prefix)
- xpath.append(':').append(prefix);
- p->setProp(xpath, uri);
- }
- void CLocalWUResult::setResultScalar(bool isScalar)
- {
- p->setPropInt("@isScalar", (int) isScalar);
- if (isScalar)
- setResultTotalRowCount(1);
- }
- void CLocalWUResult::setResultRaw(unsigned len, const void *data, WUResultFormat format)
- {
- p->setPropBin("Value", len, data);
- setResultStatus(ResultStatusSupplied);
- setResultFormat(format);
- }
- void CLocalWUResult::setResultFormat(WUResultFormat format)
- {
- switch (format)
- {
- case ResultFormatXml:
- p->setProp("@format","xml");
- break;
- case ResultFormatXmlSet:
- p->setProp("@format","xmlSet");
- break;
- case ResultFormatCsv:
- p->setProp("@format","csv");
- break;
- default:
- p->removeProp("@format");
- break;
- }
- }
- void CLocalWUResult::setResultXML(const char *val)
- {
- p->setProp("xmlValue", val);
- }
- void CLocalWUResult::addResultRaw(unsigned len, const void *data, WUResultFormat format)
- {
- p->appendPropBin("Value", len, data);
- setResultStatus(ResultStatusPartial);
- const char *existingFormat = p->queryProp("@format");
- const char *formatStr = NULL;
- switch (format)
- {
- case ResultFormatXml:
- formatStr = "xml";
- break;
- case ResultFormatXmlSet:
- formatStr = "xmlSet";
- break;
- case ResultFormatCsv:
- formatStr = "csv";
- break;
- default:
- p->removeProp("@format");
- break;
- }
- if (format)
- {
- if (existingFormat)
- {
- if (0 != stricmp(formatStr, existingFormat))
- throw MakeStringException(WUERR_ResultFormatMismatch, "addResult format %s, does not match existing format %s", formatStr, existingFormat);
- }
- else
- p->setProp("@format", formatStr);
- }
- }
- void CLocalWUResult::setResultFetchSize(unsigned rows)
- {
- p->setPropInt("fetchSize", rows);
- }
- void CLocalWUResult::setResultTotalRowCount(__int64 rows)
- {
- p->setPropInt64("totalRowCount", rows);
- }
- void CLocalWUResult::setResultRowCount(__int64 rows)
- {
- p->setPropInt64("rowCount", rows);
- }
- void CLocalWUResult::setResultDataset(const char *ecl, const char *defs)
- {
- p->setProp("datasetEcl", ecl);
- p->setProp("datasetEclDefs", defs);
- }
- void CLocalWUResult::setResultLogicalName(const char *logicalName)
- {
- p->setProp("logicalName", logicalName);
- }
- void CLocalWUResult::setResultKeyField(const char *ecl)
- {
- p->setProp("keyField", ecl);
- }
- void CLocalWUResult::setResultRequestedRows(unsigned rows)
- {
- p->setPropInt("requestedRows", rows);
- }
- void CLocalWUResult::setResultRecordSizeEntry(const char * entry)
- {
- p->setProp("@recordSizeEntry", entry);
- }
- void CLocalWUResult::setResultTransformerEntry(const char * entry)
- {
- p->setProp("@transformerEntry", entry);
- }
- void CLocalWUResult::setResultRowLimit(__int64 value)
- {
- p->setPropInt64("@rowLimit", value);
- }
- void CLocalWUResult::setResultFilename(const char * name)
- {
- p->setProp("@tempFilename", name);
- }
- // MORE - it's an undetected error if we call getResult... of a type that does not match schema
- __int64 CLocalWUResult::getResultInt() const
- {
- __int64 result = 0;
- MemoryBuffer s;
- p->getPropBin("Value", s);
- if (s.length())
- s.read(result);
- else
- result = p->getPropInt64("xmlValue");
- return result;
- }
- bool CLocalWUResult::getResultBool() const
- {
- bool result = false;
- MemoryBuffer s;
- p->getPropBin("Value", s);
- if (s.length())
- s.read(result);
- else
- result = p->getPropBool("xmlValue");
- return result;
- }
- double CLocalWUResult::getResultReal() const
- {
- double result = 0;
- MemoryBuffer s;
- p->getPropBin("Value", s);
- if (s.length())
- s.read(result);
- else
- {
- const char *xmlVal = p->queryProp("xmlValue");
- if (xmlVal)
- result = atof(xmlVal);
- }
- return result;
- }
- void CLocalWUResult::getResultDecimal(void * val, unsigned len, unsigned precision, bool isSigned) const
- {
- MemoryBuffer s;
- p->getPropBin("Value", s);
- if (s.length())
- {
- assertex(s.length() == len);
- s.read(len, val);
- }
- else
- {
- const char *xmlVal = p->queryProp("xmlValue");
- if (xmlVal)
- {
- Decimal d;
- d.setString(strlen(xmlVal), xmlVal);
- if (isSigned)
- d.getDecimal(len, precision, val);
- else
- d.getUDecimal(len, precision, val);
- }
- else
- memset(val, 0, len);
- }
- }
- IStringVal& CLocalWUResult::getResultString(IStringVal & str) const
- {
- MemoryBuffer s;
- p->getPropBin("Value", s);
- if (s.length())
- {
- unsigned len;
- s.read(len);
- str.setLen((const char *) s.readDirect(len), len);
- }
- else
- {
- p->getPropBin("xmlValue", s);
- if (p->isBinary("xmlValue"))
- str.setLen(s.toByteArray(), s.length());
- else
- {
- char *ascii = rtlUtf8ToVStr(rtlUtf8Length(s.length(), s.toByteArray()), s.toByteArray());
- str.set(ascii);
- rtlFree(ascii);
- }
- }
- return str;
- }
- WUResultFormat CLocalWUResult::getResultFormat() const
- {
- const char * format = p->queryProp("@format");
- if (!format)
- return ResultFormatRaw;
- else if (strcmp(format, "xml") == 0)
- return ResultFormatXml;
- else if (strcmp(format, "xmlSet") == 0)
- return ResultFormatXmlSet;
- else if (strcmp(format, "csv") == 0)
- return ResultFormatCsv;
- else
- throw MakeStringException(WUERR_InvalidResultFormat, "Unrecognised result format %s", format);
- }
- IDataVal& CLocalWUResult::getResultRaw(IDataVal & data, IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const
- {
- MemoryBuffer s;
- p->getPropBin("Value", s);
- unsigned len = s.length();
- if (len)
- {
- WUResultFormat format = getResultFormat();
- if (format == ResultFormatXml || format == ResultFormatXmlSet)
- {
- if (!xmlTransformer)
- throw MakeStringException(WUERR_MissingFormatTranslator, "No transformer supplied to translate XML format result");
- xmlTransformer->transform(data, len, s.readDirect(len), format == ResultFormatXml);
- }
- else if (format == ResultFormatCsv)
- {
- if (!csvTransformer)
- throw MakeStringException(WUERR_MissingFormatTranslator, "No transformer supplied to translate Csv format result");
- csvTransformer->transform(data, len, s.readDirect(len), true);
- }
- else
- data.setLen(s.readDirect(len), len);
- }
- else
- data.clear();
- return data;
- }
- unsigned CLocalWUResult::getResultHash() const
- {
- MemoryBuffer s;
- p->getPropBin("Value", s);
- unsigned len = s.length();
- const byte * data = (const byte *)s.toByteArray();
- return ~hashc(data, len, ~0);
- }
- IDataVal& CLocalWUResult::getResultUnicode(IDataVal & data) const
- {
- MemoryBuffer s;
- p->getPropBin("Value", s);
- if (s.length())
- {
- unsigned len;
- s.read(len);
- data.setLen(s.readDirect(len*2), len*2);
- }
- else
- {
- StringBuffer utf8;
- if (p->getProp("xmlValue", utf8))
- {
- unsigned outlen;
- UChar *out;
- rtlUtf8ToUnicodeX(outlen, out, utf8.length(), utf8.str());
- data.setLen(out, outlen*2);
- rtlFree(out);
- }
- else
- data.clear();
- }
- return data;
- }
- __int64 CLocalWUResult::getResultRawSize(IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const
- {
- WUResultFormat format = getResultFormat();
- if (format == ResultFormatRaw)
- {
- //MORE: This should not load the whole property...
- MemoryBuffer s;
- p->getPropBin("Value", s);
- return s.length();
- }
- else
- {
- MemoryBuffer temp;
- MemoryBuffer2IDataVal adaptor(temp);
- getResultRaw(adaptor, xmlTransformer, csvTransformer);
- return temp.length();
- }
- }
- IDataVal& CLocalWUResult::getResultRaw(IDataVal & data, __int64 from, __int64 length, IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const
- {
- WUResultFormat format = getResultFormat();
- if (format != ResultFormatRaw)
- {
- MemoryBuffer temp;
- MemoryBuffer2IDataVal adaptor(temp);
- getResultRaw(adaptor, xmlTransformer, csvTransformer);
- unsigned len = temp.length();
- if (from > len) from = len;
- if (from + length > len) length = len - from;
- data.setLen(temp.readDirect(len) + from, (size32_t)length);
- return data;
- }
- else
- {
- //MORE: This should not load the whole property, and should be different from the code above...
- MemoryBuffer s;
- p->getPropBin("Value", s);
- unsigned len = s.length();
- if (from > len) from = len;
- if (from + length > len) length = len - from;
- data.setLen(s.readDirect(len) + from, (size32_t)length);
- return data;
- }
- }
- bool CLocalWUResult::getResultIsAll() const
- {
- return p->getPropBool("@isAll", false);
- }
- // MORE - it's an undetected error if we call setResult... of a type that does not match schema
- void CLocalWUResult::setResultInt(__int64 val)
- {
- // Note: we always serialize scalar integer results as int8, and schema must reflect this
- MemoryBuffer m;
- serializeInt8(val, m);
- p->setPropBin("Value", m.length(), m.toByteArray());
- setResultRowCount(1);
- setResultTotalRowCount(1);
- }
- void CLocalWUResult::setResultUInt(unsigned __int64 val)
- {
- setResultInt((__int64) val);
- }
- void CLocalWUResult::setResultReal(double val)
- {
- // Note: we always serialize scalar real results as real8, and schema must reflect this
- MemoryBuffer m;
- serializeReal8(val, m);
- p->setPropBin("Value", m.length(), m.toByteArray());
- setResultRowCount(1);
- setResultTotalRowCount(1);
- }
- void CLocalWUResult::setResultBool(bool val)
- {
- MemoryBuffer m;
- serializeBool(val, m);
- p->setPropBin("Value", m.length(), m.toByteArray());
- setResultRowCount(1);
- setResultTotalRowCount(1);
- }
- void CLocalWUResult::setResultString(const char *val, unsigned len)
- {
- // Note: we always serialize scalar strings with length prefix, and schema must reflect this
- MemoryBuffer m;
- serializeLPString(len, val, m);
- p->setPropBin("Value", m.length(), m.toByteArray());
- setResultRowCount(1);
- setResultTotalRowCount(1);
- }
- void CLocalWUResult::setResultUnicode(const void *val, unsigned len)
- {
- // Note: we always serialize scalar strings with length prefix, and schema must reflect this
- MemoryBuffer m;
- m.append(len).append(len*2, val);
- p->setPropBin("Value", m.length(), m.toByteArray());
- setResultRowCount(1);
- setResultTotalRowCount(1);
- }
- void CLocalWUResult::setResultData(const void *val, unsigned len)
- {
- // Note: we always serialize scalar data with length prefix, and schema must reflect this
- MemoryBuffer m;
- serializeLPString(len, (const char *)val, m);
- p->setPropBin("Value", m.length(), m.toByteArray());
- setResultRowCount(1);
- setResultTotalRowCount(1);
- }
- void CLocalWUResult::setResultDecimal(const void *val, unsigned len)
- {
- // Note: serialized as data but with length known from schema
- MemoryBuffer m;
- serializeFixedData(len, val, m);
- p->setPropBin("Value", m.length(), m.toByteArray());
- setResultRowCount(1);
- setResultTotalRowCount(1);
- }
- void CLocalWUResult::setResultRow(unsigned len, const void * data)
- {
- p->setPropBin("Value", len, data);
- setResultRowCount(1);
- setResultTotalRowCount(1);
- setResultFormat(ResultFormatRaw);
- }
- void CLocalWUResult::setResultIsAll(bool value)
- {
- p->setPropBool("@isAll", value);
- }
- //==========================================================================================
- CLocalWUPlugin::CLocalWUPlugin(IPropertyTree *props) : p(props)
- {
- }
- IStringVal& CLocalWUPlugin::getPluginName(IStringVal &str) const
- {
- str.set(p->queryProp("@dllname"));
- return str;
- }
- IStringVal& CLocalWUPlugin::getPluginVersion(IStringVal &str) const
- {
- str.set(p->queryProp("@version"));
- return str;
- }
- void CLocalWUPlugin::setPluginName(const char *str)
- {
- p->setProp("@dllname", str);
- }
- void CLocalWUPlugin::setPluginVersion(const char *str)
- {
- p->setProp("@version", str);
- }
- //==========================================================================================
- CLocalWULibrary::CLocalWULibrary(IPropertyTree *props) : p(props)
- {
- }
- IStringVal& CLocalWULibrary::getName(IStringVal &str) const
- {
- str.set(p->queryProp("@name"));
- return str;
- }
- void CLocalWULibrary::setName(const char *str)
- {
- p->setProp("@name", str);
- }
- //==========================================================================================
- CLocalWUException::CLocalWUException(IPropertyTree *props) : p(props)
- {
- }
- IStringVal& CLocalWUException::getExceptionSource(IStringVal &str) const
- {
- str.set(p->queryProp("@source"));
- return str;
- }
- IStringVal& CLocalWUException::getExceptionMessage(IStringVal &str) const
- {
- str.set(p->queryProp(NULL));
- return str;
- }
- unsigned CLocalWUException::getExceptionCode() const
- {
- return p->getPropInt("@code", 0);
- }
- WUExceptionSeverity CLocalWUException::getSeverity() const
- {
- return (WUExceptionSeverity)p->getPropInt("@severity", ExceptionSeverityError);
- }
- IStringVal & CLocalWUException::getTimeStamp(IStringVal & dt) const
- {
- dt.set(p->queryProp("@time"));
- return dt;
- }
- IStringVal & CLocalWUException::getExceptionFileName(IStringVal & str) const
- {
- str.set(p->queryProp("@filename"));
- return str;
- }
- unsigned CLocalWUException::getExceptionLineNo() const
- {
- return p->getPropInt("@row", 0);
- }
- unsigned CLocalWUException::getExceptionColumn() const
- {
- return p->getPropInt("@col", 0);
- }
- void CLocalWUException::setExceptionSource(const char *str)
- {
- p->setProp("@source", str);
- }
- void CLocalWUException::setExceptionMessage(const char *str)
- {
- p->setProp(NULL, str);
- }
- void CLocalWUException::setExceptionCode(unsigned code)
- {
- p->setPropInt("@code", code);
- }
- void CLocalWUException::setSeverity(WUExceptionSeverity level)
- {
- p->setPropInt("@severity", level);
- }
- void CLocalWUException::setTimeStamp(const char *str)
- {
- p->setProp("@time", str);
- }
- void CLocalWUException::setExceptionFileName(const char *str)
- {
- p->setProp("@filename", str);
- }
- void CLocalWUException::setExceptionLineNo(unsigned r)
- {
- p->setPropInt("@row", r);
- }
- void CLocalWUException::setExceptionColumn(unsigned c)
- {
- p->setPropInt("@col", c);
- }
- //==========================================================================================
- CLocalWUAppValue::CLocalWUAppValue(IPropertyTree *props,unsigned child): p(props)
- {
- StringAttrBuilder propPath(prop);
- propPath.append("*[").append(child).append("]");
- }
- IStringVal & CLocalWUAppValue::getApplication(IStringVal & str) const
- {
- str.set(p->queryName());
- return str;
- }
- IStringVal & CLocalWUAppValue::getName(IStringVal & str) const
- {
- IPropertyTree* val=p->queryPropTree(prop.str());
- if(val)
- str.set(val->queryName());
- return str;
- }
- IStringVal & CLocalWUAppValue::getValue(IStringVal & str) const
- {
- str.set(p->queryProp(prop.str()));
- return str;
- }
- //==========================================================================================
- CLocalWUStatistic::CLocalWUStatistic(IPropertyTree *props) : p(props)
- {
- }
- IStringVal & CLocalWUStatistic::getCreator(IStringVal & str) const
- {
- const char * creator = p->queryProp("@creator");
- str.set(creator);
- return str;
- }
- IStringVal & CLocalWUStatistic::getDescription(IStringVal & str, bool createDefault) const
- {
- const char * desc = p->queryProp("@desc");
- if (desc)
- {
- str.set(desc); // legacy and in case it is overridden
- }
- else if (createDefault)
- {
- StatisticKind kind = getKind();
- assertex(kind != StKindNone);
- const char * scope = p->queryProp("@scope");
- assertex(scope);
- //Clean up the format of the scope when converting it to a description
- StringBuffer descriptionText;
- if (streq(scope, GLOBAL_SCOPE))
- {
- const char * creator = p->queryProp("@creator");
- descriptionText.append(creator).append(":");
- queryLongStatisticName(descriptionText, kind);
- }
- else
- {
- loop
- {
- char c = *scope++;
- if (!c)
- break;
- if (c == ':')
- descriptionText.append(": ");
- else
- descriptionText.append(c);
- }
- if (kind != StTimeElapsed)
- queryLongStatisticName(descriptionText.append(": "), kind);
- }
- str.set(descriptionText);
- }
- else
- str.clear();
- return str;
- }
- IStringVal & CLocalWUStatistic::getType(IStringVal & str) const
- {
- StatisticKind kind = getKind();
- if (kind != StKindNone)
- str.set(queryStatisticName(kind));
- return str;
- }
- IStringVal & CLocalWUStatistic::getFormattedValue(IStringVal & str) const
- {
- StringBuffer formatted;
- formatStatistic(formatted, getValue(), getMeasure());
- str.set(formatted);
- return str;
- }
- StatisticCreatorType CLocalWUStatistic::getCreatorType() const
- {
- return queryCreatorType(p->queryProp("@c"));
- }
- StatisticScopeType CLocalWUStatistic::getScopeType() const
- {
- return queryScopeType(p->queryProp("@s"));
- }
- StatisticKind CLocalWUStatistic::getKind() const
- {
- return queryStatisticKind(p->queryProp("@kind"));
- }
- IStringVal & CLocalWUStatistic::getScope(IStringVal & str) const
- {
- const char * scope = p->queryProp("@scope");
- str.set(scope);
- return str;
- }
- StatisticMeasure CLocalWUStatistic::getMeasure() const
- {
- return queryMeasure(p->queryProp("@unit"));
- }
- unsigned __int64 CLocalWUStatistic::getValue() const
- {
- return p->getPropInt64("@value", 0);
- }
- unsigned __int64 CLocalWUStatistic::getCount() const
- {
- return p->getPropInt64("@count", 0);
- }
- unsigned __int64 CLocalWUStatistic::getMax() const
- {
- return p->getPropInt64("@max", 0);
- }
- unsigned __int64 CLocalWUStatistic::getTimestamp() const
- {
- return p->getPropInt64("@ts", 0);
- }
- bool CLocalWUStatistic::matches(const IStatisticsFilter * filter) const
- {
- if (!filter)
- return true;
- const char * creator = p->queryProp("@creator");
- const char * scope = p->queryProp("@scope");
- return filter->matches(getCreatorType(), creator, getScopeType(), scope, getMeasure(), getKind());
- }
- //==========================================================================================
- CLocalWULegacyTiming::CLocalWULegacyTiming(IPropertyTree *props) : p(props)
- {
- }
- IStringVal & CLocalWULegacyTiming::getCreator(IStringVal & str) const
- {
- str.clear();
- return str;
- }
- IStringVal & CLocalWULegacyTiming::getDescription(IStringVal & str, bool createDefault) const
- {
- str.set(p->queryProp("@name"));
- return str;
- }
- IStringVal & CLocalWULegacyTiming::getType(IStringVal & str) const
- {
- str.set(queryStatisticName(StTimeElapsed));
- return str;
- }
- IStringVal & CLocalWULegacyTiming::getFormattedValue(IStringVal & str) const
- {
- StringBuffer formatted;
- formatStatistic(formatted, getValue(), getMeasure());
- str.set(formatted);
- return str;
- }
- StatisticCreatorType CLocalWULegacyTiming::getCreatorType() const
- {
- return SCTunknown;
- }
- StatisticScopeType CLocalWULegacyTiming::getScopeType() const
- {
- return SSTnone;
- }
- StatisticKind CLocalWULegacyTiming::getKind() const
- {
- return StTimeElapsed;
- }
- IStringVal & CLocalWULegacyTiming::getScope(IStringVal & str) const
- {
- str.clear();
- return str;
- }
- StatisticMeasure CLocalWULegacyTiming::getMeasure() const
- {
- return SMeasureTimeNs;
- }
- unsigned __int64 CLocalWULegacyTiming::getValue() const
- {
- return p->getPropInt64("@duration", 0) * 1000000;
- }
- unsigned __int64 CLocalWULegacyTiming::getCount() const
- {
- return p->getPropInt64("@count", 0);
- }
- unsigned __int64 CLocalWULegacyTiming::getMax() const
- {
- return p->getPropInt64("@max", 0);
- }
- unsigned __int64 CLocalWULegacyTiming::getTimestamp() const
- {
- return 0;
- }
- bool CLocalWULegacyTiming::matches(const IStatisticsFilter * filter) const
- {
- if (!filter)
- return true;
- const char * creator = p->queryProp("@creator");
- const char * scope = p->queryProp("@scope");
- return filter->matches(SCTall, NULL, SSTall, NULL, getMeasure(), getKind());
- }
- //==========================================================================================
- extern WORKUNIT_API ILocalWorkUnit * createLocalWorkUnit(const char *xml)
- {
- Owned<CLocalWorkUnit> cw = new CLocalWorkUnit("W_LOCAL", xml, (ISecManager*)NULL, NULL);
- ILocalWorkUnit* ret = QUERYINTERFACE(&cw->lockRemote(false), ILocalWorkUnit);
- return ret;
- }
- extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool unpack, bool includeProgress)
- {
- const CLocalWorkUnit *w = QUERYINTERFACE(wu, const CLocalWorkUnit);
- if (!w)
- {
- const CLockedWorkUnit *wl = QUERYINTERFACE(wu, const CLockedWorkUnit);
- if (wl)
- w = wl->c;
- }
- if (w)
- {
- Linked<IPropertyTree> p;
- if (unpack||includeProgress)
- p.setown(w->getUnpackedTree(includeProgress));
- else
- p.set(w->p);
- toXML(p, str, 0, XML_Format|XML_SortTags);
- }
- else
- str.append("Unrecognized workunit format");
- return str;
- }
- extern WORKUNIT_API IStringVal& exportWorkUnitToXML(const IConstWorkUnit *wu, IStringVal &str, bool unpack, bool includeProgress)
- {
- StringBuffer x;
- str.set(exportWorkUnitToXML(wu,x,unpack, includeProgress).str());
- return str;
- }
- extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool unpack, bool includeProgress)
- {
- const CLocalWorkUnit *w = QUERYINTERFACE(wu, const CLocalWorkUnit);
- if (!w)
- {
- const CLockedWorkUnit *wl = QUERYINTERFACE(wu, const CLockedWorkUnit);
- if (wl)
- w = wl->c;
- }
- if (w)
- {
- Linked<IPropertyTree> p;
- if (unpack||includeProgress)
- p.setown(w->getUnpackedTree(includeProgress));
- else
- p.set(w->p);
- saveXML(filename, p, 0, XML_Format|XML_SortTags|extraXmlFlags);
- }
- }
- extern WORKUNIT_API void submitWorkUnit(const char *wuid, const char *username, const char *password)
- {
- MemoryBuffer buffer;
- Owned<INamedQueueConnection> conn = createNamedQueueConnection(0); // MORE - security token?
- Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
- Owned<IWorkUnit> workunit = factory->updateWorkUnit(wuid);
- assertex(workunit);
- SCMStringBuffer token;
- createToken(wuid, username, password, token);
- workunit->setSecurityToken(token.str());
- StringAttr clusterName(workunit->queryClusterName());
- if (!clusterName.length())
- throw MakeStringException(WUERR_InvalidCluster, "No target cluster specified");
- workunit->commit();
- workunit.clear();
- Owned<IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(clusterName.str());
- if (!clusterInfo)
- throw MakeStringException(WUERR_InvalidCluster, "Unknown cluster %s", clusterName.str());
- SCMStringBuffer serverQueue;
- clusterInfo->getServerQueue(serverQueue);
- assertex(serverQueue.length());
- Owned<IJobQueue> queue = createJobQueue(serverQueue.str());
- if (!queue.get())
- throw MakeStringException(WUERR_InvalidQueue, "Could not create workunit queue");
- IJobQueueItem *item = createJobQueueItem(wuid);
- queue->enqueue(item);
- }
- extern WORKUNIT_API void abortWorkUnit(const char *wuid)
- {
- StringBuffer xpath("/WorkUnitAborts/");
- xpath.append(wuid);
- Owned<IRemoteConnection> acon = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE, SDS_LOCK_TIMEOUT);
- acon->queryRoot()->setPropInt(NULL, 1);
- }
- extern WORKUNIT_API void secSubmitWorkUnit(const char *wuid, ISecManager &secmgr, ISecUser &secuser)
- {
- if (checkWuSecAccess(wuid, secmgr, &secuser, SecAccess_Write, "Submit", true, true))
- submitWorkUnit(wuid, secuser.getName(), secuser.credentials().getPassword());
- }
- extern WORKUNIT_API void secAbortWorkUnit(const char *wuid, ISecManager &secmgr, ISecUser &secuser)
- {
- if (checkWuSecAccess(wuid, secmgr, &secuser, SecAccess_Write, "Submit", true, true))
- abortWorkUnit(wuid);
- }
- extern WORKUNIT_API void submitWorkUnit(const char *wuid, ISecManager *secmgr, ISecUser *secuser)
- {
- if (secmgr && secuser)
- return secSubmitWorkUnit(wuid, *secmgr, *secuser);
- if (secuser)
- return submitWorkUnit(wuid, secuser->getName(), secuser->credentials().getPassword());
- submitWorkUnit(wuid, "", "");
- }
- extern WORKUNIT_API void abortWorkUnit(const char *wuid, ISecManager *secmgr, ISecUser *secuser)
- {
- if (secmgr && secuser)
- return secAbortWorkUnit(wuid, *secmgr, *secuser);
- abortWorkUnit(wuid);
- }
- bool CLocalWorkUnit::hasWorkflow() const
- {
- return p->hasProp("Workflow");
- }
- unsigned CLocalWorkUnit::queryEventScheduledCount() const
- {
- CriticalBlock block(crit);
- return p->getPropInt("Workflow/@eventScheduledCount", 0);
- }
- void CLocalWorkUnit::incEventScheduledCount()
- {
- CriticalBlock block(crit);
- p->setPropInt("Workflow/@eventScheduledCount", p->getPropInt("Workflow/@eventScheduledCount", 0)+1);
- }
- IPropertyTree * CLocalWorkUnit::queryWorkflowTree() const
- {
- CriticalBlock block(crit);
- return p->queryPropTree("Workflow");
- }
- IConstWorkflowItemIterator* CLocalWorkUnit::getWorkflowItems() const
- {
- // For this to be legally called, we must have the read-able interface. So we are already locked for (at least) read.
- CriticalBlock block(crit);
- if(!workflowIteratorCached)
- {
- assertex(!workflowIterator);
- Owned<IPropertyTree> s = p->getPropTree("Workflow");
- if(s)
- workflowIterator.setown(createWorkflowItemIterator(s));
- workflowIteratorCached = true;
- }
- return workflowIterator.getLink();
- }
- IWorkflowItemArray * CLocalWorkUnit::getWorkflowClone() const
- {
- unsigned count = 0;
- Owned<IConstWorkflowItemIterator> iter = getWorkflowItems();
- for(iter->first(); iter->isValid(); iter->next())
- count++;
- Owned<IWorkflowItemArray> array = createWorkflowItemArray(count);
- for(iter->first(); iter->isValid(); iter->next())
- array->addClone(iter->query());
- return array.getLink();
- }
- IWorkflowItem * CLocalWorkUnit::addWorkflowItem(unsigned wfid, WFType type, WFMode mode, unsigned success, unsigned failure, unsigned recovery, unsigned retriesAllowed, unsigned contingencyFor)
- {
- // For this to be legally called, we must have the write-able interface. So we are already locked for write.
- CriticalBlock block(crit);
- workflowIterator.clear();
- workflowIteratorCached = false;
- IPropertyTree * s = p->queryPropTree("Workflow");
- if(!s)
- s = p->addPropTree("Workflow", createPTree("Workflow"));
- return createWorkflowItem(s, wfid, type, mode, success, failure, recovery, retriesAllowed, contingencyFor);
- }
- IWorkflowItemIterator * CLocalWorkUnit::updateWorkflowItems()
- {
- // For this to be legally called, we must have the write-able interface. So we are already locked for write.
- CriticalBlock block(crit);
- if(!workflowIterator)
- {
- IPropertyTree * s = p->queryPropTree("Workflow");
- if(!s)
- s = p->addPropTree("Workflow", createPTree("Workflow"));
- workflowIterator.setown(createWorkflowItemIterator(s));
- workflowIteratorCached = true;
- }
- return workflowIterator.getLink();
- }
- void CLocalWorkUnit::syncRuntimeWorkflow(IWorkflowItemArray * array)
- {
- Owned<IWorkflowItemIterator> iter = updateWorkflowItems();
- Owned<IWorkflowItem> item;
- for(iter->first(); iter->isValid(); iter->next())
- {
- item.setown(iter->get());
- item->syncRuntimeData(array->queryWfid(item->queryWfid()));
- }
- workflowIterator.clear();
- workflowIteratorCached = false;
- }
- void CLocalWorkUnit::resetWorkflow()
- {
- if (hasWorkflow())
- {
- Owned<IWorkflowItemIterator> iter = updateWorkflowItems();
- Owned<IWorkflowItem> wf;
- for(iter->first(); iter->isValid(); iter->next())
- {
- wf.setown(iter->get());
- wf->reset();
- }
- workflowIterator.clear();
- workflowIteratorCached = false;
- }
- }
- void CLocalWorkUnit::schedule()
- {
- CriticalBlock block(crit);
- if(queryEventScheduledCount() == 0) return;
- switch(getState())
- {
- case WUStateCompleted:
- setState(WUStateWait);
- break;
- case WUStateFailed:
- case WUStateArchived:
- case WUStateAborting:
- case WUStateAborted:
- case WUStateScheduled:
- throw MakeStringException(WUERR_CannotSchedule, "Cannot schedule workunit in this state");
- }
- StringBuffer rootPath;
- rootPath.append("/Schedule/").append(queryClusterName());
- Owned<IRemoteConnection> conn = querySDS().connect(rootPath.str(), myProcessSession(), RTM_LOCK_WRITE | RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
- Owned<IPropertyTree> root = conn->getRoot();
- if(!root->hasChildren())
- {
- StringBuffer addPath;
- addPath.append("/Schedulers/").append(queryClusterName());
- Owned<IRemoteConnection> addConn = querySDS().connect(addPath.str(), myProcessSession(), RTM_LOCK_WRITE | RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
- }
- char const * wuid = p->queryName();
- StringBuffer xpath("*/*/");
- ncnameEscape(wuid, xpath);
- bool more;
- do more = root->removeProp(xpath.str()); while(more);
-
- Owned<IConstWorkflowItemIterator> iter = getWorkflowItems();
- Owned<IWorkflowEvent> event;
- Owned<IPropertyTree> branch1, branch2;
- for(iter->first(); iter->isValid(); iter->next())
- {
- event.setown(iter->query()->getScheduleEvent());
- if(!event) continue;
- ncnameEscape(event->queryName(), xpath.clear());
- ensurePTree(root, xpath.str());
- branch1.setown(root->getPropTree(xpath.str()));
- ncnameEscape(event->queryText(), xpath.clear());
- ensurePTree(branch1, xpath.str());
- branch2.setown(branch1->getPropTree(xpath.str()));
- ncnameEscape(wuid, xpath.clear());
- ensurePTree(branch2, xpath.str());
- }
- }
- void CLocalWorkUnit::deschedule()
- {
- if(queryEventScheduledCount() == 0) return;
- if(getState() == WUStateWait)
- setState(WUStateCompleted);
- doDescheduleWorkkunit(p->queryName());
- }
- mapEnums localFileUploadTypes[] = {
- { UploadTypeFileSpray, "FileSpray" },
- { UploadTypeWUResult, "WUResult" },
- { UploadTypeWUResultCsv, "WUResultCsv" },
- { UploadTypeWUResultXml, "WUResultXml" },
- { UploadTypeSize, NULL }
- };
- class CLocalFileUpload : public CInterface, implements IConstLocalFileUpload
- {
- public:
- CLocalFileUpload(IPropertyTree * _tree) : tree(_tree) {}
- CLocalFileUpload(unsigned id, LocalFileUploadType type, char const * source, char const * destination, char const * eventTag)
- {
- tree.setown(createPTree());
- tree->setPropInt("@id", id);
- setEnum(tree, "@type", type, localFileUploadTypes);
- tree->setProp("@source", source);
- tree->setProp("@destination", destination);
- if (eventTag)
- tree->setProp("@eventTag", eventTag);
- }
- IMPLEMENT_IINTERFACE;
- IPropertyTree * getTree() { return tree.getLink(); }
- virtual unsigned queryID() const { return tree->getPropInt("@id"); }
- virtual LocalFileUploadType queryType() const { return (LocalFileUploadType)getEnum(tree, "@type", localFileUploadTypes); }
- virtual IStringVal & getSource(IStringVal & ret) const { ret.set(tree->queryProp("@source")); return ret; }
- virtual IStringVal & getDestination(IStringVal & ret) const { ret.set(tree->queryProp("@destination")); return ret; }
- virtual IStringVal & getEventTag(IStringVal & ret) const { if(tree->hasProp("@eventTag")) ret.set(tree->queryProp("@eventTag")); else ret.clear(); return ret; }
- private:
- Owned<IPropertyTree> tree;
- };
- class CLocalFileUploadIterator : public CInterface, implements IConstLocalFileUploadIterator
- {
- public:
- CLocalFileUploadIterator(IPropertyTree * _tree) : tree(_tree), iter(tree->getElements("LocalFileUpload")) {}
- IMPLEMENT_IINTERFACE;
- bool first() { return iter->first(); }
- bool isValid() { return iter->isValid(); }
- bool next() { return iter->next(); }
- IConstLocalFileUpload * get() { return new CLocalFileUpload(&iter->get()); }
- private:
- Owned<IPropertyTree> tree;
- Owned<IPropertyTreeIterator> iter;
- };
- IConstLocalFileUploadIterator * CLocalWorkUnit::getLocalFileUploads() const
- {
- // For this to be legally called, we must have the read-able interface. So we are already locked for (at least) read.
- CriticalBlock block(crit);
- Owned<IPropertyTree> s = p->getPropTree("LocalFileUploads");
- if(s)
- return new CLocalFileUploadIterator(s.getClear());
- else
- return NULL;
- }
- bool CLocalWorkUnit::requiresLocalFileUpload() const
- {
- SCMStringBuffer dest;
- Owned<IConstWUResult> result;
- Owned<IConstLocalFileUploadIterator> iter(getLocalFileUploads());
- if(!iter)
- return false;
- for(iter->first(); iter->isValid(); iter->next())
- {
- Owned<IConstLocalFileUpload> upload(iter->get());
- switch(upload->queryType())
- {
- case UploadTypeWUResult:
- case UploadTypeWUResultCsv:
- case UploadTypeWUResultXml:
- upload->getDestination(dest);
- result.setown(getResultByName(dest.str()));
- if(!result)
- return true;
- break;
- default:
- throw MakeStringException(WUERR_InvalidUploadFormat, "Unsupported local file upload type %s", getEnumText(upload->queryType(), localFileUploadTypes));
- }
- }
- return false;
- }
- unsigned CLocalWorkUnit::addLocalFileUpload(LocalFileUploadType type, char const * source, char const * destination, char const * eventTag)
- {
- // For this to be legally called, we must have the write-able interface. So we are already locked for write.
- CriticalBlock block(crit);
- IPropertyTree * s = p->queryPropTree("LocalFileUploads");
- if(!s)
- s = p->addPropTree("LocalFileUploads", createPTree());
- unsigned id = s->numChildren();
- Owned<CLocalFileUpload> upload = new CLocalFileUpload(id, type, source, destination, eventTag);
- s->addPropTree("LocalFileUpload", upload->getTree());
- return id;
- }
- #if 0
- void testConstWorkflow(IConstWorkflowItem * cwf, bool * okay, bool * dep)
- {
- DBGLOG("Test workflow const iface %u", cwf->queryWfid());
- unsigned deps = 0;
- Owned<IWorkflowDependencyIterator> diter;
- switch(cwf->queryWfid())
- {
- case 1:
- assertex(!cwf->isScheduled());
- assertex(cwf->queryType() == WFTypeNormal);
- assertex(cwf->queryState() == WFStateNull);
- diter.setown(cwf->getDependencies());
- for(diter->first(); diter->isValid(); diter->next())
- deps++;
- assertex(deps==0);
- okay[0] = true;
- break;
- case 2:
- assertex(!cwf->isScheduled());
- assertex(cwf->queryType() == WFTypeRecovery);
- assertex(cwf->queryState() == WFStateSkip);
- okay[1] = true;
- break;
- case 3:
- assertex(cwf->queryContingencyFor() == 4);
- okay[2] = true;
- break;
- case 4:
- assertex(cwf->isScheduled());
- assertex(cwf->queryType() == WFTypeNormal);
- assertex(cwf->queryState() == WFStateReqd);
- assertex(cwf->querySuccess() == 0);
- assertex(cwf->queryFailure() == 3);
- assertex(cwf->queryRecovery() == 2);
- assertex(cwf->queryRetriesAllowed() == 10);
- assertex(cwf->queryRetriesRemaining() == 10);
- diter.setown(cwf->getDependencies());
- for(diter->first(); diter->isValid(); diter->next())
- {
- dep[diter->query()-1] = true;
- deps++;
- }
- assertex(deps==2);
- assertex(dep[0]);
- assertex(dep[1]);
- okay[3] = true;
- break;
- case 5:
- assertex(cwf->isScheduled());
- assertex(!cwf->isScheduledNow());
- assertex(cwf->querySchedulePriority() == 75);
- assertex(cwf->queryScheduleCount() == 5);
- assertex(cwf->queryScheduleCountRemaining() == 5);
- okay[4] = true;
- break;
- case 6:
- assertex(cwf->isScheduled());
- assertex(!cwf->isScheduledNow());
- assertex(cwf->querySchedulePriority() == 25);
- assertex(!cwf->hasScheduleCount());
- okay[5] = true;
- break;
- default:
- assertex(!"unknown wfid in test");
- }
- }
- void testRuntimeWorkflow(IRuntimeWorkflowItem * rwf, bool * okay)
- {
- DBGLOG("Test workflow runtime iface %u", rwf->queryWfid());
- switch(rwf->queryWfid())
- {
- case 1:
- case 2:
- case 3:
- okay[rwf->queryWfid()-1] = true;
- break;
- case 4:
- {
- unsigned tries = 0;
- while(rwf->testAndDecRetries())
- tries++;
- assertex(tries == 10);
- assertex(rwf->queryRetriesRemaining() == 0);
- rwf->setState(WFStateFail);
- assertex(rwf->queryState() == WFStateFail);
- rwf->reset();
- assertex(rwf->queryRetriesRemaining() == 10);
- assertex(rwf->queryState() == WFStateReqd);
- }
- okay[3] = true;
- break;
- case 5:
- {
- assertex(rwf->queryScheduleCountRemaining() == 5);
- unsigned count = 0;
- do count++; while(rwf->decAndTestScheduleCountRemaining());
- assertex(count == 5);
- assertex(rwf->queryScheduleCountRemaining() == 0);
- rwf->reset();
- assertex(rwf->queryScheduleCountRemaining() == 5);
- }
- okay[4] = true;
- break;
- case 6:
- {
- assertex(!rwf->hasScheduleCount());
- unsigned count;
- for(count=0; count<20; count++)
- assertex(rwf->decAndTestScheduleCountRemaining());
- }
- okay[5] = true;
- break;
- default:
- assertex(!"unknown wfid in test");
- }
- }
- void testWorkflow()
- {
- DBGLOG("workunit.cpp : testWorkflow");
- CLocalWorkUnit wu("W-WF-TEST", 0, 0, 0);
- Owned<IWorkflowItem> wf;
- wf.setown(wu.addWorkflowItem(1, WFTypeNormal, 0, 0, 0, 0, 0));
- wf.setown(wu.addWorkflowItem(2, WFTypeRecovery, 0, 0, 0, 0, 0));
- wf.setown(wu.addWorkflowItem(3, WFTypeFailure, 0, 0, 0, 0, 4));
- wf.setown(wu.addWorkflowItem(4, WFTypeNormal, 0, 3, 2, 10, 0));
- wf->setScheduledNow();
- wf->addDependency(1);
- wf.setown(wu.addWorkflowItem(5, WFTypeNormal, 0, 0, 0, 0, 0));
- wf->setScheduledOn("test", "foo*");
- wf->setSchedulePriority(75);
- wf->setScheduleCount(5);
- wf.setown(wu.addWorkflowItem(6, WFTypeNormal, 0, 0, 0, 0, 0));
- wf->setScheduledOn("test", "bar*");
- wf->setSchedulePriority(25);
- unsigned const n = 6;
- bool okay[n];
- bool dep[n];
- unsigned i;
- for(i=0; i<n; i++)
- okay[i] = dep[i] = 0;
- Owned<IConstWorkflowItemIterator> citer(wu.getWorkflowItems());
- for(citer->first(); citer->isValid(); citer->next())
- testConstWorkflow(citer->query(), okay, dep);
- for(i=0; i<n; i++)
- {
- assertex(okay[i]);
- okay[i] = false;
- }
- Owned<IWorkflowItemIterator> miter(wu.updateWorkflowItems());
- for(miter->first(); miter->isValid(); miter->next())
- {
- Owned<IRuntimeWorkflowItem> rwf(miter->get());
- testRuntimeWorkflow(rwf, okay);
- }
- for(i=0; i<n; i++)
- {
- assertex(okay[i]);
- okay[i] = dep[i] = false;
- }
- Owned<IWorkflowItemArray> array(wu.getWorkflowClone());
- unsigned wfid;
- for(wfid = 1; array->isValid(wfid); wfid++)
- testConstWorkflow(&array->queryWfid(wfid), okay, dep);
- for(i=0; i<n; i++)
- {
- assertex(okay[i]);
- okay[i] = false;
- }
- for(wfid = 1; array->isValid(wfid); wfid++)
- testRuntimeWorkflow(&array->queryWfid(wfid), okay);
- for(i=0; i<n; i++)
- {
- assertex(okay[i]);
- okay[i] = false;
- }
- }
- #endif
- //------------------------------------------------------------------------------------------
- class WorkUnitWaiter : public CInterface, implements ISDSSubscription, implements IAbortHandler
- {
- Semaphore changed;
- SubscriptionId change;
- public:
- IMPLEMENT_IINTERFACE;
- WorkUnitWaiter(const char *xpath)
- {
- change = querySDS().subscribe(xpath, *this, false);
- aborted = false;
- }
- ~WorkUnitWaiter()
- {
- assertex(change==0);
- }
- void notify(SubscriptionId id, const char *xpath, SDSNotifyFlags flags, unsigned valueLen, const void *valueData)
- {
- changed.signal();
- }
- bool wait(unsigned timeout)
- {
- return changed.wait(timeout) && !aborted;
- }
- bool onAbort()
- {
- aborted = true;
- changed.signal();
- return false;
- }
- void unsubscribe()
- {
- querySDS().unsubscribe(change);
- change = 0;
- }
- bool aborted;
- };
- static WUState _waitForWorkUnit(const char * wuid, unsigned timeout, bool compiled, bool returnOnWaitState)
- {
- StringBuffer wuRoot;
- getXPath(wuRoot, wuid);
- Owned<WorkUnitWaiter> waiter = new WorkUnitWaiter(wuRoot.str());
- LocalIAbortHandler abortHandler(*waiter);
- WUState ret = WUStateUnknown;
- Owned<IRemoteConnection> conn = querySDS().connect(wuRoot.str(), myProcessSession(), 0, SDS_LOCK_TIMEOUT);
- if (conn)
- {
- unsigned start = msTick();
- loop
- {
- ret = (WUState) getEnum(conn->queryRoot(), "@state", states);
- switch (ret)
- {
- case WUStateCompiled:
- case WUStateUploadingFiles:
- if (!compiled)
- break;
- // fall into
- case WUStateCompleted:
- case WUStateFailed:
- case WUStateAborted:
- waiter->unsubscribe();
- return ret;
- case WUStateWait:
- if(returnOnWaitState)
- {
- waiter->unsubscribe();
- return ret;
- }
- break;
- case WUStateCompiling:
- case WUStateRunning:
- case WUStateDebugPaused:
- case WUStateDebugRunning:
- case WUStateBlocked:
- case WUStateAborting:
- if (queryDaliServerVersion().compare("2.1")>=0)
- {
- SessionId agent = conn->queryRoot()->getPropInt64("@agentSession", -1);
- if((agent>0) && querySessionManager().sessionStopped(agent, 0))
- {
- waiter->unsubscribe();
- conn->reload();
- ret = (WUState) getEnum(conn->queryRoot(), "@state", states);
- bool isEcl = false;
- switch (ret)
- {
- case WUStateCompiling:
- isEcl = true;
- // drop into
- case WUStateRunning:
- case WUStateBlocked:
- ret = WUStateFailed;
- break;
- case WUStateAborting:
- ret = WUStateAborted;
- break;
- default:
- return ret;
- }
- WARNLOG("_waitForWorkUnit terminated: %"I64F"d state = %d",(__int64)agent,(int)ret);
- Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
- Owned<IWorkUnit> wu = factory->updateWorkUnit(wuid);
- wu->setState(ret);
- Owned<IWUException> e = wu->createException();
- e->setExceptionCode(isEcl ? 1001 : 1000);
- e->setExceptionMessage(isEcl ? "EclServer terminated unexpectedly" : "Workunit terminated unexpectedly");
- return ret;
- }
- }
- break;
- }
- unsigned waited = msTick() - start;
- if (timeout==-1)
- {
- waiter->wait(20000); // recheck state every 20 seconds even if no timeout, in case eclagent has crashed.
- if (waiter->aborted)
- {
- ret = WUStateUnknown; // MORE - throw an exception?
- break;
- }
- }
- else if (waited > timeout || !waiter->wait(timeout-waited))
- {
- ret = WUStateUnknown; // MORE - throw an exception?
- break;
- }
- conn->reload();
- }
- }
- waiter->unsubscribe();
- return ret;
- }
- extern WUState waitForWorkUnitToComplete(const char * wuid, int timeout, bool returnOnWaitState)
- {
- return _waitForWorkUnit(wuid, (unsigned)timeout, false, returnOnWaitState);
- }
- extern WORKUNIT_API WUState secWaitForWorkUnitToComplete(const char * wuid, ISecManager &secmgr, ISecUser &secuser, int timeout, bool returnOnWaitState)
- {
- if (checkWuSecAccess(wuid, secmgr, &secuser, SecAccess_Read, "Wait for Complete", false, true))
- return waitForWorkUnitToComplete(wuid, timeout, returnOnWaitState);
- return WUStateUnknown;
- }
- extern bool waitForWorkUnitToCompile(const char * wuid, int timeout)
- {
- switch(_waitForWorkUnit(wuid, (unsigned)timeout, true, true))
- {
- case WUStateCompiled:
- case WUStateCompleted:
- case WUStateWait:
- case WUStateUploadingFiles:
- return true;
- default:
- return false;
- }
- }
- extern WORKUNIT_API bool secWaitForWorkUnitToCompile(const char * wuid, ISecManager &secmgr, ISecUser &secuser, int timeout)
- {
- if (checkWuSecAccess(wuid, secmgr, &secuser, SecAccess_Read, "Wait for Compile", false, true))
- return waitForWorkUnitToCompile(wuid, timeout);
- return false;
- }
- extern WORKUNIT_API bool secDebugWorkunit(const char * wuid, ISecManager &secmgr, ISecUser &secuser, const char *command, StringBuffer &response)
- {
- if (strnicmp(command, "<debug:", 7) == 0 && checkWuSecAccess(wuid, secmgr, &secuser, SecAccess_Read, "Debug", false, true))
- {
- Owned<IConstWorkUnit> wu = factory->secOpenWorkUnit(wuid, false, &secmgr, &secuser);
- SCMStringBuffer ip;
- unsigned port;
- port = wu->getDebugAgentListenerPort();
- wu->getDebugAgentListenerIP(ip);
- SocketEndpoint debugEP(ip.str(), port);
- Owned<ISocket> socket = ISocket::connect_timeout(debugEP, 1000);
- unsigned len = (size32_t)strlen(command);
- unsigned revlen = len;
- _WINREV(revlen);
- socket->write(&revlen, sizeof(revlen));
- socket->write(command, len);
- for (;;)
- {
- socket->read(&len, sizeof(len));
- _WINREV(len);
- if (len == 0)
- break;
- if (len & 0x80000000)
- {
- throwUnexpected();
- }
- char * mem = (char*) response.reserve(len);
- socket->read(mem, len);
- }
- return true;
- }
- return false;
- }
- void updateSuppliedXmlParams(IWorkUnit * w)
- {
- Owned<const IPropertyTree> params = w->getXmlParams();
- if (!params)
- return;
- Owned<IPropertyTreeIterator> elems = params->getElements("*");
- ForEach(*elems)
- {
- IPropertyTree & curVal = elems->query();
- const char *name = curVal.queryName();
- Owned<IWUResult> r = updateWorkUnitResult(w, name, -1);
- if (r)
- {
- StringBuffer s;
- if (r->isResultScalar() && !curVal.hasChildren())
- {
- curVal.getProp(".", s);
- r->setResultXML(s);
- r->setResultStatus(ResultStatusSupplied);
- }
- else
- {
- toXML(&curVal, s);
- bool isSet = (curVal.hasProp("Item") || curVal.hasProp("string"));
- r->setResultRaw(s.length(), s.str(), isSet ? ResultFormatXmlSet : ResultFormatXml);
- }
- }
- else
- DBGLOG("WARNING: no matching variable in workunit for input parameter %s", name);
- }
- }
- IWUResult * updateWorkUnitResult(IWorkUnit * w, const char *name, unsigned sequence)
- {
- switch ((int)sequence)
- {
- case ResultSequenceStored:
- return w->updateVariableByName(name);
- case ResultSequencePersist:
- return w->updateGlobalByName(name);
- case ResultSequenceInternal:
- case ResultSequenceOnce:
- return w->updateTemporaryByName(name);
- default:
- return w->updateResultBySequence(sequence);
- }
- }
- IConstWUResult * getWorkUnitResult(IConstWorkUnit * w, const char *name, unsigned sequence)
- {
- switch ((int)sequence)
- {
- case ResultSequenceStored:
- return w->getVariableByName(name);
- case ResultSequencePersist:
- return w->getGlobalByName(name);
- case ResultSequenceInternal:
- case ResultSequenceOnce:
- return w->getTemporaryByName(name);
- default:
- if (name && name[0])
- return w->getResultByName(name);//name takes precedence over sequence
- else
- return w->getResultBySequence(sequence);
- }
- }
- extern WORKUNIT_API bool getWorkUnitCreateTime(const char *wuid,CDateTime &time)
- {
- if (wuid) {
- char prefchar;
- unsigned year,month,day,hour,min,sec;
- if (sscanf(wuid, "%c%4u%2u%2u-%2u%2u%2u", &prefchar, &year, &month, &day, &hour, &min, &sec)==7) {
- time.set(year, month, day, hour, min, sec, 0, true);
- // time.setDate(year, month, day);
- // time.setTime(hour, min, sec, 0, true); // for some reason time is local
- return true;
- }
- }
- return false;
- }
- extern WORKUNIT_API IStringVal& createToken(const char *wuid, const char *user, const char *password, IStringVal &str)
- {
- StringBuffer wu, token("X");
- wu.append(wuid).append(';').append(user).append(';').append(password);
- encrypt(token,wu.str());
- str.set(token.str());
- return str;
- }
- // This will be replaced by something more secure!
- extern WORKUNIT_API void extractToken(const char *token, const char *wuid, IStringVal &user, IStringVal &password)
- {
- if (token && *token)
- {
- StringBuffer wu;
- decrypt(wu, token+1);
- const char *finger = strchr(wu.str(),';');
- if (finger && strnicmp(wuid, wu.str(), finger-wu.str())==0)
- {
- const char *finger1 = strchr(++finger,';');
- if(finger1)
- {
- user.setLen(finger, (size32_t)(finger1-finger));
- finger1++;
- password.setLen(finger1, (size32_t)(wu.str() + wu.length() - finger1));
- return;
- }
- }
- throw MakeStringException(WUERR_InvalidSecurityToken, "Invalid call to extractToken");
- }
- }
- extern WORKUNIT_API WUState getWorkUnitState(const char* state)
- {
- return (WUState) getEnum(state, states);
- }
- const LogMsgCategory MCschedconn = MCprogress(1000); // Category used to inform about schedule synchronization
- class CWorkflowScheduleConnection : public CInterface, implements IWorkflowScheduleConnection
- {
- public:
- CWorkflowScheduleConnection(char const * wuid)
- {
- basexpath.append("/WorkflowSchedule/").append(wuid);
- flagxpath.append(basexpath.str()).append("/Active");
- }
- IMPLEMENT_IINTERFACE;
- virtual void lock()
- {
- LOG(MCschedconn, "Locking base schedule connection");
- baseconn.setown(querySDS().connect(basexpath.str(), myProcessSession(), RTM_CREATE_QUERY | RTM_LOCK_WRITE, INFINITE));
- if(!baseconn)
- throw MakeStringException(WUERR_ScheduleLockFailed, "Could not get base workflow schedule lock");
- }
- virtual void unlock()
- {
- LOG(MCschedconn, "Unlocking base schedule connection");
- baseconn.clear();
- }
- virtual void setActive()
- {
- LOG(MCschedconn, "Setting active flag in schedule connection");
- flagconn.setown(querySDS().connect(flagxpath.str(), myProcessSession(), RTM_CREATE | RTM_LOCK_WRITE | RTM_DELETE_ON_DISCONNECT, INFINITE));
- if(!flagconn)
- throw MakeStringException(WUERR_ScheduleLockFailed, "Could not get active workflow schedule lock");
- }
- virtual void resetActive()
- {
- LOG(MCschedconn, "Resetting active flag in schedule connection");
- flagconn.clear();
- }
- virtual bool queryActive()
- {
- return baseconn->queryRoot()->hasProp("Active");
- }
- virtual bool pull(IWorkflowItemArray * workflow)
- {
- assertex(baseconn);
- Owned<IPropertyTree> root = baseconn->getRoot();
- Owned<IPropertyTree> eventQueue = root->getPropTree("EventQueue");
- if(!eventQueue) return false;
- if(!eventQueue->hasProp("Item")) return false;
- {
- Owned<IPropertyTreeIterator> eventItems = eventQueue->getElements("Item");
- Owned<IPropertyTree> eventItem;
- Owned<IRuntimeWorkflowItemIterator> wfItems = workflow->getSequenceIterator();
- Owned<IRuntimeWorkflowItem> wfItem;
- for(eventItems->first(); eventItems->isValid(); eventItems->next())
- {
- eventItem.setown(&eventItems->get());
- const char * eventName = eventItem->queryProp("@name");
- const char * eventText = eventItem->queryProp("@text");
- for(wfItems->first(); wfItems->isValid(); wfItems->next())
- {
- wfItem.setown(wfItems->get());
- if(wfItem->queryState() != WFStateWait)
- continue;
- Owned<IWorkflowEvent> targetEvent = wfItem->getScheduleEvent();
- if(!targetEvent || !targetEvent->matches(eventName, eventText))
- continue;
- wfItem->setEvent(eventName, eventText);
- wfItem->setState(WFStateReqd);
- resetDependentsState(workflow, *wfItem);
- }
- }
- }
- bool more;
- do
- more = eventQueue->removeProp("Item");
- while(more);
- return true;
- }
- virtual void push(char const * name, char const * text)
- {
- assertex(baseconn);
- Owned<IPropertyTree> root = baseconn->getRoot();
- ensurePTree(root, "EventQueue");
- Owned<IPropertyTree> eventQueue = root->getPropTree("EventQueue");
- Owned<IPropertyTree> eventItem = createPTree();
- eventItem->setProp("@name", name);
- eventItem->setProp("@text", text);
- eventQueue->addPropTree("Item", eventItem.getLink());
- }
- virtual void remove()
- {
- if (baseconn)
- {
- baseconn->close(true);
- baseconn.clear();
- }
- }
- private:
- void resetItemStateAndDependents(IWorkflowItemArray * workflow, unsigned wfid) const
- {
- if (wfid)
- resetItemStateAndDependents(workflow, workflow->queryWfid(wfid));
- }
- void resetItemStateAndDependents(IWorkflowItemArray * workflow, IRuntimeWorkflowItem & item) const
- {
- switch(item.queryState())
- {
- case WFStateDone:
- case WFStateFail:
- {
- item.setState(WFStateNull);
- resetItemStateAndDependents(workflow, item.queryPersistWfid());
- resetDependentsState(workflow, item);
- break;
- }
- }
- }
- void resetDependentsState(IWorkflowItemArray * workflow, IRuntimeWorkflowItem & item) const
- {
- Owned<IWorkflowDependencyIterator> iter(item.getDependencies());
- for(iter->first(); iter->isValid(); iter->next())
- {
- IRuntimeWorkflowItem & dep = workflow->queryWfid(iter->query());
- resetItemStateAndDependents(workflow, dep);
- }
- }
- private:
- StringBuffer basexpath;
- StringBuffer flagxpath;
- Owned<IRemoteConnection> baseconn;
- Owned<IRemoteConnection> flagconn;
- };
- extern WORKUNIT_API IWorkflowScheduleConnection * getWorkflowScheduleConnection(char const * wuid)
- {
- return new CWorkflowScheduleConnection(wuid);
- }
- extern WORKUNIT_API IExtendedWUInterface * queryExtendedWU(IWorkUnit * wu)
- {
- return QUERYINTERFACE(wu, IExtendedWUInterface);
- }
- extern WORKUNIT_API void addExceptionToWorkunit(IWorkUnit * wu, WUExceptionSeverity severity, const char * source, unsigned code, const char * text, const char * filename, unsigned lineno, unsigned column)
- {
- Owned<IWUException> we = wu->createException();
- we->setSeverity(severity);
- we->setExceptionMessage(text);
- if (source)
- we->setExceptionSource(source);
- if (code)
- we->setExceptionCode(code);
- if (filename)
- we->setExceptionFileName(filename);
- if (lineno)
- {
- we->setExceptionLineNo(lineno);
- if (column)
- we->setExceptionColumn(lineno);
- }
- }
- const char * skipLeadingXml(const char * text)
- {
- if (!text)
- return NULL;
- //skip utf8 BOM, probably excessive
- if (memcmp(text, UTF8_BOM, 3) == 0)
- text += 3;
- loop
- {
- if (isspace(*text))
- text++;
- else if (text[0] == '<' && text[1] == '?')
- {
- text += 2;
- loop
- {
- if (!*text) break;
- if (text[0] == '?' && text[1] == '>')
- {
- text += 2;
- break;
- }
- text++;
- }
- }
- else if (text[0] == '<' && text[1] == '!' && text[2] == '-' && text[3] == '-')
- {
- text += 4;
- loop
- {
- if (!*text) break;
- if (text[0] == '-' && text[1] == '-' && text[2] == '>')
- {
- text += 3;
- break;
- }
- text++;
- }
- }
- else
- break;
- }
- return text;
- }
- extern WORKUNIT_API bool isArchiveQuery(const char * text)
- {
- text = skipLeadingXml(text);
- if (!text)
- return false;
- const char * archivePrefix = "<Archive";
- return memicmp(text, archivePrefix, strlen(archivePrefix)) == 0;
- }
- extern WORKUNIT_API bool isQueryManifest(const char * text)
- {
- text = skipLeadingXml(text);
- if (!text)
- return false;
- const char * manifestPrefix = "<Manifest";
- return memicmp(text, manifestPrefix, strlen(manifestPrefix)) == 0;
- }
- //------------------------------------------------------------------------------
- // Named Alias helper function
- static IPropertyTree * resolveQueryByDll(IPropertyTree * queryRegistry, const char * dll)
- {
- StringBuffer xpath;
- xpath.append("Query[@dll=\"").append(dll).append("\"]");
- return queryRegistry->getPropTree(xpath);
- }
- static IPropertyTree * resolveQueryByWuid(IPropertyTree * queryRegistry, const char * wuid)
- {
- StringBuffer xpath;
- xpath.append("Query[@wuid=\"").append(wuid).append("\"]");
- return queryRegistry->getPropTree(xpath);
- }
- static void clearAliases(IPropertyTree * queryRegistry, const char * id)
- {
- StringBuffer lcId(id);
- lcId.toLowerCase();
- StringBuffer xpath;
- xpath.append("Alias[@id=\"").append(lcId).append("\"]");
- Owned<IPropertyTreeIterator> iter = queryRegistry->getElements(xpath);
- ForEach(*iter)
- {
- queryRegistry->removeProp(xpath.str());
- }
- }
- IPropertyTree * addNamedQuery(IPropertyTree * queryRegistry, const char * name, const char * wuid, const char * dll, bool library, const char *userid, const char *snapshot)
- {
- StringBuffer lcName(name);
- lcName.toLowerCase();
- StringBuffer xpath;
- xpath.append("Query[@name=\"").append(lcName.str()).append("\"]");
- Owned<IPropertyTreeIterator> iter = queryRegistry->getElements(xpath);
- unsigned seq = 1;
- ForEach(*iter)
- {
- IPropertyTree &item = iter->query();
- const char *thisWuid = item.queryProp("@wuid");
- if (strieq(wuid, thisWuid))
- return &item;
- unsigned thisSeq = item.getPropInt("@seq");
- if (thisSeq >= seq)
- seq = thisSeq + 1;
- }
- StringBuffer id;
- id.append(lcName).append(".").append(seq);
- IPropertyTree * newEntry = createPTree("Query", ipt_caseInsensitive);
- newEntry->setProp("@name", lcName);
- newEntry->setProp("@wuid", wuid);
- newEntry->setProp("@dll", dll);
- newEntry->setProp("@id", id);
- newEntry->setPropInt("@seq", seq);
- if (library)
- newEntry->setPropBool("@isLibrary", true);
- if (userid && *userid)
- newEntry->setProp("@publishedBy", userid);
- if (snapshot && *snapshot)
- newEntry->setProp("@snapshot", snapshot);
- return queryRegistry->addPropTree("Query", newEntry);
- }
- void removeNamedQuery(IPropertyTree * queryRegistry, const char * id)
- {
- StringBuffer lcId(id);
- lcId.toLowerCase();
- clearAliases(queryRegistry, lcId);
- StringBuffer xpath;
- xpath.append("Query[@id=\"").append(lcId).append("\"]");
- queryRegistry->removeProp(xpath);
- }
- void removeDllFromNamedQueries(IPropertyTree * queryRegistry, const char * dll)
- {
- Owned<IPropertyTree> match = resolveQueryByDll(queryRegistry, dll);
- if (!match)
- return;
- clearAliases(queryRegistry, match->queryProp("@id"));
- queryRegistry->removeTree(match);
- }
- void removeWuidFromNamedQueries(IPropertyTree * queryRegistry, const char * wuid)
- {
- Owned<IPropertyTree> match = resolveQueryByWuid(queryRegistry, wuid);
- if (!match)
- return;
- clearAliases(queryRegistry, match->queryProp("@id"));
- queryRegistry->removeTree(match);
- }
- void removeAliasesFromNamedQuery(IPropertyTree * queryRegistry, const char * id)
- {
- clearAliases(queryRegistry, id);
- }
- void setQueryAlias(IPropertyTree * queryRegistry, const char * name, const char * value)
- {
- StringBuffer lcName(name);
- lcName.toLowerCase();
- StringBuffer xpath;
- xpath.append("Alias[@name=\"").append(lcName).append("\"]");
- IPropertyTree * match = queryRegistry->queryPropTree(xpath);
- if (!match)
- {
- IPropertyTree * newEntry = createPTree("Alias");
- newEntry->setProp("@name", lcName);
- match = queryRegistry->addPropTree("Alias", newEntry);
- }
- match->setProp("@id", value);
- }
- extern WORKUNIT_API IPropertyTree * getQueryById(IPropertyTree * queryRegistry, const char *queryid)
- {
- if (!queryRegistry || !queryid)
- return NULL;
- StringBuffer xpath;
- xpath.append("Query[@id=\"").append(queryid).append("\"]");
- return queryRegistry->getPropTree(xpath);
- }
- extern WORKUNIT_API IPropertyTree * getQueryById(const char *queryset, const char *queryid, bool readonly)
- {
- Owned<IPropertyTree> queryRegistry = getQueryRegistry(queryset, readonly);
- return getQueryById(queryRegistry, queryid);
- }
- extern WORKUNIT_API IPropertyTree * resolveQueryAlias(IPropertyTree * queryRegistry, const char * alias)
- {
- if (!queryRegistry || !alias)
- return NULL;
- StringBuffer xpath;
- unsigned cnt = 0;
- StringBuffer lc(alias);
- const char * search = lc.toLowerCase().str();
- loop
- {
- xpath.set("Alias[@name='").append(search).append("']/@id");
- const char * queryId = queryRegistry->queryProp(xpath);
- if (!queryId)
- break;
- //Check for too many alias indirections.
- if (cnt++ > 10)
- return NULL;
- search = lc.set(queryId).toLowerCase().str();
- }
- return getQueryById(queryRegistry, search);
- }
- extern WORKUNIT_API IPropertyTree * resolveQueryAlias(const char *queryset, const char *alias, bool readonly)
- {
- Owned<IPropertyTree> queryRegistry = getQueryRegistry(queryset, readonly);
- return resolveQueryAlias(queryRegistry, alias);
- }
- void setQuerySuspendedState(IPropertyTree * queryRegistry, const char *id, bool suspend, const char *userid)
- {
- StringBuffer lcId(id);
- lcId.toLowerCase();
- StringBuffer xpath;
- xpath.append("Query[@id=\"").append(lcId).append("\"]");
- IPropertyTree *tree = queryRegistry->queryPropTree(xpath);
- if (tree)
- {
- if (tree->getPropBool("@suspended", false) == suspend)
- return;
- if (suspend)
- {
- tree->addPropBool("@suspended", true);
- if (userid && *userid)
- tree->addProp("@suspendedBy", userid);
- }
- else
- {
- tree->removeProp("@suspended");
- tree->removeProp("@suspendedBy");
- }
- }
- else
- throw MakeStringException((suspend)? QUERRREG_SUSPEND : QUERRREG_UNSUSPEND, "Modifying query suspended state failed. Could not find query %s", id);
- }
- void setQueryCommentForNamedQuery(IPropertyTree * queryRegistry, const char *id, const char *queryComment)
- {
- if (queryComment)
- {
- StringBuffer lcId(id);
- lcId.toLowerCase();
- StringBuffer xpath;
- xpath.append("Query[@id=\"").append(lcId).append("\"]");
- IPropertyTree *tree = queryRegistry->queryPropTree(xpath);
- if (tree)
- tree->setProp("@queryComment", queryComment);
- else
- throw MakeStringException(QUERRREG_COMMENT, "Could not find query %s", id);
- }
- }
- extern WORKUNIT_API IPropertyTree * getQueryRegistryRoot()
- {
- Owned<IRemoteConnection> conn = querySDS().connect("/QuerySets", myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT);
- if (conn)
- return conn->getRoot();
- else
- return NULL;
- }
- extern WORKUNIT_API void checkAddLibrariesToQueryEntry(IPropertyTree *queryTree, IConstWULibraryIterator *libraries)
- {
- if (!queryTree || !libraries)
- return;
- if (queryTree->hasProp("@libCount")) //already added
- return;
- unsigned libCount=0;
- ForEach(*libraries)
- {
- IConstWULibrary &library = libraries->query();
- SCMStringBuffer libname;
- if (!library.getName(libname).length())
- continue;
- queryTree->addProp("Library", libname.str());
- libCount++;
- }
- queryTree->setPropInt("@libCount", libCount);
- }
- extern WORKUNIT_API void checkAddLibrariesToQueryEntry(IPropertyTree *queryTree, IConstWorkUnit *cw)
- {
- Owned<IConstWULibraryIterator> libraries = &cw->getLibraries();
- checkAddLibrariesToQueryEntry(queryTree, libraries);
- }
- extern WORKUNIT_API IPropertyTree * getQueryRegistry(const char * wsEclId, bool readonly)
- {
- //Only lock the branch for the target we're interested in.
- StringBuffer xpath;
- xpath.append("/QuerySets/QuerySet[@id=\"").append(wsEclId).append("\"]");
- Owned<IRemoteConnection> conn = querySDS().connect(xpath.str(), myProcessSession(), readonly ? RTM_LOCK_READ : RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
- if (conn)
- return conn->getRoot();
- if (readonly)
- return NULL;
- //Lock the QuerySets in case another thread/client wants to check/add the same QuerySet.
- Owned<IRemoteConnection> globalLock = querySDS().connect("/QuerySets/", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
- //Re-check if the QuerySet has been added between checking the 1st time and gaining the globalLock.
- conn.setown(querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT));
- if (conn)
- return conn->getRoot();
- conn.setown(querySDS().connect("/QuerySets/QuerySet", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_ADD, SDS_LOCK_TIMEOUT));
- if (!conn)
- throwUnexpected();
- IPropertyTree * root = conn->queryRoot();
- root->setProp("@id",wsEclId);
- conn->commit();
- return LINK(root);
- }
- IPropertyTree * addNamedPackageSet(IPropertyTree * packageRegistry, const char * name, IPropertyTree *packageInfo, bool overWrite)
- {
- StringBuffer xpath;
- StringBuffer lcName(name);
- lcName.toLowerCase();
- // see if "name" already exists
- xpath.append("Package[@id='").append(name).append("']");
- IPropertyTree *pkgTree = packageRegistry->queryPropTree(xpath.str());
- if (pkgTree)
- {
- if (overWrite)
- packageRegistry->removeTree(pkgTree);
- else
- throw MakeStringException(WUERR_PackageAlreadyExists, "Package name %s already exists, either delete it or specify overwrite",name);
- }
-
- IPropertyTree *tree = packageRegistry->addPropTree("Package", packageInfo);
- tree->setProp("@id", lcName);
- return tree;
- }
- void removeNamedPackage(IPropertyTree * packageRegistry, const char * id)
- {
- StringBuffer lcId(id);
- lcId.toLowerCase();
- StringBuffer xpath;
- xpath.append("Package[@id=\"").append(lcId).append("\"]");
- packageRegistry->removeProp(xpath);
- }
- extern WORKUNIT_API IPropertyTree * getPackageSetRegistry(const char * wsEclId, bool readonly)
- {
- //Only lock the branch for the target we're interested in.
- StringBuffer xpath;
- xpath.append("/PackageSets/PackageSet[@id=\"").append(wsEclId).append("\"]");
- Owned<IRemoteConnection> conn = querySDS().connect(xpath.str(), myProcessSession(), readonly ? RTM_LOCK_READ : RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
- if (conn)
- return conn->getRoot();
- if (readonly)
- return NULL;
- //Lock the PackageSets in case another thread/client wants to check/add the same PackageSet.
- Owned<IRemoteConnection> globalLock = querySDS().connect("/PackageSets/", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
- //Re-check if the PackageSet has been added between checking the 1st time and gaining the globalLock.
- conn.setown(querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT));
- if (conn)
- return conn->getRoot();
- conn.setown(querySDS().connect("/PackageSets/PackageSet", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_ADD, SDS_LOCK_TIMEOUT));
- if (!conn)
- throwUnexpected();
- IPropertyTree* root = conn->queryRoot();
- root->setProp("@id",wsEclId);
- conn->commit();
- return LINK(root);
- }
- void addQueryToQuerySet(IWorkUnit *workunit, IPropertyTree *queryRegistry, const char *queryName, WUQueryActivationOptions activateOption, StringBuffer &newQueryId, const char *userid)
- {
- StringBuffer cleanQueryName;
- appendUtf8XmlName(cleanQueryName, strlen(queryName), queryName);
- SCMStringBuffer dllName;
- Owned<IConstWUQuery> q = workunit->getQuery();
- q->getQueryDllName(dllName);
- if (!dllName.length())
- throw MakeStringException(WUERR_InvalidDll, "Cannot deploy query - no associated dll.");
- StringBuffer currentTargetClusterType;
- queryRegistry->getProp("@targetclustertype", currentTargetClusterType);
- SCMStringBuffer targetClusterType;
- workunit->getDebugValue("targetclustertype", targetClusterType);
- SCMStringBuffer snapshot;
- workunit->getSnapshot(snapshot);
- if (currentTargetClusterType.length() < 1)
- {
- queryRegistry->setProp("@targetclustertype", targetClusterType.str());
- }
- else
- {
- if (strcmp(currentTargetClusterType.str(), "roxie") == 0 && strcmp(currentTargetClusterType.str(), targetClusterType.str())!=0)
- {
- throw MakeStringException(WUERR_MismatchClusterType, "TargetClusterTypes of workunit and queryset do not match.");
- }
- }
- IPropertyTree *newEntry = addNamedQuery(queryRegistry, cleanQueryName, workunit->queryWuid(), dllName.str(), isLibrary(workunit), userid, snapshot.str());
- Owned<IConstWULibraryIterator> libraries = &workunit->getLibraries();
- checkAddLibrariesToQueryEntry(newEntry, libraries);
- newQueryId.append(newEntry->queryProp("@id"));
- workunit->setIsQueryService(true); //will check querysets before delete
- workunit->commit();
- activateQuery(queryRegistry, activateOption, queryName, newQueryId, userid);
- }
- void activateQuery(IPropertyTree *queryRegistry, WUQueryActivationOptions activateOption, const char *queryName, const char *queryId, const char *userid)
- {
- StringBuffer cleanQueryName;
- appendUtf8XmlName(cleanQueryName, strlen(queryName), queryName);
- if (activateOption == ACTIVATE_SUSPEND_PREVIOUS|| activateOption == ACTIVATE_DELETE_PREVIOUS)
- {
- Owned<IPropertyTree> prevQuery = resolveQueryAlias(queryRegistry, cleanQueryName);
- setQueryAlias(queryRegistry, cleanQueryName, queryId);
- if (prevQuery && !streq(queryId, prevQuery->queryProp("@id")))
- {
- if (activateOption == ACTIVATE_SUSPEND_PREVIOUS)
- setQuerySuspendedState(queryRegistry, prevQuery->queryProp("@id"), true, userid);
- else
- removeNamedQuery(queryRegistry, prevQuery->queryProp("@id"));
- }
- }
- else if (activateOption == MAKE_ACTIVATE || activateOption == MAKE_ACTIVATE_LOAD_DATA_ONLY)
- setQueryAlias(queryRegistry, cleanQueryName, queryId);
- }
- void addQueryToQuerySet(IWorkUnit *workunit, const char *querySetName, const char *queryName, WUQueryActivationOptions activateOption, StringBuffer &newQueryId, const char *userid)
- {
- Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, false);
- addQueryToQuerySet(workunit, queryRegistry, queryName, activateOption, newQueryId, userid);
- }
- bool removeQuerySetAlias(const char *querySetName, const char *alias)
- {
- Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, true);
- StringBuffer xpath;
- xpath.appendf("Alias[@name='%s']", alias);
- IPropertyTree *t = queryRegistry->queryPropTree(xpath);
- return queryRegistry->removeTree(t);
- }
- void addQuerySetAlias(const char *querySetName, const char *alias, const char *id)
- {
- Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, false);
- setQueryAlias(queryRegistry, alias, id);
- }
- void setSuspendQuerySetQuery(const char *querySetName, const char *id, bool suspend, const char *userid)
- {
- Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, true);
- setQuerySuspendedState(queryRegistry, id, suspend, userid);
- }
- void deleteQuerySetQuery(const char *querySetName, const char *id)
- {
- Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, true);
- removeNamedQuery(queryRegistry, id);
- }
- void removeQuerySetAliasesFromNamedQuery(const char *querySetName, const char * id)
- {
- Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, true);
- clearAliases(queryRegistry, id);
- }
- void setQueryCommentForNamedQuery(const char *querySetName, const char *id, const char *queryComment)
- {
- Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, true);
- setQueryCommentForNamedQuery(queryRegistry, id, queryComment);
- }
- const char *queryIdFromQuerySetWuid(IPropertyTree *queryRegistry, const char *wuid, const char *queryName, IStringVal &id)
- {
- if (!queryRegistry)
- return NULL;
- StringBuffer xpath;
- xpath.appendf("Query[@wuid='%s']", wuid);
- if (queryName && *queryName)
- xpath.appendf("[@name='%s']", queryName);
- IPropertyTree *q = queryRegistry->queryPropTree(xpath.str());
- if (q)
- {
- id.set(q->queryProp("@id"));
- }
- return id.str();
- }
- const char *queryIdFromQuerySetWuid(const char *querySetName, const char *wuid, const char *queryName, IStringVal &id)
- {
- Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, true);
- return queryIdFromQuerySetWuid(queryRegistry, wuid, queryName, id);
- }
- extern WORKUNIT_API void gatherLibraryNames(StringArray &names, StringArray &unresolved, IWorkUnitFactory &workunitFactory, IConstWorkUnit &cw, IPropertyTree *queryset)
- {
- Owned<IConstWULibraryIterator> wulibraries = &cw.getLibraries();
- ForEach(*wulibraries)
- {
- SCMStringBuffer libname;
- IConstWULibrary &wulibrary = wulibraries->query();
- wulibrary.getName(libname);
- if (names.contains(libname.str()) || unresolved.contains(libname.str()))
- continue;
- Owned<IPropertyTree> query = resolveQueryAlias(queryset, libname.str());
- if (query && query->getPropBool("@isLibrary"))
- {
- const char *wuid = query->queryProp("@wuid");
- Owned<IConstWorkUnit> libcw = workunitFactory.openWorkUnit(wuid, false);
- if (libcw)
- {
- names.appendUniq(libname.str());
- gatherLibraryNames(names, unresolved, workunitFactory, *libcw, queryset);
- continue;
- }
- }
- unresolved.appendUniq(libname.str());
- }
- }
- bool looksLikeAWuid(const char * wuid)
- {
- if (!wuid)
- return false;
- if (wuid[0] != 'W')
- return false;
- if (!isdigit(wuid[1]) || !isdigit(wuid[2]) || !isdigit(wuid[3]) || !isdigit(wuid[4]))
- return false;
- if (!isdigit(wuid[5]) || !isdigit(wuid[6]) || !isdigit(wuid[7]) || !isdigit(wuid[8]))
- return false;
- return (wuid[9]=='-');
- }
- IPropertyTree * resolveDefinitionInArchive(IPropertyTree * archive, const char * path)
- {
- IPropertyTree * module = archive;
- const char * dot = strrchr(path, '.');
- StringBuffer xpath;
- if (dot)
- {
- xpath.clear().append("Module[@key='").appendLower(dot-path, path).append("']");
- module = archive->queryPropTree(xpath);
- path = dot+1;
- }
- else
- module = archive->queryPropTree("Module[@key='']");
- if (!module)
- return NULL;
- xpath.clear().append("Attribute[@key='").appendLower(strlen(path), path).append("']");
- return module->queryPropTree(xpath);
- }
- extern WORKUNIT_API void associateLocalFile(IWUQuery * query, WUFileType type, const char * name, const char * description, unsigned crc)
- {
- StringBuffer hostname;
- queryHostIP().getIpText(hostname);
- StringBuffer fullPathname;
- makeAbsolutePath(name, fullPathname);
- query->addAssociatedFile(type, fullPathname, hostname, description, crc);
- }
- extern WORKUNIT_API void descheduleWorkunit(char const * wuid)
- {
- Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
- Owned<IWorkUnit> workunit = factory->updateWorkUnit(wuid);
- if(workunit)
- workunit->deschedule();
- else
- doDescheduleWorkkunit(wuid);
- }
- extern WORKUNIT_API void updateWorkunitTimeStat(IWorkUnit * wu, StatisticScopeType scopeType, const char * scope, StatisticKind kind, const char * description, unsigned __int64 value)
- {
- wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), scopeType, scope, kind, description, value, 1, 0, StatsMergeReplace);
- }
- extern WORKUNIT_API void updateWorkunitTimings(IWorkUnit * wu, ITimeReporter *timer)
- {
- StringBuffer scope;
- for (unsigned i = 0; i < timer->numSections(); i++)
- {
- StatisticScopeType scopeType= timer->getScopeType(i);
- timer->getScope(i, scope.clear());
- StatisticKind kind = timer->getTimerType(i);
- wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), scopeType, scope, kind, NULL, timer->getTime(i), timer->getCount(i), timer->getMaxTime(i), StatsMergeReplace);
- }
- }
- extern WORKUNIT_API void getWorkunitTotalTime(IConstWorkUnit* workunit, const char* creator, unsigned __int64 & totalTimeNs, unsigned __int64 & totalThisTimeNs)
- {
- StatisticsFilter summaryTimeFilter(SCTsummary, creator, SSTglobal, GLOBAL_SCOPE, SMeasureTimeNs, StTimeElapsed);
- Owned<IConstWUStatistic> totalThorTime = getStatistic(workunit, summaryTimeFilter);
- Owned<IConstWUStatistic> totalThisThorTime = workunit->getStatistic(queryStatisticsComponentName(), GLOBAL_SCOPE, StTimeElapsed);
- if (totalThorTime)
- totalTimeNs = totalThorTime->getValue();
- else
- totalTimeNs = 0;
- if (totalThisThorTime)
- totalThisTimeNs = totalThisThorTime->getValue();
- else
- totalThisTimeNs = 0;
- }
- extern WORKUNIT_API void addTimeStamp(IWorkUnit * wu, StatisticScopeType scopeType, const char * scope, StatisticKind kind)
- {
- wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), scopeType, scope, kind, NULL, getTimeStampNowValue(), 1, 0, StatsMergeAppend);
- }
- IConstWUStatistic * getStatistic(IConstWorkUnit * wu, const IStatisticsFilter & filter)
- {
- Owned<IConstWUStatisticIterator> iter = &wu->getStatistics(&filter);
- if (iter->first())
- return &OLINK(iter->query());
- return NULL;
- }
- class GlobalStatisticGatherer : public CInterfaceOf<IStatisticGatherer>
- {
- public:
- GlobalStatisticGatherer(IWorkUnit * _wu) : wu(_wu) {}
- virtual void beginScope(const StatsScopeId & id)
- {
- prevLenStack.append(scope.length());
- if (scope.length())
- scope.append(":");
- id.getScopeText(scope);
- scopeTypeStack.append(id.queryScopeType());
- }
- virtual void beginSubGraphScope(unsigned id)
- {
- StatsScopeId scopeId(SSTsubgraph, id);
- beginScope(scopeId);
- }
- virtual void beginActivityScope(unsigned id)
- {
- StatsScopeId scopeId(SSTactivity, id);
- beginScope(scopeId);
- }
- virtual void beginEdgeScope(unsigned id, unsigned oid)
- {
- StatsScopeId scopeId(SSTedge, id, oid);
- beginScope(scopeId);
- }
- virtual void endScope()
- {
- scope.setLength(prevLenStack.popGet());
- scopeTypeStack.pop();
- }
- virtual void addStatistic(StatisticKind kind, unsigned __int64 value)
- {
- StatisticScopeType scopeType = scopeTypeStack.ordinality() ? (StatisticScopeType)scopeTypeStack.tos() : SSTglobal;
- wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), scopeType, scope, kind, NULL, value, 1, 0, StatsMergeAppend);
- }
- virtual void updateStatistic(StatisticKind kind, unsigned __int64 value, StatsMergeAction mergeAction)
- {
- StatisticScopeType scopeType = scopeTypeStack.ordinality() ? (StatisticScopeType)scopeTypeStack.tos() : SSTglobal;
- wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), scopeType, scope, kind, NULL, value, 1, 0, mergeAction);
- }
- virtual IStatisticCollection * getResult()
- {
- return NULL;
- }
- protected:
- Linked<IWorkUnit> wu;
- StringBuffer scope;
- UnsignedArray prevLenStack;
- UnsignedArray scopeTypeStack;
- };
- IStatisticGatherer * createGlobalStatisticGatherer(IWorkUnit * wu)
- {
- return new GlobalStatisticGatherer(wu);
- }
|