workunit.cpp 445 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853685468556856685768586859686068616862686368646865686668676868686968706871687268736874687568766877687868796880688168826883688468856886688768886889689068916892689368946895689668976898689969006901690269036904690569066907690869096910691169126913691469156916691769186919692069216922692369246925692669276928692969306931693269336934693569366937693869396940694169426943694469456946694769486949695069516952695369546955695669576958695969606961696269636964696569666967696869696970697169726973697469756976697769786979698069816982698369846985698669876988698969906991699269936994699569966997699869997000700170027003700470057006700770087009701070117012701370147015701670177018701970207021702270237024702570267027702870297030703170327033703470357036703770387039704070417042704370447045704670477048704970507051705270537054705570567057705870597060706170627063706470657066706770687069707070717072707370747075707670777078707970807081708270837084708570867087708870897090709170927093709470957096709770987099710071017102710371047105710671077108710971107111711271137114711571167117711871197120712171227123712471257126712771287129713071317132713371347135713671377138713971407141714271437144714571467147714871497150715171527153715471557156715771587159716071617162716371647165716671677168716971707171717271737174717571767177717871797180718171827183718471857186718771887189719071917192719371947195719671977198719972007201720272037204720572067207720872097210721172127213721472157216721772187219722072217222722372247225722672277228722972307231723272337234723572367237723872397240724172427243724472457246724772487249725072517252725372547255725672577258725972607261726272637264726572667267726872697270727172727273727472757276727772787279728072817282728372847285728672877288728972907291729272937294729572967297729872997300730173027303730473057306730773087309731073117312731373147315731673177318731973207321732273237324732573267327732873297330733173327333733473357336733773387339734073417342734373447345734673477348734973507351735273537354735573567357735873597360736173627363736473657366736773687369737073717372737373747375737673777378737973807381738273837384738573867387738873897390739173927393739473957396739773987399740074017402740374047405740674077408740974107411741274137414741574167417741874197420742174227423742474257426742774287429743074317432743374347435743674377438743974407441744274437444744574467447744874497450745174527453745474557456745774587459746074617462746374647465746674677468746974707471747274737474747574767477747874797480748174827483748474857486748774887489749074917492749374947495749674977498749975007501750275037504750575067507750875097510751175127513751475157516751775187519752075217522752375247525752675277528752975307531753275337534753575367537753875397540754175427543754475457546754775487549755075517552755375547555755675577558755975607561756275637564756575667567756875697570757175727573757475757576757775787579758075817582758375847585758675877588758975907591759275937594759575967597759875997600760176027603760476057606760776087609761076117612761376147615761676177618761976207621762276237624762576267627762876297630763176327633763476357636763776387639764076417642764376447645764676477648764976507651765276537654765576567657765876597660766176627663766476657666766776687669767076717672767376747675767676777678767976807681768276837684768576867687768876897690769176927693769476957696769776987699770077017702770377047705770677077708770977107711771277137714771577167717771877197720772177227723772477257726772777287729773077317732773377347735773677377738773977407741774277437744774577467747774877497750775177527753775477557756775777587759776077617762776377647765776677677768776977707771777277737774777577767777777877797780778177827783778477857786778777887789779077917792779377947795779677977798779978007801780278037804780578067807780878097810781178127813781478157816781778187819782078217822782378247825782678277828782978307831783278337834783578367837783878397840784178427843784478457846784778487849785078517852785378547855785678577858785978607861786278637864786578667867786878697870787178727873787478757876787778787879788078817882788378847885788678877888788978907891789278937894789578967897789878997900790179027903790479057906790779087909791079117912791379147915791679177918791979207921792279237924792579267927792879297930793179327933793479357936793779387939794079417942794379447945794679477948794979507951795279537954795579567957795879597960796179627963796479657966796779687969797079717972797379747975797679777978797979807981798279837984798579867987798879897990799179927993799479957996799779987999800080018002800380048005800680078008800980108011801280138014801580168017801880198020802180228023802480258026802780288029803080318032803380348035803680378038803980408041804280438044804580468047804880498050805180528053805480558056805780588059806080618062806380648065806680678068806980708071807280738074807580768077807880798080808180828083808480858086808780888089809080918092809380948095809680978098809981008101810281038104810581068107810881098110811181128113811481158116811781188119812081218122812381248125812681278128812981308131813281338134813581368137813881398140814181428143814481458146814781488149815081518152815381548155815681578158815981608161816281638164816581668167816881698170817181728173817481758176817781788179818081818182818381848185818681878188818981908191819281938194819581968197819881998200820182028203820482058206820782088209821082118212821382148215821682178218821982208221822282238224822582268227822882298230823182328233823482358236823782388239824082418242824382448245824682478248824982508251825282538254825582568257825882598260826182628263826482658266826782688269827082718272827382748275827682778278827982808281828282838284828582868287828882898290829182928293829482958296829782988299830083018302830383048305830683078308830983108311831283138314831583168317831883198320832183228323832483258326832783288329833083318332833383348335833683378338833983408341834283438344834583468347834883498350835183528353835483558356835783588359836083618362836383648365836683678368836983708371837283738374837583768377837883798380838183828383838483858386838783888389839083918392839383948395839683978398839984008401840284038404840584068407840884098410841184128413841484158416841784188419842084218422842384248425842684278428842984308431843284338434843584368437843884398440844184428443844484458446844784488449845084518452845384548455845684578458845984608461846284638464846584668467846884698470847184728473847484758476847784788479848084818482848384848485848684878488848984908491849284938494849584968497849884998500850185028503850485058506850785088509851085118512851385148515851685178518851985208521852285238524852585268527852885298530853185328533853485358536853785388539854085418542854385448545854685478548854985508551855285538554855585568557855885598560856185628563856485658566856785688569857085718572857385748575857685778578857985808581858285838584858585868587858885898590859185928593859485958596859785988599860086018602860386048605860686078608860986108611861286138614861586168617861886198620862186228623862486258626862786288629863086318632863386348635863686378638863986408641864286438644864586468647864886498650865186528653865486558656865786588659866086618662866386648665866686678668866986708671867286738674867586768677867886798680868186828683868486858686868786888689869086918692869386948695869686978698869987008701870287038704870587068707870887098710871187128713871487158716871787188719872087218722872387248725872687278728872987308731873287338734873587368737873887398740874187428743874487458746874787488749875087518752875387548755875687578758875987608761876287638764876587668767876887698770877187728773877487758776877787788779878087818782878387848785878687878788878987908791879287938794879587968797879887998800880188028803880488058806880788088809881088118812881388148815881688178818881988208821882288238824882588268827882888298830883188328833883488358836883788388839884088418842884388448845884688478848884988508851885288538854885588568857885888598860886188628863886488658866886788688869887088718872887388748875887688778878887988808881888288838884888588868887888888898890889188928893889488958896889788988899890089018902890389048905890689078908890989108911891289138914891589168917891889198920892189228923892489258926892789288929893089318932893389348935893689378938893989408941894289438944894589468947894889498950895189528953895489558956895789588959896089618962896389648965896689678968896989708971897289738974897589768977897889798980898189828983898489858986898789888989899089918992899389948995899689978998899990009001900290039004900590069007900890099010901190129013901490159016901790189019902090219022902390249025902690279028902990309031903290339034903590369037903890399040904190429043904490459046904790489049905090519052905390549055905690579058905990609061906290639064906590669067906890699070907190729073907490759076907790789079908090819082908390849085908690879088908990909091909290939094909590969097909890999100910191029103910491059106910791089109911091119112911391149115911691179118911991209121912291239124912591269127912891299130913191329133913491359136913791389139914091419142914391449145914691479148914991509151915291539154915591569157915891599160916191629163916491659166916791689169917091719172917391749175917691779178917991809181918291839184918591869187918891899190919191929193919491959196919791989199920092019202920392049205920692079208920992109211921292139214921592169217921892199220922192229223922492259226922792289229923092319232923392349235923692379238923992409241924292439244924592469247924892499250925192529253925492559256925792589259926092619262926392649265926692679268926992709271927292739274927592769277927892799280928192829283928492859286928792889289929092919292929392949295929692979298929993009301930293039304930593069307930893099310931193129313931493159316931793189319932093219322932393249325932693279328932993309331933293339334933593369337933893399340934193429343934493459346934793489349935093519352935393549355935693579358935993609361936293639364936593669367936893699370937193729373937493759376937793789379938093819382938393849385938693879388938993909391939293939394939593969397939893999400940194029403940494059406940794089409941094119412941394149415941694179418941994209421942294239424942594269427942894299430943194329433943494359436943794389439944094419442944394449445944694479448944994509451945294539454945594569457945894599460946194629463946494659466946794689469947094719472947394749475947694779478947994809481948294839484948594869487948894899490949194929493949494959496949794989499950095019502950395049505950695079508950995109511951295139514951595169517951895199520952195229523952495259526952795289529953095319532953395349535953695379538953995409541954295439544954595469547954895499550955195529553955495559556955795589559956095619562956395649565956695679568956995709571957295739574957595769577957895799580958195829583958495859586958795889589959095919592959395949595959695979598959996009601960296039604960596069607960896099610961196129613961496159616961796189619962096219622962396249625962696279628962996309631963296339634963596369637963896399640964196429643964496459646964796489649965096519652965396549655965696579658965996609661966296639664966596669667966896699670967196729673967496759676967796789679968096819682968396849685968696879688968996909691969296939694969596969697969896999700970197029703970497059706970797089709971097119712971397149715971697179718971997209721972297239724972597269727972897299730973197329733973497359736973797389739974097419742974397449745974697479748974997509751975297539754975597569757975897599760976197629763976497659766976797689769977097719772977397749775977697779778977997809781978297839784978597869787978897899790979197929793979497959796979797989799980098019802980398049805980698079808980998109811981298139814981598169817981898199820982198229823982498259826982798289829983098319832983398349835983698379838983998409841984298439844984598469847984898499850985198529853985498559856985798589859986098619862986398649865986698679868986998709871987298739874987598769877987898799880988198829883988498859886988798889889989098919892989398949895989698979898989999009901990299039904990599069907990899099910991199129913991499159916991799189919992099219922992399249925992699279928992999309931993299339934993599369937993899399940994199429943994499459946994799489949995099519952995399549955995699579958995999609961996299639964996599669967996899699970997199729973997499759976997799789979998099819982998399849985998699879988998999909991999299939994999599969997999899991000010001100021000310004100051000610007100081000910010100111001210013100141001510016100171001810019100201002110022100231002410025100261002710028100291003010031100321003310034100351003610037100381003910040100411004210043100441004510046100471004810049100501005110052100531005410055100561005710058100591006010061100621006310064100651006610067100681006910070100711007210073100741007510076100771007810079100801008110082100831008410085100861008710088100891009010091100921009310094100951009610097100981009910100101011010210103101041010510106101071010810109101101011110112101131011410115101161011710118101191012010121101221012310124101251012610127101281012910130101311013210133101341013510136101371013810139101401014110142101431014410145101461014710148101491015010151101521015310154101551015610157101581015910160101611016210163101641016510166101671016810169101701017110172101731017410175101761017710178101791018010181101821018310184101851018610187101881018910190101911019210193101941019510196101971019810199102001020110202102031020410205102061020710208102091021010211102121021310214102151021610217102181021910220102211022210223102241022510226102271022810229102301023110232102331023410235102361023710238102391024010241102421024310244102451024610247102481024910250102511025210253102541025510256102571025810259102601026110262102631026410265102661026710268102691027010271102721027310274102751027610277102781027910280102811028210283102841028510286102871028810289102901029110292102931029410295102961029710298102991030010301103021030310304103051030610307103081030910310103111031210313103141031510316103171031810319103201032110322103231032410325103261032710328103291033010331103321033310334103351033610337103381033910340103411034210343103441034510346103471034810349103501035110352103531035410355103561035710358103591036010361103621036310364103651036610367103681036910370103711037210373103741037510376103771037810379103801038110382103831038410385103861038710388103891039010391103921039310394103951039610397103981039910400104011040210403104041040510406104071040810409104101041110412104131041410415104161041710418104191042010421104221042310424104251042610427104281042910430104311043210433104341043510436104371043810439104401044110442104431044410445104461044710448104491045010451104521045310454104551045610457104581045910460104611046210463104641046510466104671046810469104701047110472104731047410475104761047710478104791048010481104821048310484104851048610487104881048910490104911049210493104941049510496104971049810499105001050110502105031050410505105061050710508105091051010511105121051310514105151051610517105181051910520105211052210523105241052510526105271052810529105301053110532105331053410535105361053710538105391054010541105421054310544105451054610547105481054910550105511055210553105541055510556105571055810559105601056110562105631056410565105661056710568105691057010571105721057310574105751057610577105781057910580105811058210583105841058510586105871058810589105901059110592105931059410595105961059710598105991060010601106021060310604106051060610607106081060910610106111061210613106141061510616106171061810619106201062110622106231062410625106261062710628106291063010631106321063310634106351063610637106381063910640106411064210643106441064510646106471064810649106501065110652106531065410655106561065710658106591066010661106621066310664106651066610667106681066910670106711067210673106741067510676106771067810679106801068110682106831068410685106861068710688106891069010691106921069310694106951069610697106981069910700107011070210703107041070510706107071070810709107101071110712107131071410715107161071710718107191072010721107221072310724107251072610727107281072910730107311073210733107341073510736107371073810739107401074110742107431074410745107461074710748107491075010751107521075310754107551075610757107581075910760107611076210763107641076510766107671076810769107701077110772107731077410775107761077710778107791078010781107821078310784107851078610787107881078910790107911079210793107941079510796107971079810799108001080110802108031080410805108061080710808108091081010811108121081310814108151081610817108181081910820108211082210823108241082510826108271082810829108301083110832108331083410835108361083710838108391084010841108421084310844108451084610847108481084910850108511085210853108541085510856108571085810859108601086110862108631086410865108661086710868108691087010871108721087310874108751087610877108781087910880108811088210883108841088510886108871088810889108901089110892108931089410895108961089710898108991090010901109021090310904109051090610907109081090910910109111091210913109141091510916109171091810919109201092110922109231092410925109261092710928109291093010931109321093310934109351093610937109381093910940109411094210943109441094510946109471094810949109501095110952109531095410955109561095710958109591096010961109621096310964109651096610967109681096910970109711097210973109741097510976109771097810979109801098110982109831098410985109861098710988109891099010991109921099310994109951099610997109981099911000110011100211003110041100511006110071100811009110101101111012110131101411015110161101711018110191102011021110221102311024110251102611027110281102911030110311103211033110341103511036110371103811039110401104111042110431104411045110461104711048110491105011051110521105311054110551105611057110581105911060110611106211063110641106511066110671106811069110701107111072110731107411075110761107711078110791108011081110821108311084110851108611087110881108911090110911109211093110941109511096110971109811099111001110111102111031110411105111061110711108111091111011111111121111311114111151111611117111181111911120111211112211123111241112511126111271112811129111301113111132111331113411135111361113711138111391114011141111421114311144111451114611147111481114911150111511115211153111541115511156111571115811159111601116111162111631116411165111661116711168111691117011171111721117311174111751117611177111781117911180111811118211183111841118511186111871118811189111901119111192111931119411195111961119711198111991120011201112021120311204112051120611207112081120911210112111121211213112141121511216112171121811219112201122111222112231122411225112261122711228112291123011231112321123311234112351123611237112381123911240112411124211243112441124511246112471124811249112501125111252112531125411255112561125711258112591126011261112621126311264112651126611267112681126911270112711127211273112741127511276112771127811279112801128111282112831128411285112861128711288112891129011291112921129311294112951129611297112981129911300113011130211303113041130511306113071130811309113101131111312113131131411315113161131711318113191132011321113221132311324113251132611327113281132911330113311133211333113341133511336113371133811339113401134111342113431134411345113461134711348113491135011351113521135311354113551135611357113581135911360113611136211363113641136511366113671136811369113701137111372113731137411375113761137711378113791138011381113821138311384113851138611387113881138911390113911139211393113941139511396113971139811399114001140111402114031140411405114061140711408114091141011411114121141311414114151141611417114181141911420114211142211423114241142511426114271142811429114301143111432114331143411435114361143711438114391144011441114421144311444114451144611447114481144911450114511145211453114541145511456114571145811459114601146111462114631146411465114661146711468114691147011471114721147311474114751147611477114781147911480114811148211483114841148511486114871148811489114901149111492114931149411495114961149711498114991150011501115021150311504115051150611507115081150911510115111151211513115141151511516115171151811519115201152111522115231152411525115261152711528115291153011531115321153311534115351153611537115381153911540115411154211543115441154511546115471154811549115501155111552115531155411555115561155711558115591156011561115621156311564115651156611567115681156911570115711157211573115741157511576115771157811579115801158111582115831158411585115861158711588115891159011591115921159311594115951159611597115981159911600116011160211603116041160511606116071160811609116101161111612116131161411615116161161711618116191162011621116221162311624116251162611627116281162911630116311163211633116341163511636116371163811639116401164111642116431164411645116461164711648116491165011651116521165311654116551165611657116581165911660116611166211663116641166511666116671166811669116701167111672116731167411675116761167711678116791168011681116821168311684116851168611687116881168911690116911169211693116941169511696116971169811699117001170111702117031170411705117061170711708117091171011711117121171311714117151171611717117181171911720117211172211723117241172511726117271172811729117301173111732117331173411735117361173711738117391174011741117421174311744117451174611747117481174911750117511175211753117541175511756117571175811759117601176111762117631176411765117661176711768117691177011771117721177311774117751177611777117781177911780117811178211783117841178511786117871178811789117901179111792117931179411795117961179711798117991180011801118021180311804118051180611807118081180911810118111181211813118141181511816118171181811819118201182111822118231182411825118261182711828118291183011831118321183311834118351183611837118381183911840118411184211843118441184511846118471184811849118501185111852118531185411855118561185711858118591186011861118621186311864118651186611867118681186911870118711187211873118741187511876118771187811879118801188111882118831188411885118861188711888118891189011891118921189311894118951189611897118981189911900119011190211903119041190511906119071190811909119101191111912119131191411915119161191711918119191192011921119221192311924119251192611927119281192911930119311193211933119341193511936119371193811939119401194111942119431194411945119461194711948119491195011951119521195311954119551195611957119581195911960119611196211963119641196511966119671196811969119701197111972119731197411975119761197711978119791198011981119821198311984119851198611987119881198911990119911199211993119941199511996119971199811999120001200112002120031200412005120061200712008120091201012011120121201312014120151201612017120181201912020120211202212023120241202512026120271202812029120301203112032120331203412035120361203712038120391204012041120421204312044120451204612047120481204912050120511205212053120541205512056120571205812059120601206112062120631206412065120661206712068120691207012071120721207312074120751207612077120781207912080120811208212083120841208512086120871208812089120901209112092120931209412095120961209712098120991210012101121021210312104121051210612107121081210912110121111211212113121141211512116121171211812119121201212112122121231212412125121261212712128121291213012131121321213312134121351213612137121381213912140121411214212143121441214512146121471214812149121501215112152121531215412155121561215712158121591216012161121621216312164121651216612167121681216912170121711217212173121741217512176121771217812179121801218112182121831218412185121861218712188121891219012191121921219312194121951219612197121981219912200122011220212203122041220512206122071220812209122101221112212122131221412215122161221712218122191222012221122221222312224122251222612227122281222912230122311223212233122341223512236122371223812239122401224112242122431224412245122461224712248122491225012251122521225312254122551225612257122581225912260122611226212263122641226512266122671226812269122701227112272122731227412275122761227712278122791228012281122821228312284122851228612287122881228912290122911229212293122941229512296122971229812299123001230112302123031230412305123061230712308123091231012311123121231312314123151231612317123181231912320123211232212323123241232512326123271232812329123301233112332123331233412335123361233712338123391234012341123421234312344123451234612347123481234912350123511235212353123541235512356123571235812359123601236112362123631236412365123661236712368123691237012371123721237312374123751237612377123781237912380123811238212383123841238512386123871238812389123901239112392123931239412395123961239712398123991240012401124021240312404124051240612407124081240912410124111241212413124141241512416124171241812419124201242112422124231242412425124261242712428124291243012431124321243312434124351243612437124381243912440124411244212443124441244512446124471244812449124501245112452124531245412455124561245712458124591246012461124621246312464124651246612467124681246912470124711247212473124741247512476124771247812479124801248112482124831248412485124861248712488124891249012491124921249312494124951249612497124981249912500125011250212503125041250512506125071250812509125101251112512125131251412515125161251712518125191252012521125221252312524125251252612527125281252912530125311253212533125341253512536125371253812539125401254112542125431254412545125461254712548125491255012551125521255312554125551255612557125581255912560125611256212563125641256512566125671256812569125701257112572125731257412575125761257712578125791258012581125821258312584125851258612587125881258912590125911259212593125941259512596125971259812599126001260112602126031260412605126061260712608126091261012611126121261312614126151261612617126181261912620126211262212623126241262512626126271262812629126301263112632126331263412635126361263712638126391264012641126421264312644126451264612647126481264912650126511265212653126541265512656126571265812659126601266112662126631266412665126661266712668126691267012671126721267312674126751267612677126781267912680126811268212683126841268512686126871268812689126901269112692126931269412695126961269712698126991270012701127021270312704127051270612707127081270912710127111271212713127141271512716127171271812719127201272112722127231272412725127261272712728127291273012731127321273312734127351273612737127381273912740127411274212743127441274512746127471274812749127501275112752127531275412755127561275712758127591276012761127621276312764127651276612767127681276912770127711277212773127741277512776127771277812779127801278112782127831278412785127861278712788127891279012791127921279312794127951279612797127981279912800128011280212803128041280512806128071280812809128101281112812128131281412815128161281712818128191282012821128221282312824128251282612827128281282912830128311283212833128341283512836128371283812839128401284112842128431284412845128461284712848128491285012851128521285312854128551285612857128581285912860128611286212863128641286512866128671286812869128701287112872128731287412875128761287712878128791288012881128821288312884128851288612887128881288912890128911289212893128941289512896128971289812899129001290112902129031290412905129061290712908129091291012911129121291312914129151291612917129181291912920129211292212923129241292512926129271292812929129301293112932129331293412935129361293712938129391294012941129421294312944129451294612947129481294912950129511295212953129541295512956129571295812959129601296112962129631296412965129661296712968129691297012971129721297312974129751297612977129781297912980129811298212983129841298512986129871298812989129901299112992129931299412995129961299712998129991300013001130021300313004130051300613007130081300913010130111301213013130141301513016130171301813019130201302113022130231302413025130261302713028130291303013031130321303313034130351303613037130381303913040130411304213043130441304513046130471304813049130501305113052130531305413055130561305713058130591306013061130621306313064130651306613067130681306913070130711307213073130741307513076130771307813079130801308113082130831308413085130861308713088130891309013091130921309313094130951309613097130981309913100131011310213103131041310513106131071310813109131101311113112131131311413115131161311713118131191312013121131221312313124131251312613127131281312913130131311313213133131341313513136131371313813139131401314113142131431314413145131461314713148131491315013151131521315313154131551315613157131581315913160131611316213163131641316513166131671316813169131701317113172131731317413175131761317713178131791318013181131821318313184131851318613187131881318913190131911319213193131941319513196131971319813199132001320113202132031320413205132061320713208132091321013211132121321313214132151321613217132181321913220132211322213223132241322513226132271322813229132301323113232132331323413235132361323713238132391324013241132421324313244132451324613247132481324913250132511325213253132541325513256132571325813259132601326113262132631326413265132661326713268132691327013271132721327313274132751327613277132781327913280132811328213283132841328513286132871328813289132901329113292132931329413295132961329713298132991330013301133021330313304133051330613307133081330913310133111331213313133141331513316133171331813319133201332113322133231332413325133261332713328133291333013331133321333313334133351333613337133381333913340133411334213343133441334513346133471334813349133501335113352133531335413355133561335713358133591336013361133621336313364133651336613367133681336913370133711337213373133741337513376133771337813379133801338113382133831338413385133861338713388133891339013391133921339313394133951339613397133981339913400134011340213403134041340513406134071340813409134101341113412134131341413415134161341713418134191342013421134221342313424134251342613427134281342913430134311343213433134341343513436134371343813439134401344113442134431344413445134461344713448134491345013451134521345313454134551345613457134581345913460134611346213463134641346513466134671346813469134701347113472134731347413475134761347713478134791348013481134821348313484134851348613487134881348913490134911349213493134941349513496134971349813499135001350113502135031350413505135061350713508135091351013511135121351313514135151351613517135181351913520135211352213523135241352513526135271352813529135301353113532135331353413535135361353713538135391354013541135421354313544135451354613547135481354913550135511355213553135541355513556135571355813559135601356113562135631356413565135661356713568135691357013571135721357313574135751357613577135781357913580135811358213583135841358513586135871358813589135901359113592135931359413595135961359713598135991360013601136021360313604136051360613607136081360913610136111361213613136141361513616136171361813619136201362113622136231362413625136261362713628136291363013631136321363313634136351363613637136381363913640136411364213643136441364513646136471364813649136501365113652136531365413655136561365713658136591366013661136621366313664136651366613667136681366913670136711367213673136741367513676136771367813679136801368113682136831368413685136861368713688136891369013691136921369313694136951369613697136981369913700137011370213703137041370513706137071370813709137101371113712137131371413715137161371713718137191372013721137221372313724137251372613727137281372913730137311373213733137341373513736137371373813739137401374113742137431374413745137461374713748137491375013751137521375313754137551375613757137581375913760137611376213763137641376513766137671376813769137701377113772137731377413775137761377713778137791378013781137821378313784137851378613787137881378913790
  1. /*##############################################################################
  2. HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. ############################################################################## */
  13. #include "jlib.hpp"
  14. #include "workunit.hpp"
  15. #include "jprop.hpp"
  16. #include "jmisc.hpp"
  17. #include "jexcept.hpp"
  18. #include "jiter.ipp"
  19. #include "jptree.hpp"
  20. #include "jtime.ipp"
  21. #include "jencrypt.hpp"
  22. #include "junicode.hpp"
  23. #include "jlzw.hpp"
  24. #include "jregexp.hpp"
  25. #include "eclrtl.hpp"
  26. #include "deftype.hpp"
  27. #include <time.h>
  28. #include "mpbase.hpp"
  29. #include "daclient.hpp"
  30. #include "dadfs.hpp"
  31. #include "dafdesc.hpp"
  32. #include "dasds.hpp"
  33. #include "danqs.hpp"
  34. #include "dautils.hpp"
  35. #include "dllserver.hpp"
  36. #include "thorplugin.hpp"
  37. #include "thorhelper.hpp"
  38. #include "workflow.hpp"
  39. #include "nbcd.hpp"
  40. #include "seclib.hpp"
  41. #include "wuerror.hpp"
  42. #include "wujobq.hpp"
  43. #include "environment.hpp"
  44. #include "workunit.ipp"
  45. #include "digisign.hpp"
  46. using namespace cryptohelper;
  47. static int workUnitTraceLevel = 1;
  48. static StringBuffer &getXPath(StringBuffer &wuRoot, const char *wuid)
  49. {
  50. // MORE - can fold in the date
  51. return wuRoot.append("/WorkUnits/").append(wuid);
  52. }
  53. //To be called by eclserver, but esp etc. won't know, so we need to store it.
  54. static StringBuffer & appendLibrarySuffix(StringBuffer & suffix)
  55. {
  56. #ifdef _WIN32
  57. suffix.append("W");
  58. #else
  59. suffix.append("L");
  60. #endif
  61. #ifdef __64BIT__
  62. suffix.append("64");
  63. #else
  64. suffix.append("32");
  65. #endif
  66. return suffix;
  67. }
  68. typedef MapStringTo<bool> UniqueScopes;
  69. static void wuAccessError(const char *username, const char *action, const char *wuscope, const char *wuid, bool excpt, bool log)
  70. {
  71. StringBuffer err;
  72. err.append("Workunit Access Denied - action: ").append(action).append(" user:").append(username ? username : "<Unknown>");
  73. if (wuid)
  74. err.append(" workunit:").append(wuid);
  75. if (wuscope)
  76. err.append(" scope:").append(wuscope);
  77. //MORE - we would need more information passed in from outside if we want to make the audit message format the same as from higher level ESP calls
  78. SYSLOG(AUDIT_TYPE_ACCESS_FAILURE, err.str());
  79. if (log)
  80. LOG(MCuserError, "%s", err.str());
  81. if (excpt)
  82. throw MakeStringException(WUERR_AccessError, "%s", err.str());
  83. }
  84. static bool checkWuScopeSecAccess(const char *wuscope, ISecManager *secmgr, ISecUser *secuser, int required, const char *action, bool excpt, bool log)
  85. {
  86. if (!secmgr || !secuser)
  87. return true;
  88. bool ret = secmgr->authorizeEx(RT_WORKUNIT_SCOPE, *secuser, wuscope)>=required;
  89. if (!ret && (log || excpt))
  90. wuAccessError(secuser->getName(), action, wuscope, NULL, excpt, log);
  91. return ret;
  92. }
  93. static bool checkWuScopeListSecAccess(const char *wuscope, ISecResourceList *scopes, int required, const char *action, bool excpt, bool log)
  94. {
  95. if (!scopes)
  96. return true;
  97. bool ret=true;
  98. if (wuscope)
  99. {
  100. Owned<ISecResource> res=scopes->getResource(wuscope);
  101. if (!res || res->getAccessFlags()<required)
  102. ret=false;
  103. }
  104. else
  105. {
  106. for (int seq=0; ret && seq<scopes->count(); seq++)
  107. {
  108. ISecResource *res=scopes->queryResource(seq);
  109. if (res && res->getAccessFlags()<required)
  110. return false;
  111. }
  112. }
  113. if (!ret && (log || excpt))
  114. wuAccessError(NULL, action, wuscope, NULL, excpt, log);
  115. return ret;
  116. }
  117. static bool checkWuSecAccess(IConstWorkUnit &cw, ISecManager *secmgr, ISecUser *secuser, int required, const char *action, bool excpt, bool log)
  118. {
  119. if (!secmgr || !secuser)
  120. return true;
  121. bool ret=secmgr->authorizeEx(RT_WORKUNIT_SCOPE, *secuser, cw.queryWuScope())>=required;
  122. if (!ret && (log || excpt))
  123. {
  124. wuAccessError(secuser->getName(), action, cw.queryWuScope(), cw.queryWuid(), excpt, log);
  125. }
  126. return ret;
  127. }
  128. static bool checkWuSecAccess(const char *wuid, ISecManager *secmgr, ISecUser *secuser, int required, const char *action, bool excpt, bool log)
  129. {
  130. if (!secmgr || !secuser)
  131. return true;
  132. Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
  133. Owned<IConstWorkUnit> cw = factory->openWorkUnit(wuid);
  134. bool ret=secmgr->authorizeEx(RT_WORKUNIT_SCOPE, *secuser, cw->queryWuScope())>=required;
  135. if (!ret && (log || excpt))
  136. {
  137. wuAccessError(secuser->getName(), action, cw->queryWuScope(), cw->queryWuid(), excpt, log);
  138. }
  139. return ret;
  140. }
  141. void doDescheduleWorkkunit(char const * wuid)
  142. {
  143. StringBuffer xpath;
  144. xpath.append("*/*/*/");
  145. ncnameEscape(wuid, xpath);
  146. Owned<IRemoteConnection> conn = querySDS().connect("/Schedule", myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
  147. if(!conn) return;
  148. Owned<IPropertyTree> root = conn->getRoot();
  149. bool more;
  150. do more = root->removeProp(xpath.str()); while(more);
  151. }
  152. //======================================================
  153. /*
  154. * Graph progress support
  155. */
  156. CWuGraphStats::CWuGraphStats(IPropertyTree *_progress, StatisticCreatorType _creatorType, const char * _creator, unsigned wfid, const char * _rootScope, unsigned _id)
  157. : progress(_progress), creatorType(_creatorType), creator(_creator), id(_id)
  158. {
  159. StatsScopeId graphScopeId;
  160. verifyex(graphScopeId.setScopeText(_rootScope));
  161. StatsScopeId rootScopeId(SSTworkflow,wfid);
  162. collector.setown(createStatisticsGatherer(_creatorType, _creator, rootScopeId));
  163. collector->beginScope(graphScopeId);
  164. }
  165. void CWuGraphStats::beforeDispose()
  166. {
  167. Owned<IStatisticCollection> stats = collector->getResult();
  168. MemoryBuffer compressed;
  169. {
  170. MemoryBuffer serialized;
  171. serializeStatisticCollection(serialized, stats);
  172. compressToBuffer(compressed, serialized.length(), serialized.toByteArray());
  173. }
  174. unsigned minActivity = 0;
  175. unsigned maxActivity = 0;
  176. stats->getMinMaxActivity(minActivity, maxActivity);
  177. StringBuffer tag;
  178. tag.append("sg").append(id);
  179. //Replace the particular subgraph statistics added by this creator
  180. StringBuffer qualified(tag);
  181. qualified.append("[@creator='").append(creator).append("']");
  182. progress->removeProp(qualified);
  183. IPropertyTree * subgraph = progress->addPropTree(tag);
  184. subgraph->setProp("@c", queryCreatorTypeName(creatorType));
  185. subgraph->setProp("@creator", creator);
  186. subgraph->setPropInt("@minActivity", minActivity);
  187. subgraph->setPropInt("@maxActivity", maxActivity);
  188. subgraph->setPropBin("Stats", compressed.length(), compressed.toByteArray());
  189. if (!progress->getPropBool("@stats", false))
  190. progress->setPropBool("@stats", true);
  191. }
  192. IStatisticGatherer & CWuGraphStats::queryStatsBuilder()
  193. {
  194. return *collector;
  195. }
  196. class CConstGraphProgress : public CInterface, implements IConstWUGraphProgress
  197. {
  198. public:
  199. IMPLEMENT_IINTERFACE;
  200. CConstGraphProgress(const char *_wuid, const char *_graphName, IPropertyTree *_progress) : wuid(_wuid), graphName(_graphName), progress(_progress)
  201. {
  202. if (!progress)
  203. progress.setown(createPTree());
  204. formatVersion = progress->getPropInt("@format", PROGRESS_FORMAT_V);
  205. }
  206. virtual IPropertyTree * getProgressTree()
  207. {
  208. if (progress->getPropBool("@stats"))
  209. return createProcessTreeFromStats(); // Should we cache that?
  210. return LINK(progress);
  211. }
  212. virtual unsigned queryFormatVersion()
  213. {
  214. return formatVersion;
  215. }
  216. protected:
  217. CConstGraphProgress(const char *_wuid, const char *_graphName) : wuid(_wuid), graphName(_graphName)
  218. {
  219. formatVersion = PROGRESS_FORMAT_V;
  220. }
  221. static void expandStats(IPropertyTree * target, IStatisticCollection & collection)
  222. {
  223. StringBuffer formattedValue;
  224. unsigned numStats = collection.getNumStatistics();
  225. for (unsigned i=0; i < numStats; i++)
  226. {
  227. StatisticKind kind;
  228. unsigned __int64 value;
  229. collection.getStatistic(kind, value, i);
  230. formatStatistic(formattedValue.clear(), value, kind);
  231. target->setProp(queryTreeTag(kind), formattedValue);
  232. }
  233. }
  234. void expandProcessTreeFromStats(IPropertyTree * rootTarget, IPropertyTree * target, IStatisticCollection * collection)
  235. {
  236. expandStats(target, *collection);
  237. StringBuffer scopeName;
  238. Owned<IStatisticCollectionIterator> activityIter = &collection->getScopes(NULL, false);
  239. ForEach(*activityIter)
  240. {
  241. IStatisticCollection & cur = activityIter->query();
  242. cur.getScope(scopeName.clear());
  243. const char * id = scopeName.str();
  244. const char * tag;
  245. IPropertyTree * curTarget = target;
  246. switch (cur.queryScopeType())
  247. {
  248. case SSTedge:
  249. tag = "edge";
  250. id += strlen(EdgeScopePrefix);
  251. break;
  252. case SSTactivity:
  253. tag = "node";
  254. id += strlen(ActivityScopePrefix);
  255. break;
  256. case SSTsubgraph:
  257. //All subgraphs are added a root elements in the progress tree
  258. curTarget = rootTarget;
  259. tag = "node";
  260. id += strlen(SubGraphScopePrefix);
  261. break;
  262. case SSTchildgraph:
  263. case SSTworkflow:
  264. case SSTgraph:
  265. // SSTworkflow and SSTgraph may be safely ignored. They are not required to produce the statistics.
  266. continue;
  267. case SSTfunction:
  268. //MORE:Should function scopes be included in the graph scope somehow, and if so how?
  269. continue;
  270. default:
  271. throwUnexpected();
  272. }
  273. IPropertyTree * next = curTarget->addPropTree(tag);
  274. next->setProp("@id", id);
  275. expandProcessTreeFromStats(rootTarget, next, &cur);
  276. }
  277. }
  278. IPropertyTree * createProcessTreeFromStats()
  279. {
  280. MemoryBuffer compressed;
  281. MemoryBuffer serialized;
  282. Owned<IPropertyTree> progressTree = createPTree();
  283. Owned<IPropertyTreeIterator> iter = progress->getElements("sg*");
  284. ForEach(*iter)
  285. {
  286. IPropertyTree & curSubGraph = iter->query();
  287. curSubGraph.getPropBin("Stats", compressed.clear());
  288. //Protect against updates that delete the stats while we are iterating
  289. if (compressed.length())
  290. {
  291. decompressToBuffer(serialized.clear(), compressed);
  292. Owned<IStatisticCollection> collection = createStatisticCollection(serialized);
  293. expandProcessTreeFromStats(progressTree, progressTree, collection);
  294. }
  295. }
  296. return progressTree.getClear();
  297. }
  298. protected:
  299. Linked<IPropertyTree> progress;
  300. StringAttr wuid, graphName;
  301. unsigned formatVersion;
  302. };
  303. extern WORKUNIT_API IConstWUGraphProgress *createConstGraphProgress(const char *_wuid, const char *_graphName, IPropertyTree *_progress)
  304. {
  305. return new CConstGraphProgress(_wuid, _graphName, _progress);
  306. }
  307. //--------------------------------------------------------------------------------------------------------------------
  308. /*
  309. * Create a user friendly description a scope/stats combination. Only currently used for elapsed time for root subgraphs
  310. */
  311. static void createDefaultDescription(StringBuffer & description, StatisticKind kind, StatisticScopeType scopeType, const char * scope)
  312. {
  313. switch (kind)
  314. {
  315. case StTimeElapsed:
  316. {
  317. if (scopeType != SSTsubgraph)
  318. break;
  319. //Create a default description for a root subgraph
  320. const char * colon = strchr(scope, ':');
  321. if (!colon)
  322. break;
  323. const char * subgraph = colon+1;
  324. //Check for nested subgraph
  325. if (strchr(subgraph, ':'))
  326. break;
  327. assertex(strncmp(subgraph, SubGraphScopePrefix, strlen(SubGraphScopePrefix)) == 0);
  328. StringAttr graphname;
  329. graphname.set(scope, colon - scope);
  330. unsigned subId = atoi(subgraph + strlen(SubGraphScopePrefix));
  331. formatGraphTimerLabel(description, graphname, 0, subId);
  332. return;
  333. }
  334. }
  335. describeScope(description, scope);
  336. }
  337. /* Represents a single statistic */
  338. class ExtractedStatistic : public CInterfaceOf<IConstWUStatistic>
  339. {
  340. public:
  341. virtual IStringVal & getDescription(IStringVal & str, bool createDefault) const
  342. {
  343. if (!description && createDefault)
  344. {
  345. StringBuffer desc;
  346. createDefaultDescription(desc, kind, scopeType, scope);
  347. str.set(desc);
  348. return str;
  349. }
  350. str.set(description);
  351. return str;
  352. }
  353. virtual IStringVal & getCreator(IStringVal & str) const
  354. {
  355. str.set(creator);
  356. return str;
  357. }
  358. virtual const char * queryScope() const
  359. {
  360. return scope ? scope : "";
  361. }
  362. virtual IStringVal & getFormattedValue(IStringVal & str) const
  363. {
  364. StringBuffer formatted;
  365. formatStatistic(formatted, value, measure);
  366. str.set(formatted);
  367. return str;
  368. }
  369. virtual StatisticMeasure getMeasure() const
  370. {
  371. return measure;
  372. }
  373. virtual StatisticKind getKind() const
  374. {
  375. return kind;
  376. }
  377. virtual StatisticCreatorType getCreatorType() const
  378. {
  379. return creatorType;
  380. }
  381. virtual StatisticScopeType getScopeType() const
  382. {
  383. return scopeType;
  384. }
  385. virtual unsigned __int64 getValue() const
  386. {
  387. return value;
  388. }
  389. virtual unsigned __int64 getCount() const
  390. {
  391. return count;
  392. }
  393. virtual unsigned __int64 getMax() const
  394. {
  395. return max;
  396. }
  397. virtual unsigned __int64 getTimestamp() const
  398. {
  399. return timeStamp;
  400. }
  401. public:
  402. StringBuffer creator;
  403. StringBuffer description;
  404. StringBuffer scope;
  405. StatisticMeasure measure;
  406. StatisticKind kind;
  407. StatisticCreatorType creatorType;
  408. StatisticScopeType scopeType;
  409. unsigned __int64 value;
  410. unsigned __int64 count;
  411. unsigned __int64 max;
  412. unsigned __int64 timeStamp;
  413. };
  414. //---------------------------------------------------------------------------------------------------------------------
  415. /*
  416. * The following compare functions are used to ensure that comparison are consistent with
  417. * compareScopeName() in jstats. This ensures that the scope iterators from different sources
  418. * are processed in a consistent order
  419. */
  420. static int compareGraphNode(IInterface * const *ll, IInterface * const *rr)
  421. {
  422. IPropertyTree *l = (IPropertyTree *) *ll;
  423. IPropertyTree *r = (IPropertyTree *) *rr;
  424. unsigned lwfid = l->getPropInt("@wfid");
  425. unsigned rwfid = r->getPropInt("@wfid");
  426. if (lwfid != rwfid)
  427. return lwfid > rwfid ? +1 : -1;
  428. const char * lname = l->queryName();
  429. const char * rname = r->queryName();
  430. return compareScopeName(lname, rname);
  431. }
  432. static int compareSubGraphStatsNode(IInterface * const *ll, IInterface * const *rr)
  433. {
  434. IPropertyTree *l = (IPropertyTree *) *ll;
  435. IPropertyTree *r = (IPropertyTree *) *rr;
  436. return compareScopeName(l->queryName(), r->queryName());
  437. }
  438. static int compareSubGraphNode(IInterface * const *ll, IInterface * const *rr)
  439. {
  440. IPropertyTree *l = (IPropertyTree *) *ll;
  441. IPropertyTree *r = (IPropertyTree *) *rr;
  442. return l->getPropInt("@id") - r->getPropInt("@id");
  443. }
  444. static int compareActivityNode(IInterface * const *ll, IInterface * const *rr)
  445. {
  446. IPropertyTree *l = (IPropertyTree *) *ll;
  447. IPropertyTree *r = (IPropertyTree *) *rr;
  448. return l->getPropInt("@id") - r->getPropInt("@id");
  449. }
  450. static int compareEdgeNode(IInterface * const *ll, IInterface * const *rr)
  451. {
  452. IPropertyTree *l = (IPropertyTree *) *ll;
  453. IPropertyTree *r = (IPropertyTree *) *rr;
  454. //MORE: Edge needs more work
  455. const char * leftId = l->queryProp("@id");
  456. const char * rightId = r->queryProp("@id");
  457. unsigned leftAc = atoi(leftId);
  458. unsigned rightAc = atoi(rightId);
  459. if (leftAc != rightAc)
  460. return (int)(leftAc - rightAc);
  461. const char * leftSep = strchr(leftId, '_');
  462. const char * rightSep = strchr(rightId, '_');
  463. assertex(leftSep && rightSep);
  464. return atoi(leftSep+1) - atoi(rightSep+1);
  465. }
  466. //---------------------------------------------------------------------------------------------------------------------
  467. /*
  468. * A class for implementing a scope iterator that walks through graph progress information
  469. */
  470. class CConstGraphProgressScopeIterator : public CInterfaceOf<IConstWUScopeIterator>
  471. {
  472. public:
  473. CConstGraphProgressScopeIterator(const char * wuid, const ScopeFilter & _filter, __uint64 _minVersion) : filter(_filter), minVersion(_minVersion)
  474. {
  475. //Examine the filter, and determine if we only need to look at a single graph/subgraph
  476. StringAttr singleGraph;
  477. const StringArray & scopesToMatch = filter.queryScopes();
  478. if (scopesToMatch)
  479. {
  480. bool seenGraph = false;
  481. bool seenSubGraph = false;
  482. ForEachItemIn(iScope, scopesToMatch)
  483. {
  484. StringArray ids;
  485. ids.appendList(scopesToMatch.item(iScope), ":");
  486. ForEachItemIn(i, ids)
  487. {
  488. const char * curId = ids.item(i);
  489. StatsScopeId id(curId);
  490. switch (id.queryScopeType())
  491. {
  492. case SSTgraph:
  493. if (seenGraph)
  494. {
  495. if (singleGraph && !streq(singleGraph, curId))
  496. singleGraph.clear();
  497. }
  498. else
  499. {
  500. if (!id.isWildcard())
  501. singleGraph.set(curId);
  502. seenGraph = true;
  503. }
  504. break;
  505. case SSTsubgraph:
  506. if (seenSubGraph)
  507. {
  508. if (singleSubGraph && !streq(singleSubGraph, curId))
  509. singleSubGraph.clear();
  510. }
  511. else
  512. {
  513. if (!id.isWildcard())
  514. singleSubGraph.set(curId);
  515. seenSubGraph = true;
  516. }
  517. break;
  518. }
  519. }
  520. }
  521. }
  522. rootPath.append("/GraphProgress/").append(wuid).append('/');
  523. if (singleGraph)
  524. rootPath.append(singleGraph).append("/");
  525. //Don't lock the statistics while we iterate - any partial updates must not cause problems
  526. if (daliClientActive())
  527. conn.setown(querySDS().connect(rootPath.str(), myProcessSession(), RTM_NONE, SDS_LOCK_TIMEOUT));
  528. if (conn && !singleGraph)
  529. {
  530. graphIter.setown(conn->queryRoot()->getElements("*"));
  531. graphIter.setown(createSortedIterator(*graphIter, compareGraphNode));
  532. }
  533. valid = false;
  534. }
  535. IMPLEMENT_IINTERFACE_USING(CInterfaceOf<IConstWUScopeIterator>)
  536. virtual bool first()
  537. {
  538. valid = false;
  539. if (!conn)
  540. return false;
  541. if (graphIter && !graphIter->first())
  542. return false;
  543. if (!firstSubGraph())
  544. {
  545. if (!nextGraph())
  546. return false;
  547. }
  548. valid = true;
  549. return true;
  550. }
  551. virtual bool next()
  552. {
  553. if (!nextChildScope())
  554. {
  555. if (!nextSubGraph())
  556. {
  557. if (!nextGraph())
  558. {
  559. valid = false;
  560. return false;
  561. }
  562. }
  563. }
  564. return true;
  565. }
  566. virtual bool nextSibling() override
  567. {
  568. if (collections.ordinality() == 0)
  569. return false;
  570. assertex(childIterators.ordinality() < collections.ordinality());
  571. collections.pop();
  572. //next will call childIterator.next() - walking the next sibling
  573. return next();
  574. }
  575. virtual bool nextParent() override
  576. {
  577. if (collections.ordinality() == 0)
  578. return false;
  579. assertex(childIterators.ordinality() < collections.ordinality());
  580. collections.pop();
  581. if (collections.ordinality() == 0)
  582. return false;
  583. //Finish with this node - so next will move onto the sibling of the parent node.
  584. finishCollection();
  585. return next();
  586. }
  587. virtual bool isValid()
  588. {
  589. return valid;
  590. }
  591. protected:
  592. bool firstSubGraph()
  593. {
  594. IPropertyTree & graphNode = graphIter ? graphIter->query() : *conn->queryRoot();
  595. const char * xpath = "sg*";
  596. StringBuffer childXpath;
  597. if (singleSubGraph)
  598. {
  599. childXpath.append(singleSubGraph);
  600. xpath = childXpath.str();
  601. }
  602. subgraphIter.setown(graphNode.getElements(xpath));
  603. if (subgraphIter)
  604. {
  605. if (!singleSubGraph)
  606. subgraphIter.setown(createSortedIterator(*subgraphIter, compareSubGraphStatsNode));
  607. }
  608. else
  609. subgraphIter.setown(graphNode.getElements("sg0"));
  610. if (!subgraphIter->first())
  611. return false;
  612. if (firstStat())
  613. return true;
  614. return nextSubGraph();
  615. }
  616. bool nextSubGraph()
  617. {
  618. for (;;)
  619. {
  620. if (!subgraphIter->next())
  621. return false;
  622. if (firstStat())
  623. return true;
  624. }
  625. }
  626. bool nextGraph()
  627. {
  628. if (!graphIter)
  629. return false;
  630. for (;;)
  631. {
  632. if (!graphIter->next())
  633. return false;
  634. if (firstSubGraph())
  635. return true;
  636. }
  637. }
  638. bool firstStat()
  639. {
  640. IPropertyTree & curSubGraph = subgraphIter->query();
  641. if (!checkSubGraph())
  642. return false;
  643. curSubGraph.getPropBin("Stats", compressed.clear());
  644. //Don't crash on old format progress...
  645. if (compressed.length() == 0)
  646. return false;
  647. decompressToBuffer(serialized.clear(), compressed);
  648. Owned<IStatisticCollection> collection = createStatisticCollection(serialized);
  649. statsIterator.timeStamp = collection->queryWhenCreated();
  650. if (!beginCollection(*collection))
  651. return false;
  652. // When workflow is root element, it is just a container. Ignore the workflow element here
  653. // as WorkUnitStatisticsScopeIterator will produce workflow scope - don't want duplicates.
  654. // (Note: workflow element never contains stats).
  655. if (collections.tos().queryScopeType() == SSTworkflow)
  656. {
  657. if (!next())
  658. return false;
  659. }
  660. //The root element of a collection is a graph - but it is only there to nest the subgraphs in.
  661. //Do not iterate it as a separate element - unless it has some stats.
  662. IStatisticCollection & curCollection = collections.tos();
  663. if ((curCollection.queryScopeType() != SSTgraph) || (curCollection.getNumStatistics() != 0))
  664. return true;
  665. return next();
  666. }
  667. bool beginCollection(IStatisticCollection & collection)
  668. {
  669. collections.append(OLINK(collection));
  670. curScopeType = collection.queryScopeType();
  671. collection.getFullScope(curScopeName.clear());
  672. ScopeCompare result = filter.compare(curScopeName);
  673. if (result & SCequal)
  674. return true;
  675. //If this scope cannot be the parent of a match then discard it.
  676. if (!(result & SCparent))
  677. collections.pop();
  678. //walk the next element
  679. return nextChildScope();
  680. }
  681. bool nextChildScope()
  682. {
  683. for (;;)
  684. {
  685. if (collections.ordinality() == 0)
  686. return false;
  687. IStatisticCollection * curCollection = &collections.tos();
  688. if (childIterators.ordinality() < collections.ordinality())
  689. {
  690. ScopeCompare result = filter.compare(curScopeName);
  691. //Do not walk children scopes if it is unrelated
  692. if (result & SCparent)
  693. {
  694. //Start iterating the children for the current collection
  695. childIterators.append(curCollection->getScopes(NULL, true));
  696. if (!childIterators.tos().first())
  697. {
  698. finishCollection();
  699. continue;
  700. }
  701. }
  702. else
  703. {
  704. //Don't walk the child scopes
  705. collections.pop();
  706. continue;
  707. }
  708. }
  709. else if (!childIterators.tos().next())
  710. {
  711. finishCollection();
  712. continue;
  713. }
  714. if (beginCollection(childIterators.tos().query()))
  715. return true;
  716. }
  717. }
  718. void finishCollection()
  719. {
  720. collections.pop();
  721. childIterators.pop();
  722. }
  723. bool checkSubGraph()
  724. {
  725. IPropertyTree & curSubGraph = subgraphIter->query();
  726. StatisticCreatorType creatorType = queryCreatorType(curSubGraph.queryProp("@c"), SCTnone);
  727. const char * creator = curSubGraph.queryProp("@creator");
  728. //MORE: Check minVersion and allow early filtering
  729. //MORE: Potentially filter by creator type??
  730. // if (!filter->matches(creatorType, creator, SSTall, NULL, SMeasureAll, StKindAll, AnyStatisticValue))
  731. // return false;
  732. statsIterator.creatorType = creatorType;
  733. statsIterator.creator.set(creator);
  734. return true;
  735. }
  736. virtual void playProperties(IWuScopeVisitor & visitor, WuPropertyTypes whichProperties) override
  737. {
  738. if ((whichProperties & PTstatistics))
  739. {
  740. /*
  741. MORE: Code from the statsIterator class should be inlined here - code will be much simpler
  742. but it currently needs an implementation of the IConstWUStatistic 3rd parameter
  743. IStatisticCollection & collection = collections.tos();
  744. ForEachItemIn(i, collection)
  745. {
  746. StatisticKind kind;
  747. unsigned __int64 value;
  748. collection->getStatistic(kind, value, i);
  749. visitor.noteStatistic(kind, value, *this);
  750. }
  751. */
  752. statsIterator.reset(curScopeName, curScopeType, collections.tos());
  753. ForEach(statsIterator)
  754. statsIterator.play(visitor);
  755. }
  756. }
  757. virtual bool getStat(StatisticKind kind, unsigned __int64 & value) const override
  758. {
  759. return collections.tos().getStatistic(kind, value);
  760. }
  761. virtual const char * queryAttribute(WuAttr attr, StringBuffer & scratchpad) const override
  762. {
  763. return nullptr;
  764. }
  765. virtual const char * queryHint(const char * kind) const override
  766. {
  767. return nullptr;
  768. }
  769. virtual const char * queryScope() const override
  770. {
  771. return curScopeName;
  772. }
  773. virtual StatisticScopeType getScopeType() const override
  774. {
  775. return curScopeType;
  776. }
  777. private:
  778. class ScopeStatisticsIterator : public CInterfaceOf<IConstWUStatistic>
  779. {
  780. friend class CConstGraphProgressScopeIterator; // cleaner if this was removed + setScope() functions added.
  781. public:
  782. void reset(const char * _scopeName, StatisticScopeType _scopeType, IStatisticCollection & _collection)
  783. {
  784. scope.set(_scopeName);
  785. scopeType = _scopeType;
  786. collection = &_collection;
  787. numStats = collection->getNumStatistics();
  788. }
  789. // interface IConstWUStatisticIterator
  790. IConstWUStatistic & query()
  791. {
  792. return *this;
  793. }
  794. bool first()
  795. {
  796. curStatIndex = 0;
  797. if (curStatIndex >= numStats)
  798. return false;
  799. collection->getStatistic(kind, value, curStatIndex);
  800. //MORE: Allow filtering:
  801. /*
  802. * if (filter && !filter->matches(SCTall, NULL, SSTall, NULL, queryMeasure(kind), kind, value))
  803. continue;
  804. */
  805. return true;
  806. }
  807. bool next()
  808. {
  809. for (;;)
  810. {
  811. ++curStatIndex;
  812. if (curStatIndex >= numStats)
  813. return false;
  814. collection->getStatistic(kind, value, curStatIndex);
  815. //MORE: Allow stats filtering
  816. return true;
  817. }
  818. }
  819. bool isValid()
  820. {
  821. return curStatIndex < numStats;
  822. }
  823. //interface IConstWUStatistic
  824. virtual IStringVal & getDescription(IStringVal & str, bool createDefault) const override
  825. {
  826. if (createDefault)
  827. {
  828. StringBuffer description;
  829. createDefaultDescription(description, kind, scopeType, scope);
  830. str.set(description);
  831. }
  832. return str;
  833. }
  834. virtual IStringVal & getCreator(IStringVal & str) const override
  835. {
  836. str.set(creator);
  837. return str;
  838. }
  839. virtual const char * queryScope() const override
  840. {
  841. return scope;
  842. }
  843. virtual IStringVal & getFormattedValue(IStringVal & str) const override
  844. {
  845. StringBuffer formatted;
  846. formatStatistic(formatted, value, kind);
  847. str.set(formatted);
  848. return str;
  849. }
  850. virtual StatisticMeasure getMeasure() const override
  851. {
  852. return queryMeasure(kind);
  853. }
  854. virtual StatisticKind getKind() const override
  855. {
  856. return kind;
  857. }
  858. virtual StatisticCreatorType getCreatorType() const override
  859. {
  860. return creatorType;
  861. }
  862. virtual StatisticScopeType getScopeType() const override
  863. {
  864. return scopeType;
  865. }
  866. virtual unsigned __int64 getValue() const override
  867. {
  868. return value;
  869. }
  870. virtual unsigned __int64 getCount() const override
  871. {
  872. return 1;
  873. }
  874. virtual unsigned __int64 getMax() const override
  875. {
  876. return 0;
  877. }
  878. virtual unsigned __int64 getTimestamp() const override
  879. {
  880. return timeStamp;
  881. }
  882. void play(IWuScopeVisitor & visitor)
  883. {
  884. visitor.noteStatistic(kind, value, *this);
  885. }
  886. protected:
  887. IStatisticCollection * collection = nullptr;
  888. StringBuffer creator;
  889. StringBuffer scope;
  890. StatisticKind kind;
  891. StatisticCreatorType creatorType;
  892. StatisticScopeType scopeType;
  893. unsigned __int64 value = 0;
  894. unsigned __int64 timeStamp;
  895. unsigned curStatIndex = 0;
  896. unsigned numStats = 0;
  897. } statsIterator;
  898. Owned<IRemoteConnection> conn;
  899. StringBuffer curScopeName;
  900. StatisticScopeType curScopeType = SSTnone;
  901. __uint64 minVersion;
  902. const ScopeFilter & filter;
  903. StringBuffer rootPath;
  904. StringAttr singleSubGraph;
  905. Owned<IPropertyTreeIterator> graphIter;
  906. Owned<IPropertyTreeIterator> subgraphIter;
  907. IArrayOf<IStatisticCollection> collections;
  908. IArrayOf<IStatisticCollectionIterator> childIterators; // Iterator(n) through collections(n) - created once iterating children
  909. MemoryBuffer compressed;
  910. MemoryBuffer serialized;
  911. bool valid;
  912. };
  913. int compareNoteScopeOrder(IConstWUException & left, IConstWUException & right)
  914. {
  915. return compareScopeName(left.queryScope(), right.queryScope());
  916. }
  917. int compareNoteScopeOrder(IInterface * const * left, IInterface * const * right)
  918. {
  919. return compareNoteScopeOrder(*static_cast<IConstWUException *>(*left), *static_cast<IConstWUException *>(*right));
  920. }
  921. class NotesIterator : public CInterfaceOf<IConstWUScopeIterator>
  922. {
  923. public:
  924. NotesIterator(const IConstWorkUnit * wu, const ScopeFilter & _filter)
  925. {
  926. Owned<IConstWUExceptionIterator> exceptions = &wu->getExceptions();
  927. ForEach(*exceptions)
  928. {
  929. IConstWUException & exception = exceptions->query();
  930. if (exception.queryScope()!=nullptr)
  931. notes.append(OLINK(exception));
  932. }
  933. notes.sort(compareNoteScopeOrder);
  934. }
  935. virtual bool first() override
  936. {
  937. baseIndex = 0;
  938. return updateCurrent();
  939. }
  940. virtual bool next() override
  941. {
  942. baseIndex += numCurrentScope;
  943. return updateCurrent();
  944. }
  945. virtual bool isValid() override
  946. {
  947. return notes.isItem(baseIndex);
  948. }
  949. virtual bool nextSibling() override
  950. {
  951. //Search until the current scope is not a child of the previous scope
  952. StringBuffer savedScope(queryScope());
  953. for (;;)
  954. {
  955. if (!next())
  956. return false;
  957. if (compareScopes(queryScope(), savedScope) != SCchild)
  958. return true;
  959. }
  960. }
  961. virtual bool nextParent() override
  962. {
  963. //Search until the current scope is not a child of the previous parent scope
  964. StringBuffer parentScope;
  965. if (getParentScope(parentScope, queryScope()))
  966. {
  967. for (;;)
  968. {
  969. if (!next())
  970. return false;
  971. if (compareScopes(queryScope(), parentScope) != SCchild)
  972. return true;
  973. }
  974. }
  975. else
  976. {
  977. finish();
  978. return false;
  979. }
  980. }
  981. virtual const char * queryScope() const override
  982. {
  983. const char * scope = notes.item(baseIndex).queryScope();
  984. return scope ? scope : "";
  985. }
  986. virtual StatisticScopeType getScopeType() const override
  987. {
  988. const char * tail = queryScopeTail(queryScope());
  989. StatsScopeId id(tail);
  990. return id.queryScopeType();
  991. }
  992. virtual void playProperties(IWuScopeVisitor & visitor, WuPropertyTypes whichProperties = PTall) override
  993. {
  994. if (whichProperties & PTnotes)
  995. {
  996. for (unsigned i=0; i < numCurrentScope; i++)
  997. {
  998. IConstWUException & cur = notes.item(baseIndex + i);
  999. visitor.noteException(cur);
  1000. }
  1001. }
  1002. }
  1003. virtual bool getStat(StatisticKind kind, unsigned __int64 & value) const override
  1004. {
  1005. return false;
  1006. }
  1007. virtual const char * queryAttribute(WuAttr attr, StringBuffer & scratchpad) const override
  1008. {
  1009. return nullptr;
  1010. }
  1011. virtual const char * queryHint(const char * kind) const override
  1012. {
  1013. return nullptr;
  1014. }
  1015. private:
  1016. bool updateCurrent()
  1017. {
  1018. if (!notes.isItem(baseIndex))
  1019. return false;
  1020. // set numCurrentScope to number of notes in the current scope
  1021. unsigned next = baseIndex+1;
  1022. while (next < notes.ordinality())
  1023. {
  1024. if (compareNoteScopeOrder(notes.item(baseIndex), notes.item(next)) != 0)
  1025. break;
  1026. next++;
  1027. }
  1028. numCurrentScope = (next - baseIndex);
  1029. return true;
  1030. }
  1031. void finish()
  1032. {
  1033. baseIndex = notes.ordinality();
  1034. }
  1035. unsigned baseIndex = 0;
  1036. unsigned numCurrentScope = 0;
  1037. IArrayOf<IConstWUException> notes;
  1038. };
  1039. int compareStatisticScopes(IConstWUStatistic & left, IConstWUStatistic & right)
  1040. {
  1041. return compareScopeName(left.queryScope(), right.queryScope());
  1042. }
  1043. int compareStatisticScopes(IInterface * const * left, IInterface * const * right)
  1044. {
  1045. return compareStatisticScopes(*static_cast<IConstWUStatistic *>(*left), *static_cast<IConstWUStatistic *>(*right));
  1046. }
  1047. /*
  1048. * An implementation of IConstWUScopeIterator for global workunit statistics.
  1049. */
  1050. class WorkUnitStatisticsScopeIterator : public CInterfaceOf<IConstWUScopeIterator>
  1051. {
  1052. public:
  1053. WorkUnitStatisticsScopeIterator(const IArrayOf<IConstWUStatistic> & _statistics, const ScopeFilter & _filter)
  1054. {
  1055. ForEachItemIn(i, _statistics)
  1056. {
  1057. IConstWUStatistic & cur = _statistics.item(i);
  1058. if (_filter.compare(cur.queryScope()) & SCequal)
  1059. statistics.append(OLINK(cur));
  1060. }
  1061. statistics.sort(compareStatisticScopes);
  1062. }
  1063. virtual bool first() override
  1064. {
  1065. curIndex = 0;
  1066. return initScope();
  1067. }
  1068. virtual bool next() override
  1069. {
  1070. curIndex += numStatistics;
  1071. return initScope();
  1072. }
  1073. virtual bool nextSibling() override
  1074. {
  1075. //Search until the current scope is not a child of the previous scope
  1076. StringBuffer savedScope(queryScope());
  1077. for (;;)
  1078. {
  1079. if (!next())
  1080. return false;
  1081. if (compareScopes(queryScope(), savedScope) != SCchild)
  1082. return true;
  1083. }
  1084. }
  1085. virtual bool nextParent() override
  1086. {
  1087. //Search until the current scope is not a child of the previous parent scope
  1088. StringBuffer parentScope;
  1089. if (getParentScope(parentScope, queryScope()))
  1090. {
  1091. for (;;)
  1092. {
  1093. if (!next())
  1094. return false;
  1095. if (compareScopes(queryScope(), parentScope) != SCchild)
  1096. return true;
  1097. }
  1098. }
  1099. else
  1100. {
  1101. finish();
  1102. return false;
  1103. }
  1104. }
  1105. virtual bool isValid() override
  1106. {
  1107. return statistics.isItem(curIndex);
  1108. }
  1109. virtual const char * queryScope() const override
  1110. {
  1111. return statistics.item(curIndex).queryScope();
  1112. }
  1113. virtual StatisticScopeType getScopeType() const override
  1114. {
  1115. return statistics.item(curIndex).getScopeType();
  1116. }
  1117. virtual void playProperties(IWuScopeVisitor & visitor, WuPropertyTypes whichProperties) override
  1118. {
  1119. if (whichProperties & PTstatistics)
  1120. {
  1121. for (unsigned i=0; i < numStatistics; i++)
  1122. {
  1123. IConstWUStatistic & cur = statistics.item(curIndex + i);
  1124. visitor.noteStatistic(cur.getKind(), cur.getValue(), cur);
  1125. }
  1126. }
  1127. }
  1128. virtual bool getStat(StatisticKind kind, unsigned __int64 & value) const override
  1129. {
  1130. for (unsigned i=0; i < numStatistics; i++)
  1131. {
  1132. IConstWUStatistic & cur = statistics.item(curIndex + i);
  1133. if (cur.getKind() == kind)
  1134. {
  1135. value = cur.getValue();
  1136. return true;
  1137. }
  1138. }
  1139. return false;
  1140. }
  1141. virtual const char * queryAttribute(WuAttr attr, StringBuffer & scratchpad) const override
  1142. {
  1143. return nullptr;
  1144. }
  1145. virtual const char * queryHint(const char * kind) const
  1146. {
  1147. return nullptr;
  1148. }
  1149. protected:
  1150. inline IConstWUStatistic & queryStatistic(unsigned i)
  1151. {
  1152. return statistics.item(curIndex + i);
  1153. }
  1154. bool initScope()
  1155. {
  1156. if (!statistics.isItem(curIndex))
  1157. return false;
  1158. unsigned next = curIndex+1;
  1159. while (next < statistics.ordinality())
  1160. {
  1161. if (compareStatisticScopes(statistics.item(curIndex), statistics.item(next)) != 0)
  1162. break;
  1163. next++;
  1164. }
  1165. numStatistics = (next - curIndex);
  1166. return true;
  1167. }
  1168. void finish()
  1169. {
  1170. curIndex = statistics.ordinality();
  1171. }
  1172. protected:
  1173. IArrayOf<IConstWUStatistic> statistics;
  1174. unsigned curIndex = 0;
  1175. unsigned numStatistics = 1;
  1176. };
  1177. /*
  1178. * An implementation of IConstWUScopeIterator for the query graphs.
  1179. */
  1180. class GraphScopeIterator : public CInterfaceOf<IConstWUScopeIterator>
  1181. {
  1182. private:
  1183. //This uses a state machine - this enumeration contains the different states.
  1184. enum State
  1185. {
  1186. //Following are the states which represent valid scopes
  1187. SGraph,
  1188. SChildGraph,
  1189. SSubGraph,
  1190. SEdge,
  1191. SActivity,
  1192. //The following are internal states.
  1193. SGraphFirstEdge,
  1194. SGraphFirstSubGraph,
  1195. SGraphFirst,
  1196. SGraphEnd,
  1197. SGraphNext,
  1198. SSubGraphFirstEdge,
  1199. SSubGraphFirstActivity,
  1200. SSubGraphEnd,
  1201. SSubGraphNext,
  1202. SEdgeNext,
  1203. SEdgeEnd,
  1204. SActivityNext,
  1205. SActivityEnd,
  1206. SChildGraphFirstEdge,
  1207. SChildGraphFirstSubGraph,
  1208. SChildGraphFirst,
  1209. SChildGraphNext,
  1210. SChildGraphEnd,
  1211. SDone
  1212. };
  1213. State state = SDone;
  1214. State nextState = SDone;
  1215. public:
  1216. GraphScopeIterator(const IConstWorkUnit * wu, const ScopeFilter & _filter) : graphIter(&wu->getGraphs(GraphTypeAny)), filter(_filter)
  1217. {
  1218. }
  1219. virtual bool first() override
  1220. {
  1221. state = SGraphFirst;
  1222. return nextScope();
  1223. }
  1224. virtual bool next() override
  1225. {
  1226. if (!selectNext())
  1227. return false;
  1228. return nextScope();
  1229. }
  1230. virtual bool nextSibling() override
  1231. {
  1232. if (!selectNextSibling())
  1233. return false;
  1234. return nextScope();
  1235. }
  1236. virtual bool nextParent() override
  1237. {
  1238. if (!selectNextParent())
  1239. return false;
  1240. return nextScope();
  1241. }
  1242. virtual bool isValid() override
  1243. {
  1244. return graphIter->isValid();
  1245. }
  1246. virtual const char * queryScope() const override
  1247. {
  1248. return curScopeName.str();
  1249. }
  1250. virtual StatisticScopeType getScopeType() const override
  1251. {
  1252. return scopeType;
  1253. }
  1254. virtual void playProperties(IWuScopeVisitor & visitor, WuPropertyTypes whichProperties) override
  1255. {
  1256. switch (scopeType)
  1257. {
  1258. case SSTgraph:
  1259. return;
  1260. }
  1261. IPropertyTree & cur = treeIters.tos().query();
  1262. switch (scopeType)
  1263. {
  1264. case SSTgraph:
  1265. break;
  1266. case SSTsubgraph:
  1267. break;
  1268. case SSTactivity:
  1269. {
  1270. if (whichProperties & PTattributes)
  1271. {
  1272. playAttribute(visitor, WaLabel);
  1273. Owned<IPropertyTreeIterator> attrs = cur.getElements("att");
  1274. ForEach(*attrs)
  1275. {
  1276. IPropertyTree & cur = attrs->query();
  1277. WuAttr attr = queryGraphChildAttToWuAttr(cur.queryProp("@name"));
  1278. if (attr != WaNone)
  1279. visitor.noteAttribute(attr, cur.queryProp("@value"));
  1280. }
  1281. }
  1282. if (whichProperties & PThints)
  1283. {
  1284. Owned<IPropertyTreeIterator> hints = cur.getElements("hint");
  1285. ForEach(*hints)
  1286. {
  1287. IPropertyTree & cur = hints->query();
  1288. visitor.noteHint(cur.queryProp("@name"), cur.queryProp("@value"));
  1289. }
  1290. }
  1291. break;
  1292. }
  1293. case SSTedge:
  1294. if (whichProperties & PTattributes)
  1295. {
  1296. //MORE This will eventually need to walk the attributes and map the names.
  1297. //Need to be careful if they need to be mapped differently depending on the context.
  1298. playAttribute(visitor, WaLabel);
  1299. playAttribute(visitor, WaIdSource);
  1300. playAttribute(visitor, WaIdTarget);
  1301. playAttribute(visitor, WaSourceIndex);
  1302. playAttribute(visitor, WaTargetIndex);
  1303. playAttribute(visitor, WaIsDependency);
  1304. }
  1305. break;
  1306. }
  1307. }
  1308. virtual bool getStat(StatisticKind kind, unsigned __int64 & value) const override
  1309. {
  1310. return false;
  1311. }
  1312. virtual const char * queryAttribute(WuAttr attr, StringBuffer & scratchpad) const override
  1313. {
  1314. if (!treeIters.ordinality())
  1315. return nullptr;
  1316. //MORE - check that the attribute is value for the current scope type (to prevent defaults being returned)
  1317. return queryAttributeValue(treeIters.tos().query(), attr, scratchpad);
  1318. }
  1319. virtual const char * queryHint(const char * kind) const override
  1320. {
  1321. //MORE: Needs to be implemented!
  1322. return nullptr;
  1323. }
  1324. private:
  1325. void playAttribute(IWuScopeVisitor & visitor, WuAttr kind)
  1326. {
  1327. StringBuffer scratchpad;
  1328. const char * value = queryAttributeValue(treeIters.tos().query(), kind, scratchpad);
  1329. if (value)
  1330. visitor.noteAttribute(kind, value);
  1331. }
  1332. void pushIterator(IPropertyTreeIterator * iter, State state)
  1333. {
  1334. treeIters.append(*LINK(iter));
  1335. stateStack.append(state);
  1336. }
  1337. State popIterator()
  1338. {
  1339. treeIters.pop();
  1340. return (State)stateStack.popGet();
  1341. }
  1342. void pushScope(const char * id)
  1343. {
  1344. scopeLengths.append(curScopeName.length());
  1345. curScopeName.append(":").append(id);
  1346. }
  1347. void popScope()
  1348. {
  1349. curScopeName.setLength(scopeLengths.popGet());
  1350. }
  1351. bool doNextScope()
  1352. {
  1353. for(;;)
  1354. {
  1355. switch (state)
  1356. {
  1357. case SGraph:
  1358. {
  1359. IConstWUGraph & graph = graphIter->query();
  1360. unsigned wfid = graph.getWfid();
  1361. curScopeName.clear();
  1362. if (wfid != 0)
  1363. curScopeName.append(WorkflowScopePrefix).append(wfid).append(':');
  1364. graph.getName(StringBufferAdaptor(curScopeName));
  1365. scopeType = SSTgraph;
  1366. return true;
  1367. }
  1368. case SSubGraph:
  1369. scopeId.set(SubGraphScopePrefix).append(treeIters.tos().query().getPropInt("@id"));
  1370. pushScope(scopeId);
  1371. scopeType = SSTsubgraph;
  1372. return true;
  1373. case SEdge:
  1374. scopeId.set(EdgeScopePrefix).append(treeIters.tos().query().queryProp("@id"));
  1375. pushScope(scopeId);
  1376. scopeType = SSTedge;
  1377. return true;
  1378. case SActivity:
  1379. if (treeIters.tos().query().getPropInt("att[@name='_kind']/@value") == TAKsubgraph)
  1380. {
  1381. state = SActivityNext;
  1382. break;
  1383. }
  1384. scopeId.set(ActivityScopePrefix).append(treeIters.tos().query().getPropInt("@id"));
  1385. pushScope(scopeId);
  1386. scopeType = SSTactivity;
  1387. return true;
  1388. case SChildGraph:
  1389. {
  1390. unsigned numIters = treeIters.ordinality();
  1391. //This should really be implemented by a filter on the node - but it would require _kind/_parentActivity to move to the node tag
  1392. if (treeIters.tos().query().getPropInt("att[@name='_kind']/@value") != TAKsubgraph)
  1393. {
  1394. state = SChildGraphNext;
  1395. break;
  1396. }
  1397. unsigned parentActivityId = treeIters.item(numIters-2).query().getPropInt("@id");
  1398. unsigned parentId = treeIters.tos().query().getPropInt("att[@name='_parentActivity']/@value");
  1399. if (parentId != parentActivityId)
  1400. {
  1401. state = SChildGraphNext;
  1402. break;
  1403. }
  1404. scopeId.set(ChildGraphScopePrefix).append(treeIters.tos().query().getPropInt("@id"));
  1405. pushScope(scopeId);
  1406. scopeType = SSTchildgraph;
  1407. return true;
  1408. }
  1409. //Graph iteration
  1410. case SGraphFirst:
  1411. if (!graphIter->first())
  1412. state = SDone;
  1413. else
  1414. state = SGraph;
  1415. break;
  1416. case SGraphEnd:
  1417. state = SGraphNext;
  1418. break;
  1419. case SGraphNext:
  1420. if (!graphIter->next())
  1421. state = SDone;
  1422. else
  1423. state = SGraph;
  1424. break;
  1425. //Edge iteration
  1426. case SGraphFirstEdge:
  1427. {
  1428. //Walk dependencies - should possibly have a different SST e.g., SSTdependency since they do not
  1429. //share many characteristics with edges - e.g. no flowing records => few/no stats.
  1430. curGraph.setown(graphIter->query().getXGMMLTree(false));
  1431. Owned<IPropertyTreeIterator> treeIter = curGraph->getElements("edge");
  1432. if (treeIter && treeIter->first())
  1433. {
  1434. treeIter.setown(createSortedIterator(*treeIter, compareEdgeNode));
  1435. treeIter->first();
  1436. pushIterator(treeIter, SGraphFirstSubGraph);
  1437. state = SEdge;
  1438. }
  1439. else
  1440. state = SGraphFirstSubGraph;
  1441. break;
  1442. }
  1443. case SChildGraphFirstEdge:
  1444. {
  1445. Owned<IPropertyTreeIterator> treeIter = treeIters.tos().query().getElements("att/graph/edge");
  1446. if (treeIter && treeIter->first())
  1447. {
  1448. treeIter.setown(createSortedIterator(*treeIter, compareEdgeNode));
  1449. pushIterator(treeIter, SChildGraphFirstSubGraph);
  1450. state = SEdge;
  1451. }
  1452. else
  1453. state = SChildGraphFirstSubGraph;
  1454. break;
  1455. }
  1456. case SEdgeEnd:
  1457. popScope();
  1458. state = SEdgeNext;
  1459. break;
  1460. case SEdgeNext:
  1461. if (treeIters.tos().next())
  1462. state = SEdge;
  1463. else
  1464. state = popIterator();
  1465. break;
  1466. //Subgraph iteration
  1467. case SGraphFirstSubGraph:
  1468. {
  1469. Owned<IPropertyTreeIterator> treeIter = curGraph->getElements("node");
  1470. if (treeIter && treeIter->first())
  1471. {
  1472. treeIter.setown(createSortedIterator(*treeIter, compareSubGraphNode));
  1473. treeIter->first();
  1474. pushIterator(treeIter, SGraphNext);
  1475. state = SSubGraph;
  1476. }
  1477. else
  1478. state = SGraphNext;
  1479. break;
  1480. }
  1481. case SChildGraphFirstSubGraph:
  1482. {
  1483. Owned<IPropertyTreeIterator> treeIter = treeIters.tos().query().getElements("att/graph/node");
  1484. if (treeIter && treeIter->first())
  1485. {
  1486. treeIter.setown(createSortedIterator(*treeIter, compareSubGraphNode));
  1487. pushIterator(treeIter, SChildGraphEnd);
  1488. state = SSubGraph;
  1489. }
  1490. else
  1491. state = SChildGraphEnd;
  1492. break;
  1493. }
  1494. case SSubGraphFirstEdge:
  1495. {
  1496. Owned<IPropertyTreeIterator> treeIter = treeIters.tos().query().getElements("att/graph/edge");
  1497. if (treeIter && treeIter->first())
  1498. {
  1499. treeIter.setown(createSortedIterator(*treeIter, compareEdgeNode));
  1500. pushIterator(treeIter, SSubGraphFirstActivity);
  1501. state = SEdge;
  1502. }
  1503. else
  1504. state = SSubGraphFirstActivity;
  1505. break;
  1506. }
  1507. case SSubGraphFirstActivity:
  1508. {
  1509. Owned<IPropertyTreeIterator> treeIter = treeIters.tos().query().getElements("att/graph/node");
  1510. if (treeIter && treeIter->first())
  1511. {
  1512. treeIter.setown(createSortedIterator(*treeIter, compareActivityNode));
  1513. pushIterator(treeIter, SSubGraphEnd);
  1514. state = SActivity;
  1515. }
  1516. else
  1517. state = SSubGraphEnd;
  1518. break;
  1519. }
  1520. case SSubGraphEnd:
  1521. popScope();
  1522. state = SSubGraphNext;
  1523. break;
  1524. case SChildGraphEnd:
  1525. popScope();
  1526. state = SChildGraphNext;
  1527. break;
  1528. case SSubGraphNext:
  1529. if (treeIters.tos().next())
  1530. state = SSubGraph;
  1531. else
  1532. state = popIterator();
  1533. break;
  1534. case SActivityEnd:
  1535. popScope();
  1536. state = SActivityNext;
  1537. break;
  1538. case SActivityNext:
  1539. if (treeIters.tos().next())
  1540. state = SActivity;
  1541. else
  1542. state = popIterator();
  1543. break;
  1544. case SChildGraphNext:
  1545. if (treeIters.tos().next())
  1546. state = SChildGraph;
  1547. else
  1548. state = popIterator();
  1549. break;
  1550. case SChildGraphFirst:
  1551. {
  1552. unsigned numIters = treeIters.ordinality();
  1553. IPropertyTreeIterator & graphIter = treeIters.item(numIters-2);
  1554. Owned<IPropertyTreeIterator> treeIter = graphIter.query().getElements("att/graph/node");
  1555. //Really want to filter by <att name="_parentActivity" value="<parentid>">
  1556. if (treeIter && treeIter->first())
  1557. {
  1558. treeIter.setown(createSortedIterator(*treeIter, compareSubGraphNode));
  1559. pushIterator(treeIter, SActivityEnd);
  1560. state = SChildGraph;
  1561. }
  1562. else
  1563. state = SActivityEnd;
  1564. break;
  1565. }
  1566. case SDone:
  1567. return false;
  1568. default:
  1569. throwUnexpected();
  1570. }
  1571. }
  1572. }
  1573. bool nextScope()
  1574. {
  1575. for(;;)
  1576. {
  1577. if (!doNextScope())
  1578. return false;
  1579. ScopeCompare cmp = filter.compare(curScopeName);
  1580. if (cmp & SCequal)
  1581. return true;
  1582. //MORE: Optimize next based on result of compare
  1583. if (!selectNext())
  1584. return false;
  1585. }
  1586. }
  1587. bool selectNext()
  1588. {
  1589. switch (state)
  1590. {
  1591. case SGraph:
  1592. state = SGraphFirstEdge;
  1593. break;
  1594. case SChildGraph:
  1595. state = SChildGraphFirstEdge;
  1596. break;
  1597. case SSubGraph:
  1598. state = SSubGraphFirstEdge;
  1599. break;
  1600. case SEdge:
  1601. state = SEdgeEnd;
  1602. break;
  1603. case SActivity:
  1604. state = SChildGraphFirst;
  1605. break;
  1606. case SDone:
  1607. return false;
  1608. default:
  1609. throwUnexpected();
  1610. }
  1611. return true;
  1612. }
  1613. bool selectNextSibling()
  1614. {
  1615. switch (state)
  1616. {
  1617. case SGraph:
  1618. state = SGraphEnd;
  1619. break;
  1620. case SChildGraph:
  1621. state = SChildGraphEnd;
  1622. break;
  1623. case SSubGraph:
  1624. state = SSubGraphEnd;
  1625. break;
  1626. case SEdge:
  1627. state = SEdgeEnd;
  1628. break;
  1629. case SActivity:
  1630. state = SActivityEnd;
  1631. break;
  1632. case SDone:
  1633. return false;
  1634. default:
  1635. throwUnexpected();
  1636. }
  1637. return true;
  1638. }
  1639. bool selectNextParent()
  1640. {
  1641. switch (state)
  1642. {
  1643. case SGraph:
  1644. state = SDone;
  1645. break;
  1646. case SChildGraph:
  1647. case SSubGraph:
  1648. case SEdge:
  1649. case SActivity:
  1650. popScope();
  1651. state = popIterator();
  1652. break;
  1653. case SDone:
  1654. return false;
  1655. default:
  1656. throwUnexpected();
  1657. }
  1658. return true;
  1659. }
  1660. protected:
  1661. const ScopeFilter & filter;
  1662. Owned<IConstWUGraphIterator> graphIter;
  1663. Owned<IPropertyTree> curGraph;
  1664. IArrayOf<IPropertyTreeIterator> treeIters;
  1665. UnsignedArray scopeLengths;
  1666. UnsignedArray stateStack;
  1667. StringBuffer curScopeName;
  1668. StringBuffer scopeId;
  1669. StatisticScopeType scopeType = SSTnone;
  1670. };
  1671. static int compareWorkflow(IInterface * const * pLeft, IInterface * const * pRight)
  1672. {
  1673. IConstWorkflowItem * left = static_cast<IConstWorkflowItem *>(*pLeft);
  1674. IConstWorkflowItem * right = static_cast<IConstWorkflowItem *>(*pRight);
  1675. return left->queryWfid() - right->queryWfid();
  1676. }
  1677. static const char * trueToStr(bool value) { return value ? "true" : nullptr; }
  1678. /*
  1679. * An implementation of IConstWUScopeIterator for the workflow information.
  1680. */
  1681. class WorkflowStatisticsScopeIterator : public CInterfaceOf<IConstWUScopeIterator>
  1682. {
  1683. public:
  1684. WorkflowStatisticsScopeIterator(IConstWorkflowItemIterator * wfIter)
  1685. {
  1686. ForEach(*wfIter)
  1687. workflow.append(*LINK(wfIter->query()));
  1688. workflow.sort(compareWorkflow);
  1689. }
  1690. virtual bool first() override
  1691. {
  1692. if (workflow.empty())
  1693. return false;
  1694. curWorkflow = 0;
  1695. return initWorkflowItem();
  1696. }
  1697. virtual bool next() override
  1698. {
  1699. if (!workflow.isItem(++curWorkflow))
  1700. return false;
  1701. return initWorkflowItem();
  1702. }
  1703. virtual bool nextSibling() override
  1704. {
  1705. return next();
  1706. }
  1707. virtual bool nextParent() override
  1708. {
  1709. curWorkflow = NotFound;
  1710. return false;
  1711. }
  1712. virtual bool isValid() override
  1713. {
  1714. return workflow.isItem(curWorkflow);
  1715. }
  1716. virtual const char * queryScope() const override
  1717. {
  1718. return curScope.str();
  1719. }
  1720. virtual StatisticScopeType getScopeType() const override
  1721. {
  1722. return SSTworkflow;
  1723. }
  1724. virtual void playProperties(IWuScopeVisitor & visitor, WuPropertyTypes whichProperties) override
  1725. {
  1726. if (whichProperties & PTattributes)
  1727. {
  1728. {
  1729. StringBuffer scratchpad;
  1730. Owned<IWorkflowDependencyIterator> depends = workflow.item(curWorkflow).getDependencies();
  1731. ForEach(*depends)
  1732. visitor.noteAttribute(WaIdDependency, getValueText(depends->query(), scratchpad, WorkflowScopePrefix));
  1733. }
  1734. play(visitor, { WaIsScheduled, WaIdSuccess, WaIdFailure, WaIdRecovery, WaIdPersist, WaIdScheduled,
  1735. WaPersistName, WaLabel, WaMode, WaType, WaState, WaCluster, WaCriticalSection });
  1736. }
  1737. }
  1738. virtual bool getStat(StatisticKind kind, unsigned __int64 & value) const override
  1739. {
  1740. return false;
  1741. }
  1742. virtual const char * queryAttribute(WuAttr attr, StringBuffer & scratchpad) const override
  1743. {
  1744. auto wf = &workflow.item(curWorkflow);
  1745. StringBufferAdaptor adaptor(scratchpad);
  1746. switch (attr)
  1747. {
  1748. case WaIdDependencyList:
  1749. {
  1750. bool first = true;
  1751. Owned<IWorkflowDependencyIterator> depends = workflow.item(curWorkflow).getDependencies();
  1752. ForEach(*depends)
  1753. {
  1754. if (first)
  1755. scratchpad.append("[");
  1756. else
  1757. scratchpad.append(",");
  1758. scratchpad.append('"').append(WorkflowScopePrefix).append(depends->query()).append('"');
  1759. first = false;
  1760. }
  1761. if (first)
  1762. return nullptr;
  1763. scratchpad.append("]");
  1764. return scratchpad.str();
  1765. }
  1766. case WaIsScheduled: return trueToStr(wf->isScheduled());
  1767. case WaIdSuccess: return getWfidText(wf->querySuccess(), scratchpad);
  1768. case WaIdFailure: return getWfidText(wf->queryFailure(), scratchpad);
  1769. case WaIdRecovery: return getWfidText(wf->queryRecovery(), scratchpad);
  1770. case WaIdPersist: return getWfidText(wf->queryPersistWfid(), scratchpad);
  1771. case WaIdScheduled: return getWfidText(wf->queryScheduledWfid(), scratchpad);
  1772. case WaPersistName: return queryOptString(wf->getPersistName(adaptor));
  1773. case WaLabel: return queryOptString(wf->getLabel(adaptor));
  1774. case WaMode: return wf->queryMode() != WFModeNormal ? queryWorkflowModeText(wf->queryMode()) : nullptr;
  1775. case WaType: return wf->queryType() != WFTypeNormal ? queryWorkflowTypeText(wf->queryType()) : nullptr;
  1776. case WaState: return queryWorkflowStateText(wf->queryState());
  1777. case WaCluster: return queryOptString(wf->queryCluster(adaptor));
  1778. case WaCriticalSection: return queryOptString(wf->getCriticalName(adaptor));
  1779. /*
  1780. The followng attributes are not generated - I'm not convinced they are very useful, but they could be added later.
  1781. virtual bool isScheduledNow() const = 0;
  1782. virtual IWorkflowEvent * getScheduleEvent() const = 0;
  1783. virtual unsigned querySchedulePriority() const = 0;
  1784. virtual bool hasScheduleCount() const = 0;
  1785. virtual unsigned queryScheduleCount() const = 0;
  1786. virtual unsigned queryRetriesAllowed() const = 0;
  1787. virtual int queryPersistCopies() const = 0; // 0 - unmangled name, < 0 - use default, > 0 - max number
  1788. virtual bool queryPersistRefresh() const = 0;
  1789. virtual unsigned queryScheduleCountRemaining() const = 0;
  1790. virtual unsigned queryRetriesRemaining() const = 0;
  1791. virtual int queryFailCode() const = 0;
  1792. virtual const char * queryFailMessage() const = 0;
  1793. virtual const char * queryEventName() const = 0;
  1794. virtual const char * queryEventExtra() const = 0;
  1795. */
  1796. }
  1797. return nullptr;
  1798. }
  1799. virtual const char * queryHint(const char * kind) const
  1800. {
  1801. return nullptr;
  1802. }
  1803. protected:
  1804. bool initWorkflowItem()
  1805. {
  1806. curScope.clear().append(WorkflowScopePrefix).append(workflow.item(curWorkflow).queryWfid());
  1807. return true;
  1808. }
  1809. const char * getValueText(unsigned value, StringBuffer & scratchpad, const char * prefix = nullptr) const
  1810. {
  1811. return scratchpad.clear().append(prefix).append(value).str();
  1812. }
  1813. const char * queryOptString(IStringVal & value) const
  1814. {
  1815. const char * text = value.str();
  1816. if (!text || !*text)
  1817. return nullptr;
  1818. return text;
  1819. }
  1820. const char * getWfidText(unsigned value, StringBuffer & scratchpad) const
  1821. {
  1822. if (!value)
  1823. return nullptr;
  1824. return scratchpad.clear().append(WorkflowScopePrefix).append(value).str();
  1825. }
  1826. void play(IWuScopeVisitor & visitor, const std::initializer_list<WuAttr> & attrs) const
  1827. {
  1828. StringBuffer scratchpad;
  1829. for (auto attr : attrs)
  1830. {
  1831. const char * value = queryAttribute(attr, scratchpad.clear());
  1832. if (value)
  1833. visitor.noteAttribute(attr, value);
  1834. }
  1835. }
  1836. protected:
  1837. IArrayOf<IConstWorkflowItem> workflow;
  1838. unsigned curWorkflow = 0;
  1839. StringBuffer curScope;
  1840. };
  1841. /*
  1842. * An implementation of IConstWUScopeIterator that combines results from multiple sources.
  1843. */
  1844. class CompoundStatisticsScopeIterator : public CInterfaceOf<IConstWUScopeIterator>
  1845. {
  1846. class VisitorMapper : implements IWuScopeVisitor
  1847. {
  1848. public:
  1849. VisitorMapper(const WuScopeFilter & _filter, IWuScopeVisitor & _visitor) :
  1850. filter(_filter), visitor(_visitor)
  1851. {}
  1852. virtual void noteStatistic(StatisticKind kind, unsigned __int64 value, IConstWUStatistic & extra) override
  1853. {
  1854. if (filter.includeStatistic(kind))
  1855. visitor.noteStatistic(kind, value, extra);
  1856. }
  1857. virtual void noteAttribute(WuAttr attr, const char * value) override
  1858. {
  1859. if (filter.includeAttribute(attr))
  1860. visitor.noteAttribute(attr, value);
  1861. }
  1862. virtual void noteHint(const char * kind, const char * value) override
  1863. {
  1864. if (filter.includeHint(kind))
  1865. visitor.noteHint(kind, value);
  1866. }
  1867. virtual void noteException(IConstWUException & exception) override
  1868. {
  1869. visitor.noteException(exception);
  1870. }
  1871. protected:
  1872. const WuScopeFilter & filter;
  1873. IWuScopeVisitor & visitor;
  1874. };
  1875. public:
  1876. CompoundStatisticsScopeIterator(const WuScopeFilter & _filter) : filter(_filter)
  1877. {
  1878. }
  1879. void addIter(IConstWUScopeIterator * iter)
  1880. {
  1881. if (iter)
  1882. {
  1883. iters.append(OLINK(*iter));
  1884. assertex(iters.ordinality() <= sizeof(activeIterMask)*8);
  1885. }
  1886. }
  1887. virtual bool first() override
  1888. {
  1889. activeIterMask = 0;
  1890. ForEachItemIn(i, iters)
  1891. {
  1892. if (iters.item(i).first())
  1893. activeIterMask |= (1U << i);
  1894. }
  1895. return findNextScope();
  1896. }
  1897. virtual bool next() override
  1898. {
  1899. selectNext();
  1900. return findNextScope();
  1901. }
  1902. virtual bool nextSibling() override
  1903. {
  1904. selectNextSibling();
  1905. return findNextScope();
  1906. }
  1907. virtual bool nextParent() override
  1908. {
  1909. selectNextParent();
  1910. return findNextScope();
  1911. }
  1912. virtual bool isValid() override
  1913. {
  1914. return (activeIterMask != 0);
  1915. }
  1916. virtual const char * queryScope() const override
  1917. {
  1918. return iters.item(firstMatchIter).queryScope();
  1919. }
  1920. virtual StatisticScopeType getScopeType() const override
  1921. {
  1922. return iters.item(firstMatchIter).getScopeType();
  1923. }
  1924. virtual void playProperties(IWuScopeVisitor & visitor, WuPropertyTypes whichProperties) override
  1925. {
  1926. VisitorMapper mappedVisitor(filter, visitor);
  1927. whichProperties &= filter.properties;
  1928. ForEachItemIn(i, iters)
  1929. {
  1930. if (iterMatchesCurrentScope(i))
  1931. iters.item(i).playProperties(mappedVisitor, whichProperties);
  1932. }
  1933. }
  1934. virtual bool getStat(StatisticKind kind, unsigned __int64 & value) const override
  1935. {
  1936. ForEachItemIn(i, iters)
  1937. {
  1938. if (iterMatchesCurrentScope(i))
  1939. {
  1940. if (iters.item(i).getStat(kind, value))
  1941. return true;
  1942. }
  1943. }
  1944. return false;
  1945. }
  1946. virtual const char * queryAttribute(WuAttr attr, StringBuffer & scratchpad) const override
  1947. {
  1948. ForEachItemIn(i, iters)
  1949. {
  1950. if (iterMatchesCurrentScope(i))
  1951. {
  1952. const char * value = iters.item(i).queryAttribute(attr, scratchpad);
  1953. if (value)
  1954. return value;
  1955. }
  1956. }
  1957. return nullptr;
  1958. }
  1959. virtual const char * queryHint(const char * kind) const override
  1960. {
  1961. ForEachItemIn(i, iters)
  1962. {
  1963. if (iterMatchesCurrentScope(i))
  1964. {
  1965. const char * value = iters.item(i).queryHint(kind);
  1966. if (value)
  1967. return value;
  1968. }
  1969. }
  1970. return nullptr;
  1971. }
  1972. inline bool iterMatchesCurrentScope(unsigned i) const { return ((1U << i) & matchIterMask) != 0; }
  1973. inline bool isAlive(unsigned i) const { return ((1U << i) & activeIterMask) != 0; }
  1974. protected:
  1975. //Calculate which iterators contain the scope that should come next
  1976. bool findNextScope()
  1977. {
  1978. for(;;)
  1979. {
  1980. if (activeIterMask == 0)
  1981. return false;
  1982. unsigned mask = 0;
  1983. const char * scope = nullptr;
  1984. ForEachItemIn(i, iters)
  1985. {
  1986. if (isAlive(i))
  1987. {
  1988. const char * iterScope = iters.item(i).queryScope();
  1989. if (mask)
  1990. {
  1991. int compare = compareScopeName(scope, iterScope);
  1992. if (compare == 0)
  1993. {
  1994. mask |= (1U << i);
  1995. }
  1996. else if (compare > 0)
  1997. {
  1998. scope = iterScope;
  1999. mask = (1U << i);
  2000. firstMatchIter = i;
  2001. }
  2002. }
  2003. else
  2004. {
  2005. scope = iterScope;
  2006. mask = (1U << i);
  2007. firstMatchIter = i;
  2008. }
  2009. }
  2010. }
  2011. matchIterMask = mask;
  2012. while (activeScopes)
  2013. {
  2014. const char * activeScope = activeScopes.tos();
  2015. //If the next scope if not a child of one of the active scopes then that active scope will no longer match.
  2016. if (compareScopes(scope, activeScope) & SCchild)
  2017. break;
  2018. activeScopes.pop();
  2019. }
  2020. //The top most scope will be the deepest. Check if the current scope is close enough to return as a match.
  2021. bool include = false;
  2022. if (activeScopes)
  2023. {
  2024. const char * activeScope = activeScopes.tos();
  2025. //code above has ensured that this scope must be a child of (and therefore deeper) than activeScope
  2026. unsigned nesting = queryScopeDepth(scope) - queryScopeDepth(activeScope);
  2027. if (nesting <= filter.include.nestedDepth)
  2028. include = true;
  2029. }
  2030. //Check to see if this is a new match
  2031. if (filter.compareMatchScopes(scope) & SCequal)
  2032. {
  2033. if (!include)
  2034. include = filter.include.matchedScope;
  2035. //Only add it to the list of active scopes if it can match child elements
  2036. if (filter.include.nestedDepth != 0)
  2037. activeScopes.append(scope);
  2038. }
  2039. if (include)
  2040. {
  2041. if (!filter.includeScope(scope))
  2042. include = false;
  2043. else if (filter.requiredStats.size())
  2044. {
  2045. //MORE: This would be cleaner as a member of filter - but it needs access to the stats.
  2046. for (unsigned iReq=0; iReq < filter.requiredStats.size(); iReq++)
  2047. {
  2048. const StatisticValueFilter & cur = filter.requiredStats[iReq];
  2049. unsigned __int64 value;
  2050. if (getStat(cur.queryKind(), value))
  2051. {
  2052. if (!cur.matches(value))
  2053. include = false;
  2054. }
  2055. else
  2056. include = false;
  2057. }
  2058. }
  2059. }
  2060. if (include)
  2061. return true;
  2062. //MORE: Optimize based on filter.compareScope()
  2063. selectNext();
  2064. }
  2065. }
  2066. void checkScopeOrder(unsigned input, const char * prevScope)
  2067. {
  2068. if (isAlive(input))
  2069. {
  2070. const char * curScope = iters.item(input).queryScope();
  2071. int compare = compareScopeName(prevScope, curScope);
  2072. if (compare >= 0)
  2073. throw MakeStringException(0, "Out of order (%u) scopes %s,%s = %d", input, prevScope, curScope, compare);
  2074. }
  2075. }
  2076. void selectNext()
  2077. {
  2078. ForEachItemIn(i, iters)
  2079. {
  2080. if (iterMatchesCurrentScope(i))
  2081. {
  2082. #ifdef _DEBUG
  2083. StringBuffer prevScope(iters.item(i).queryScope());
  2084. #endif
  2085. if (!iters.item(i).next())
  2086. activeIterMask &= ~(1U << i);
  2087. #ifdef _DEBUG
  2088. checkScopeOrder(i, prevScope);
  2089. #endif
  2090. }
  2091. }
  2092. }
  2093. void selectNextSibling()
  2094. {
  2095. ForEachItemIn(i, iters)
  2096. {
  2097. if (iterMatchesCurrentScope(i))
  2098. {
  2099. #ifdef _DEBUG
  2100. StringBuffer prevScope(iters.item(i).queryScope());
  2101. #endif
  2102. if (!iters.item(i).nextSibling())
  2103. activeIterMask &= ~(1U << i);
  2104. #ifdef _DEBUG
  2105. checkScopeOrder(i, prevScope);
  2106. #endif
  2107. }
  2108. }
  2109. }
  2110. void selectNextParent()
  2111. {
  2112. ForEachItemIn(i, iters)
  2113. {
  2114. if (iterMatchesCurrentScope(i))
  2115. {
  2116. #ifdef _DEBUG
  2117. StringBuffer prevScope(iters.item(i).queryScope());
  2118. #endif
  2119. if (!iters.item(i).nextParent())
  2120. activeIterMask &= ~(1U << i);
  2121. #ifdef _DEBUG
  2122. checkScopeOrder(i, prevScope);
  2123. #endif
  2124. }
  2125. }
  2126. }
  2127. protected:
  2128. const WuScopeFilter & filter;
  2129. IArrayOf<IConstWUScopeIterator> iters;
  2130. unsigned curIter = 0;
  2131. unsigned firstMatchIter = 0;
  2132. unsigned matchIterMask = 0; // bit set of iterators which have a valid entry for the current scope
  2133. unsigned activeIterMask = 0; // bit set of iterators which are still active
  2134. StringArray activeScopes;
  2135. };
  2136. //---------------------------------------------------------------------------------------------------------------------
  2137. class AggregateFilter
  2138. {
  2139. public:
  2140. AggregateFilter(StatisticKind _search) : search(_search) {}
  2141. inline bool matches(StatisticKind kind, unsigned __int64 value, IConstWUStatistic & extra) const
  2142. {
  2143. return (kind == search);
  2144. }
  2145. protected:
  2146. StatisticKind search;
  2147. //MORE: Allow filtering by scope?
  2148. };
  2149. class StatisticAggregator : public CInterfaceOf<IWuScopeVisitor>
  2150. {
  2151. public:
  2152. StatisticAggregator(StatisticKind _search) : filter(_search) {}
  2153. virtual void noteAttribute(WuAttr attr, const char * value) override { throwUnexpected(); }
  2154. virtual void noteHint(const char * kind, const char * value) override { throwUnexpected(); }
  2155. virtual void noteException(IConstWUException & exception) { throwUnexpected();}
  2156. protected:
  2157. AggregateFilter filter;
  2158. };
  2159. class SimpleAggregator : public StatisticAggregator
  2160. {
  2161. public:
  2162. SimpleAggregator(StatisticKind _search) : StatisticAggregator(_search) {}
  2163. virtual void noteStatistic(StatisticKind kind, unsigned __int64 value, IConstWUStatistic & extra) override
  2164. {
  2165. if (filter.matches(kind, value, extra))
  2166. summary.noteValue(value);
  2167. }
  2168. //How should these be reported? Should there be a playAggregates(IWuAggregatedScopeVisitor)
  2169. //with a noteAggregate(value, variant, value, grouping)?
  2170. protected:
  2171. StatsAggregation summary;
  2172. };
  2173. class SimpleReferenceAggregator : public StatisticAggregator
  2174. {
  2175. public:
  2176. SimpleReferenceAggregator(StatisticKind _search, StatsAggregation & _summary) : StatisticAggregator(_search), summary(_summary) {}
  2177. virtual void noteStatistic(StatisticKind kind, unsigned __int64 value, IConstWUStatistic & extra) override
  2178. {
  2179. if (filter.matches(kind, value, extra))
  2180. summary.noteValue(value);
  2181. }
  2182. //How should these be reported? Should there be a playAggregates(IWuAggregatedScopeVisitor)
  2183. //with a noteAggregate(value, variant, value, grouping)?
  2184. protected:
  2185. StatsAggregation & summary;
  2186. };
  2187. class GroupedAggregator : public StatisticAggregator
  2188. {
  2189. public:
  2190. virtual void noteStatistic(StatisticKind kind, unsigned __int64 value, IConstWUStatistic & extra) override
  2191. {
  2192. if (filter.matches(kind, value, extra))
  2193. {
  2194. StatsAggregation & match = summary; // look up in hash table.
  2195. match.noteValue(value);
  2196. }
  2197. }
  2198. protected:
  2199. //HashTable of (possible groupings->summary)
  2200. StatsAggregation summary;
  2201. };
  2202. class CompoundAggregator : implements StatisticAggregator
  2203. {
  2204. public:
  2205. virtual void noteStatistic(StatisticKind kind, unsigned __int64 value, IConstWUStatistic & extra)
  2206. {
  2207. ForEachItemIn(i, aggregators)
  2208. aggregators.item(i).noteStatistic(kind, value, extra);
  2209. }
  2210. protected:
  2211. IArrayOf<StatisticAggregator> aggregators;
  2212. };
  2213. //To calculate aggregates, create a scope iterator, and an instance of a StatisticAggregator, and play the attributes through the interface
  2214. //---------------------------------------------------------------------------------------------------------------------
  2215. //Extract an argument of the format abc[def[ghi...,]],... as (abc,def[...])
  2216. static bool extractOption(const char * & finger, StringBuffer & option, StringBuffer & arg)
  2217. {
  2218. const char * start = finger;
  2219. if (!*start)
  2220. return false;
  2221. const char * cur = start;
  2222. const char * bra = nullptr;
  2223. const char * end = nullptr;
  2224. unsigned braDepth = 0;
  2225. char next;
  2226. arg.clear();
  2227. for (;;)
  2228. {
  2229. next = *cur;
  2230. if (!next || ((next == ',') && (braDepth == 0)))
  2231. break;
  2232. switch (next)
  2233. {
  2234. case '[':
  2235. if (braDepth == 0)
  2236. {
  2237. if (bra)
  2238. throw makeStringExceptionV(0, "Multiple [ in filter : %s", bra);
  2239. bra = cur;
  2240. }
  2241. braDepth++;
  2242. break;
  2243. case ']':
  2244. if (braDepth > 0)
  2245. {
  2246. if (--braDepth == 0)
  2247. {
  2248. end = cur;
  2249. arg.append(cur - (bra+1), bra+1);
  2250. }
  2251. }
  2252. break;
  2253. }
  2254. cur++;
  2255. }
  2256. if (braDepth != 0)
  2257. throw makeStringExceptionV(0, "Mismatched ] in filter : %s", start);
  2258. option.clear();
  2259. if (bra)
  2260. {
  2261. if (cur != end+1)
  2262. throw makeStringExceptionV(0, "Text follows closing bracket: %s", end);
  2263. option.append(bra-start, start);
  2264. }
  2265. else
  2266. option.append(cur-start, start);
  2267. if (next)
  2268. finger = cur+1;
  2269. else
  2270. finger = cur;
  2271. return true;
  2272. }
  2273. static unsigned readOptValue(const char * start, const char * end, unsigned dft, const char * type)
  2274. {
  2275. if (start == end)
  2276. return dft;
  2277. char * next;
  2278. unsigned value = (unsigned)strtoll(start, &next, 10);
  2279. if (next != end)
  2280. throw makeStringExceptionV(0, "Unexpected characters in %s option '%s'", type, next);
  2281. return value;
  2282. }
  2283. static unsigned readValue(const char * start, const char * type)
  2284. {
  2285. if (*start == '\0')
  2286. throw makeStringExceptionV(0, "Expected a value for the %s option", type);
  2287. char * next;
  2288. unsigned value = (unsigned)strtoll(start, &next, 10);
  2289. if (*next != '\0')
  2290. throw makeStringExceptionV(0, "Unexpected characters in %s option '%s'", type, next);
  2291. return value;
  2292. }
  2293. /*
  2294. Scope service matching Syntax: * indicates
  2295. Which items are matched:
  2296. scope[<scope-id>]* | stype[<scope-type>]* | id[<id>]* - which scopes should be matched?
  2297. depth[n | low..high] - range of depths to search for a match
  2298. source[global|stats|graph|all]* - which sources to search within the workunit
  2299. where[<statistickind> | <statistickind> (=|<|<=|>|>=) value | <statistickind>=low..high] - a statistic filter
  2300. Which items are include in the results:
  2301. matched[true|false] - are the matched scopes returned?
  2302. nested[<depth>|all] - how deep within a scope should be matched (default = 0 if matched[true], all if matched[false])
  2303. includetype[<scope-type>] - which scope types should be included?
  2304. Which properties of the items are returned:
  2305. properties[statistics|hints|attributes|scope|all]
  2306. statistic[<statistic-kind>|none|all] - include statistic
  2307. attribute[<attribute-name>|none|all] - include attribute
  2308. hint[<hint-name>] - include hint
  2309. property[<statistic-kind>|<attribute-name>|<hint-name>] - include property
  2310. measure[<measure>] - all statistics with a particular measure
  2311. version[<version>] - minimum version to return
  2312. */
  2313. enum { FOscope, FOstype, FOid, FOdepth, FOsource, FOwhere, FOmatched, FOnested, FOinclude, FOproperties, FOstatistic, FOattribute, FOhint, FOproperty, FOmeasure, FOversion, FOunknown };
  2314. //Some of the following contains aliases for the same option e.g. stat and statistic
  2315. static constexpr EnumMapping filterOptions[] = {
  2316. { FOscope, "scope" }, { FOstype, "stype" }, { FOid, "id" },
  2317. { FOdepth, "depth" }, { FOsource, "source" }, { FOwhere, "where" },
  2318. { FOmatched, "matched" }, { FOnested, "nested" },
  2319. { FOinclude, "include" }, { FOinclude, "includetype" },
  2320. { FOproperties, "props" }, { FOproperties, "properties" }, // some aliases
  2321. { FOstatistic, "stat" }, { FOstatistic, "statistic" }, { FOattribute, "attr" }, { FOattribute, "attribute" }, { FOhint, "hint" },
  2322. { FOproperty, "prop" }, { FOproperty, "property" },
  2323. { FOmeasure, "measure" }, { FOversion, "version" }, { 0, nullptr} };
  2324. static constexpr EnumMapping sourceMappings[] = {
  2325. { SSFsearchGlobalStats, "global" }, { SSFsearchGraphStats, "stats" }, { SSFsearchGraphStats, "statistics" }, { SSFsearchGraph, "graph" }, { SSFsearchExceptions, "exception" }, { SSFsearchWorkflow, "workflow" },
  2326. { (int)SSFsearchAll, "all" }, { 0, nullptr } };
  2327. static constexpr EnumMapping propertyMappings[] = {
  2328. { PTstatistics, "stat" }, { PTstatistics, "statistic" }, { PTattributes, "attr" }, { PTattributes, "attribute" }, { PThints, "hint" },{ PTnotes, "note" },
  2329. { PTstatistics, "stats" }, { PTstatistics, "statistics" }, { PTattributes, "attrs" }, { PTattributes, "attributes" }, { PThints, "hints" }, { PTnotes, "notes" },
  2330. { PTnone, "none" }, { PTscope, "scope" }, { PTall, "all" }, { 0, nullptr } };
  2331. WuScopeSourceFlags querySource(const char * source) { return (WuScopeSourceFlags)getEnum(source, sourceMappings, SSFunknown); }
  2332. const char * querySourceText(WuScopeSourceFlags source) { return getEnumText(source, sourceMappings, nullptr); }
  2333. WuScopeFilter::WuScopeFilter(const char * filter)
  2334. {
  2335. addFilter(filter);
  2336. finishedFilter();
  2337. }
  2338. WuScopeFilter & WuScopeFilter::addFilter(const char * filter)
  2339. {
  2340. if (!filter)
  2341. return *this;
  2342. StringBuffer option;
  2343. StringBuffer arg;
  2344. while (extractOption(filter, option, arg))
  2345. {
  2346. switch (getEnum(option, filterOptions, FOunknown))
  2347. {
  2348. case FOscope:
  2349. addScope(arg);
  2350. break;
  2351. case FOstype:
  2352. addScopeType(arg);
  2353. break;
  2354. case FOid:
  2355. addId(arg);
  2356. break;
  2357. case FOdepth:
  2358. {
  2359. //Allow depth[n], depth[a,b] or depth[a..b]
  2360. const char * comma = strchr(arg, ',');
  2361. const char * dotdot = strstr(arg, "..");
  2362. if (comma)
  2363. {
  2364. unsigned low = readOptValue(arg, comma, 0, "depth");
  2365. if (comma[1])
  2366. {
  2367. scopeFilter.setDepth(low, readValue(comma+1, "depth"));
  2368. }
  2369. else
  2370. scopeFilter.setDepth(low, UINT_MAX);
  2371. }
  2372. else if (dotdot)
  2373. {
  2374. unsigned low = readOptValue(arg, dotdot, 0, "depth");
  2375. if (dotdot[2])
  2376. scopeFilter.setDepth(low, readValue(dotdot+2, "depth"));
  2377. else
  2378. scopeFilter.setDepth(low, UINT_MAX);
  2379. }
  2380. else
  2381. {
  2382. scopeFilter.setDepth(readValue(arg, "depth"));
  2383. }
  2384. break;
  2385. }
  2386. case FOsource:
  2387. addSource(arg);
  2388. break;
  2389. case FOwhere: // where[stat<op>value]
  2390. addRequiredStat(arg);
  2391. break;
  2392. case FOmatched:
  2393. setIncludeMatch(strToBool(arg));
  2394. break;
  2395. case FOnested:
  2396. if (strieq(arg, "all"))
  2397. setIncludeNesting(UINT_MAX);
  2398. else if (isdigit(*arg))
  2399. setIncludeNesting(atoi(arg));
  2400. else
  2401. throw makeStringExceptionV(0, "Expected a value for the nesting depth: %s", arg.str());
  2402. break;
  2403. case FOinclude:
  2404. setIncludeScopeType(arg);
  2405. break;
  2406. case FOproperties:
  2407. {
  2408. WuPropertyTypes prop = (WuPropertyTypes)getEnum(arg, propertyMappings, PTunknown);
  2409. if (prop == PTunknown)
  2410. throw makeStringExceptionV(0, "Unexpected properties '%s'", arg.str());
  2411. addOutputProperties(prop);
  2412. break;
  2413. }
  2414. case FOstatistic:
  2415. addOutputStatistic(arg);
  2416. break;
  2417. case FOattribute:
  2418. addOutputAttribute(arg);
  2419. break;
  2420. case FOhint:
  2421. addOutputHint(arg);
  2422. break;
  2423. case FOproperty:
  2424. addOutput(arg);
  2425. break;
  2426. case FOmeasure:
  2427. setMeasure(arg);
  2428. break;
  2429. case FOversion:
  2430. if (isdigit(*arg))
  2431. minVersion = atoi64(arg);
  2432. else
  2433. throw makeStringExceptionV(0, "Expected a value for the version: %s", arg.str());
  2434. break;
  2435. default:
  2436. throw makeStringExceptionV(0, "Unrecognised filter option: %s", option.str());
  2437. }
  2438. }
  2439. return *this;
  2440. }
  2441. WuScopeFilter & WuScopeFilter::addScope(const char * scope)
  2442. {
  2443. if (scope)
  2444. {
  2445. validateScope(scope);
  2446. scopeFilter.addScope(scope);
  2447. }
  2448. return *this;
  2449. }
  2450. WuScopeFilter & WuScopeFilter::addScopeType(const char * scopeType)
  2451. {
  2452. if (scopeType)
  2453. {
  2454. StatisticScopeType sst = queryScopeType(scopeType, SSTmax);
  2455. if (sst == SSTmax)
  2456. throw makeStringExceptionV(0, "Unrecognised scope type '%s'", scopeType);
  2457. scopeFilter.addScopeType(sst);
  2458. }
  2459. return *this;
  2460. }
  2461. WuScopeFilter & WuScopeFilter::addId(const char * id)
  2462. {
  2463. validateScopeId(id);
  2464. scopeFilter.addId(id);
  2465. return *this;
  2466. }
  2467. WuScopeFilter & WuScopeFilter::addOutput(const char * prop)
  2468. {
  2469. WuAttr attr = queryWuAttribute(prop, WaNone);
  2470. if (attr != WaNone)
  2471. {
  2472. WuAttr singleKindAttr = getSingleKindOfListAttribute(attr);
  2473. if (singleKindAttr != WaNone)
  2474. addOutputAttribute(singleKindAttr);
  2475. else
  2476. addOutputAttribute(attr);
  2477. } else if (queryStatisticKind(prop, StMax) != StMax)
  2478. addOutputStatistic(prop);
  2479. else
  2480. addOutputHint(prop);
  2481. return *this;
  2482. }
  2483. WuScopeFilter & WuScopeFilter::addOutputStatistic(const char * prop)
  2484. {
  2485. if (!prop)
  2486. return *this;
  2487. StatisticKind kind = queryStatisticKind(prop, StMax);
  2488. if (kind == StMax)
  2489. throw makeStringExceptionV(0, "Unrecognised statistic '%s'", prop);
  2490. return addOutputStatistic(kind);
  2491. }
  2492. WuScopeFilter & WuScopeFilter::addOutputStatistic(StatisticKind stat)
  2493. {
  2494. if (stat != StKindNone)
  2495. {
  2496. if (stat != StKindAll)
  2497. desiredStats.append(stat);
  2498. else
  2499. desiredStats.kill();
  2500. properties |= PTstatistics;
  2501. }
  2502. else
  2503. properties &= ~PTstatistics;
  2504. return *this;
  2505. }
  2506. WuScopeFilter & WuScopeFilter::addOutputAttribute(const char * prop)
  2507. {
  2508. if (!prop)
  2509. return *this;
  2510. WuAttr attr = queryWuAttribute(prop, WaMax);
  2511. if (attr == WaMax)
  2512. throw makeStringExceptionV(0, "Unrecognised attribute '%s'", prop);
  2513. return addOutputAttribute(attr);
  2514. }
  2515. WuScopeFilter & WuScopeFilter::addOutputAttribute(WuAttr attr)
  2516. {
  2517. if (attr != WaNone)
  2518. {
  2519. if (attr != WaAll)
  2520. desiredAttrs.append(attr);
  2521. else
  2522. desiredAttrs.kill();
  2523. properties |= PTattributes;
  2524. }
  2525. else
  2526. properties &= ~PTattributes;
  2527. return *this;
  2528. }
  2529. WuScopeFilter & WuScopeFilter::addOutputHint(const char * prop)
  2530. {
  2531. if (strieq(prop, "none"))
  2532. {
  2533. desiredHints.kill();
  2534. properties &= ~PThints;
  2535. }
  2536. else
  2537. {
  2538. if (!strieq(prop, "all"))
  2539. desiredHints.append(prop);
  2540. else
  2541. desiredHints.kill();
  2542. properties |= PThints;
  2543. }
  2544. return *this;
  2545. }
  2546. WuScopeFilter & WuScopeFilter::setIncludeMatch(bool value)
  2547. {
  2548. include.matchedScope = value;
  2549. return *this;
  2550. }
  2551. WuScopeFilter & WuScopeFilter::setIncludeNesting(unsigned depth)
  2552. {
  2553. include.nestedDepth = depth;
  2554. return *this;
  2555. }
  2556. WuScopeFilter & WuScopeFilter::setIncludeScopeType(const char * scopeType)
  2557. {
  2558. if (scopeType)
  2559. {
  2560. StatisticScopeType sst = queryScopeType(scopeType, SSTmax);
  2561. if (sst == SSTmax)
  2562. throw makeStringExceptionV(0, "Unrecognised scope type '%s'", scopeType);
  2563. include.scopeTypes.append(sst);
  2564. }
  2565. return *this;
  2566. }
  2567. WuScopeFilter & WuScopeFilter::setMeasure(const char * measure)
  2568. {
  2569. if (measure)
  2570. {
  2571. desiredMeasure = queryMeasure(measure, SMeasureNone);
  2572. if (desiredMeasure == SMeasureNone)
  2573. throw makeStringExceptionV(0, "Unrecognised measure '%s'", measure);
  2574. properties |= PTstatistics;
  2575. }
  2576. return *this;
  2577. }
  2578. WuScopeFilter & WuScopeFilter::addOutputProperties(WuPropertyTypes mask)
  2579. {
  2580. if (properties == PTnone)
  2581. properties = mask;
  2582. else
  2583. properties |= mask;
  2584. return *this;
  2585. }
  2586. WuScopeFilter & WuScopeFilter::addRequiredStat(StatisticKind statKind, stat_type lowValue, stat_type highValue)
  2587. {
  2588. requiredStats.emplace_back(statKind, lowValue, highValue);
  2589. return *this;
  2590. }
  2591. WuScopeFilter & WuScopeFilter::addRequiredStat(StatisticKind statKind)
  2592. {
  2593. requiredStats.emplace_back(statKind, 0, MaxStatisticValue);
  2594. return *this;
  2595. }
  2596. //process a filter in one of the following forms:
  2597. // <statistic-name>
  2598. // <statistic-name> (=|<|<=|>|>=) <value>
  2599. // <statistic-name>=[<low>]..[<high>]
  2600. void WuScopeFilter::addRequiredStat(const char * filter)
  2601. {
  2602. const char * stat = filter;
  2603. const char * cur = stat;
  2604. while (isalpha(*cur))
  2605. cur++;
  2606. StringBuffer statisticName(cur-stat, stat);
  2607. StatisticKind statKind = queryStatisticKind(statisticName, StKindNone);
  2608. if (statKind == StKindNone)
  2609. throw makeStringExceptionV(0, "Unknown statistic name '%s'", statisticName.str());
  2610. //Skip any spaces before a comparison operator.
  2611. while (*cur && isspace(*cur))
  2612. cur++;
  2613. //Save the operator, and skip over any non digits.
  2614. const char * op = cur;
  2615. switch (*op)
  2616. {
  2617. case '=':
  2618. cur++;
  2619. break;
  2620. case '<':
  2621. case '>':
  2622. if (op[1] == '=')
  2623. cur += 2;
  2624. else
  2625. cur++;
  2626. break;
  2627. case '\0':
  2628. break;
  2629. default:
  2630. throw makeStringExceptionV(0, "Unknown comparison '%s'", op);
  2631. }
  2632. const char * next;
  2633. stat_type value = readStatisticValue(cur, &next, queryMeasure(statKind));
  2634. stat_type lowValue = 0;
  2635. stat_type highValue = MaxStatisticValue;
  2636. switch (op[0])
  2637. {
  2638. case '=':
  2639. {
  2640. //Allow a,b or a..b to specify a range - either bound may be omitted.
  2641. if (next[0] == ',')
  2642. {
  2643. lowValue = value;
  2644. next++;
  2645. if (*next != '\0')
  2646. highValue = readStatisticValue(next, &next, queryMeasure(statKind));
  2647. }
  2648. else if (strncmp(next, "..", 2) == 0)
  2649. {
  2650. lowValue = value;
  2651. next += 2;
  2652. if (*next != '\0')
  2653. highValue = readStatisticValue(next, &next, queryMeasure(statKind));
  2654. }
  2655. else
  2656. {
  2657. lowValue = value;
  2658. highValue = value;
  2659. }
  2660. break;
  2661. }
  2662. case '<':
  2663. if (op[1] == '=')
  2664. highValue = value;
  2665. else
  2666. highValue = value-1;
  2667. break;
  2668. case '>':
  2669. if (op[1] == '=')
  2670. lowValue = value;
  2671. else
  2672. lowValue = value+1;
  2673. break;
  2674. }
  2675. if (*next)
  2676. throw makeStringExceptionV(0, "Trailing characters in where '%s'", next);
  2677. requiredStats.emplace_back(statKind, lowValue, highValue);
  2678. }
  2679. WuScopeFilter & WuScopeFilter::addSource(const char * source)
  2680. {
  2681. WuScopeSourceFlags mask = querySource(source);
  2682. if (mask == SSFunknown)
  2683. throw makeStringExceptionV(0, "Unexpected source '%s'", source);
  2684. if (!mask)
  2685. sourceFlags = mask;
  2686. else
  2687. sourceFlags |= mask;
  2688. return *this;
  2689. }
  2690. WuScopeFilter & WuScopeFilter::setDepth(unsigned low, unsigned high)
  2691. {
  2692. scopeFilter.setDepth(low, high);
  2693. return *this;
  2694. }
  2695. bool WuScopeFilter::matchOnly(StatisticScopeType scopeType) const
  2696. {
  2697. //If there is a post filter on the scopeType, then check if it matches
  2698. if (include.scopeTypes.ordinality() == 1)
  2699. return (include.scopeTypes.item(0) == scopeType);
  2700. //If filter doesn't match nested items, then check if the scope filter matches.
  2701. if (include.nestedDepth == 0)
  2702. {
  2703. if (scopeFilter.matchOnly(scopeType))
  2704. return true;
  2705. }
  2706. return false;
  2707. }
  2708. //Called once the filter has been updated to optimize the filter
  2709. void WuScopeFilter::finishedFilter()
  2710. {
  2711. scopeFilter.finishedFilter();
  2712. assertex(!optimized);
  2713. optimized = true;
  2714. if ((include.nestedDepth == 0) && !include.matchedScope)
  2715. include.nestedDepth = UINT_MAX;
  2716. preFilterScope = include.matchedScope && (include.nestedDepth == 0);
  2717. if (scopeFilter.canAlwaysPreFilter())
  2718. preFilterScope = true;
  2719. //If the source flags have not been explicitly set then calculate which sources will provide the results
  2720. if (!sourceFlags)
  2721. {
  2722. sourceFlags = SSFsearchAll;
  2723. //Use the other options to reduce the number of elements that are searched.
  2724. //If not interested in scopes on their own then...
  2725. if (!(properties & PTscope))
  2726. {
  2727. //Global stats and graph stats only contain stats => remove if not interested in them
  2728. if (!(properties & PTstatistics))
  2729. sourceFlags &= ~(SSFsearchGlobalStats|SSFsearchGraphStats);
  2730. //graph, workflow only contains attributes and hints => remove if not interested
  2731. if (!(properties & (PTattributes|PThints)))
  2732. sourceFlags &= ~(SSFsearchGraph|SSFsearchWorkflow);
  2733. }
  2734. if (!(properties & PTnotes))
  2735. sourceFlags &= ~SSFsearchExceptions;
  2736. }
  2737. //Optimize sources if they haven't been explicitly specified
  2738. //Most of the following are dependent on internal knowledge of the way that information is represented
  2739. if (matchOnly(SSTworkflow))
  2740. {
  2741. //Workflow is not nested within a graph
  2742. sourceFlags &= ~(SSFsearchGraph|SSFsearchGraphStats);
  2743. setDepth(1, 1);
  2744. }
  2745. else if (matchOnly(SSTgraph))
  2746. {
  2747. //Graph starts are stored globally, not in the graph stats.
  2748. sourceFlags &= ~(SSFsearchGraphStats|SSFsearchWorkflow);
  2749. if (!(properties & (PTattributes|PThints)))
  2750. sourceFlags &= ~(SSFsearchGraph);
  2751. //This should really be setDepth(2,2) but workunits prior to 7.4 did not have graph ids prefixed by the wfid
  2752. //Remove once 7.2 is a distant memory (see HPCC-22887)
  2753. setDepth(1, 2);
  2754. }
  2755. else if (matchOnly(SSTsubgraph))
  2756. {
  2757. sourceFlags &= ~(SSFsearchWorkflow);
  2758. }
  2759. else if (matchOnly(SSTcompilestage))
  2760. {
  2761. //compile stages are not stored in the graph
  2762. sourceFlags &= ~(SSFsearchGraphStats|SSFsearchGraph|SSFsearchWorkflow);
  2763. }
  2764. else if (matchOnly(SSTactivity))
  2765. {
  2766. //information about activities is not stored globally
  2767. sourceFlags &= ~(SSFsearchGlobalStats|SSFsearchWorkflow);
  2768. }
  2769. // Everything stored in the graphs stats has a depth of 3 or more - so ignore if there are not matches in that depth
  2770. if (include.nestedDepth == 0)
  2771. {
  2772. if (scopeFilter.compareDepth(3) > 0) // 3 is larger than any scope in the filter
  2773. sourceFlags &= ~(SSFsearchGraphStats);
  2774. else if (scopeFilter.compareDepth(3) == 0) // 3 matches the maximum scope in the filter
  2775. {
  2776. //Subgraph WhenStarted and TimeElapsed are stored globally. If that is all that is required
  2777. //then do not include the subgraph timings
  2778. if (desiredStats.ordinality())
  2779. {
  2780. bool needGraphStats = false;
  2781. ForEachItemIn(i, desiredStats)
  2782. {
  2783. unsigned stat = desiredStats.item(i);
  2784. if ((stat != StWhenStarted) && (stat != StTimeElapsed))
  2785. needGraphStats = true;
  2786. }
  2787. if (!needGraphStats)
  2788. {
  2789. sourceFlags &= ~(SSFsearchGraphStats);
  2790. if (matchOnly(SSTsubgraph))
  2791. sourceFlags &= SSFsearchGlobalStats;
  2792. }
  2793. }
  2794. }
  2795. }
  2796. if (scopeFilter.compareDepth(1) < 0)
  2797. {
  2798. //If minimum match depth is > 1 then it will never match workflow
  2799. sourceFlags &= ~(SSFsearchWorkflow);
  2800. }
  2801. //The xml graph is never updated, so do not check it if minVersion != 0
  2802. if (minVersion != 0)
  2803. {
  2804. sourceFlags &= ~(SSFsearchGraph|SSFsearchWorkflow);
  2805. }
  2806. }
  2807. bool WuScopeFilter::includeStatistic(StatisticKind kind) const
  2808. {
  2809. if (!(properties & PTstatistics))
  2810. return false;
  2811. if ((desiredMeasure != SMeasureAll) && (queryMeasure(kind) != desiredMeasure))
  2812. return false;
  2813. if (desiredStats.empty())
  2814. return true;
  2815. return desiredStats.contains(kind);
  2816. }
  2817. bool WuScopeFilter::includeAttribute(WuAttr attr) const
  2818. {
  2819. if (!(properties & PTattributes))
  2820. return false;
  2821. if (desiredAttrs.empty())
  2822. return true;
  2823. return desiredAttrs.contains(attr);
  2824. }
  2825. bool WuScopeFilter::includeHint(const char * kind) const
  2826. {
  2827. if (!(properties & PThints))
  2828. return false;
  2829. if (desiredHints.empty())
  2830. return true;
  2831. return desiredHints.contains(kind);
  2832. }
  2833. bool WuScopeFilter::includeScope(const char * scope) const
  2834. {
  2835. if (include.scopeTypes)
  2836. {
  2837. const char * tail = queryScopeTail(scope);
  2838. StatsScopeId id(tail);
  2839. if (!include.scopeTypes.contains(id.queryScopeType()))
  2840. return false;
  2841. }
  2842. return true;
  2843. }
  2844. const static ScopeFilter nullScopeFilter {};
  2845. const ScopeFilter & WuScopeFilter::queryIterFilter() const
  2846. {
  2847. if (preFilterScope)
  2848. return scopeFilter;
  2849. else
  2850. return nullScopeFilter;
  2851. }
  2852. ScopeCompare WuScopeFilter::compareMatchScopes(const char * scope) const
  2853. {
  2854. return scopeFilter.compare(scope);
  2855. }
  2856. StringBuffer & WuScopeFilter::describe(StringBuffer & out) const
  2857. {
  2858. scopeFilter.describe(out);
  2859. if (requiredStats.size())
  2860. {
  2861. out.append(",where[");
  2862. bool first = false;
  2863. for (const auto & stat : requiredStats)
  2864. {
  2865. if (!first)
  2866. out.append(",");
  2867. stat.describe(out);
  2868. first = false;
  2869. }
  2870. out.append("]");
  2871. }
  2872. {
  2873. StringBuffer sources;
  2874. for (unsigned mask=1; mask; mask *= 2)
  2875. {
  2876. if (sourceFlags & mask)
  2877. {
  2878. const char * source = querySourceText((WuScopeSourceFlags)mask);
  2879. if (source)
  2880. sources.append(",").append(source);
  2881. }
  2882. }
  2883. if (sources)
  2884. out.append(",source[").append(sources.str()+1).append("]");
  2885. }
  2886. if (include.nestedDepth != UINT_MAX)
  2887. out.appendf(",nested[%u]", include.nestedDepth);
  2888. if (include.scopeTypes)
  2889. {
  2890. out.append(",include[");
  2891. ForEachItemIn(i, include.scopeTypes)
  2892. {
  2893. if (i)
  2894. out.append(",");
  2895. out.append(queryScopeTypeName((StatisticScopeType)include.scopeTypes.item(i)));
  2896. }
  2897. out.append("]");
  2898. }
  2899. {
  2900. StringBuffer props;
  2901. if (properties == PTnone)
  2902. props.append(",none");
  2903. else if (properties == PTall)
  2904. props.append(",all");
  2905. else
  2906. {
  2907. if (properties & PTstatistics)
  2908. props.append(",stat");
  2909. if (properties & PTattributes)
  2910. props.append(",attr");
  2911. if (properties & PThints)
  2912. props.append(",hint");
  2913. if (properties & PTscope)
  2914. props.append(",scope");
  2915. if (properties & PTnotes)
  2916. props.append(",note");
  2917. }
  2918. out.append(",properties[").append(props.str()+1).append("]");
  2919. }
  2920. if (desiredStats)
  2921. {
  2922. out.append(",stat[");
  2923. ForEachItemIn(i, desiredStats)
  2924. {
  2925. if (i)
  2926. out.append(",");
  2927. out.append(queryStatisticName((StatisticKind)desiredStats.item(i)));
  2928. }
  2929. out.append("]");
  2930. }
  2931. if (desiredAttrs)
  2932. {
  2933. out.append(",attr[");
  2934. ForEachItemIn(i, desiredAttrs)
  2935. {
  2936. if (i)
  2937. out.append(",");
  2938. out.append(queryWuAttributeName((WuAttr)desiredAttrs.item(i)));
  2939. }
  2940. out.append("]");
  2941. }
  2942. if (desiredHints)
  2943. {
  2944. out.append(",hint[");
  2945. ForEachItemIn(i, desiredHints)
  2946. {
  2947. if (i)
  2948. out.append(",");
  2949. out.append(desiredHints.item(i));
  2950. }
  2951. out.append("]");
  2952. }
  2953. if (desiredMeasure != SMeasureAll)
  2954. out.append(",measure[").append(queryMeasureName(desiredMeasure)).append("]");
  2955. if (minVersion != 0)
  2956. out.appendf(",version(%" I64F "u)", minVersion);
  2957. return out;
  2958. }
  2959. //--------------------------------------------------------------------------------------------------------------------
  2960. EnumMapping states[] = {
  2961. { WUStateUnknown, "unknown" },
  2962. { WUStateCompiled, "compiled" },
  2963. { WUStateRunning, "running" },
  2964. { WUStateCompleted, "completed" },
  2965. { WUStateFailed, "failed" },
  2966. { WUStateArchived, "archived" },
  2967. { WUStateAborting, "aborting" },
  2968. { WUStateAborted, "aborted" },
  2969. { WUStateBlocked, "blocked" },
  2970. { WUStateSubmitted, "submitted" },
  2971. { WUStateScheduled, "scheduled" },
  2972. { WUStateCompiling, "compiling" },
  2973. { WUStateWait, "wait" },
  2974. { WUStateUploadingFiles, "uploading_files" },
  2975. { WUStateDebugPaused, "debugging" },
  2976. { WUStateDebugRunning, "debug_running" },
  2977. { WUStatePaused, "paused" },
  2978. { WUStateSize, NULL }
  2979. };
  2980. EnumMapping actions[] = {
  2981. { WUActionUnknown, "unknown" },
  2982. { WUActionCompile, "compile" },
  2983. { WUActionCheck, "check" },
  2984. { WUActionRun, "run" },
  2985. { WUActionExecuteExisting, "execute" },
  2986. { WUActionPause, "pause" },
  2987. { WUActionPauseNow, "pausenow" },
  2988. { WUActionResume, "resume" },
  2989. { WUActionSize, NULL },
  2990. };
  2991. EnumMapping priorityClasses[] = {
  2992. { PriorityClassUnknown, "unknown" },
  2993. { PriorityClassLow, "low" },
  2994. { PriorityClassNormal, "normal" },
  2995. { PriorityClassHigh, "high" },
  2996. { PriorityClassSize, NULL },
  2997. };
  2998. const char * getWorkunitStateStr(WUState state)
  2999. {
  3000. dbgassertex(state < WUStateSize);
  3001. return states[state].str; // MORE - should be using getEnumText, or need to take steps to ensure values remain contiguous and in order.
  3002. }
  3003. void setEnum(IPropertyTree *p, const char *propname, int value, const EnumMapping *map)
  3004. {
  3005. const char *defval = map->str;
  3006. while (map->str)
  3007. {
  3008. if (value==map->val)
  3009. {
  3010. p->setProp(propname, map->str);
  3011. return;
  3012. }
  3013. map++;
  3014. }
  3015. assertex(!"Unexpected value in setEnum");
  3016. p->setProp(propname, defval);
  3017. }
  3018. static int getEnum(const IPropertyTree *p, const char *propname, const EnumMapping *map)
  3019. {
  3020. return getEnum(p->queryProp(propname),map);
  3021. }
  3022. const char * getWorkunitActionStr(WUAction action)
  3023. {
  3024. return getEnumText(action, actions);
  3025. }
  3026. WUAction getWorkunitAction(const char *actionStr)
  3027. {
  3028. return (WUAction) getEnum(actionStr, actions);
  3029. }
  3030. //==========================================================================================
  3031. class CLightweightWorkunitInfo : public CInterfaceOf<IConstWorkUnitInfo>
  3032. {
  3033. public:
  3034. CLightweightWorkunitInfo(IPropertyTree &p)
  3035. {
  3036. wuid.set(p.queryName());
  3037. user.set(p.queryProp("@submitID"));
  3038. jobName.set(p.queryProp("@jobName"));
  3039. clusterName.set(p.queryProp("@clusterName"));
  3040. timeScheduled.set(p.queryProp("@timeScheduled"));
  3041. state = (WUState) getEnum(&p, "@state", states);
  3042. action = (WUAction) getEnum(&p, "Action", actions);
  3043. priority = (WUPriorityClass) getEnum(&p, "@priorityClass", priorityClasses);
  3044. priorityLevel = calcPriorityValue(&p);
  3045. wuscope.set(p.queryProp("@scope"));
  3046. appvalues.loadBranch(&p,"Application");
  3047. totalThorTime = (unsigned)nanoToMilli(extractTimeCollatable(p.queryProp("@totalThorTime"), false));
  3048. _isProtected = p.getPropBool("@protected", false);
  3049. }
  3050. virtual const char *queryWuid() const { return wuid.str(); }
  3051. virtual const char *queryUser() const { return user.str(); }
  3052. virtual const char *queryJobName() const { return jobName.str(); }
  3053. virtual const char *queryClusterName() const { return clusterName.str(); }
  3054. virtual const char *queryWuScope() const { return wuscope.str(); }
  3055. virtual WUState getState() const { return state; }
  3056. virtual const char *queryStateDesc() const { return getEnumText(state, states); }
  3057. virtual WUAction getAction() const { return action; }
  3058. virtual const char *queryActionDesc() const { return getEnumText(action, actions); }
  3059. virtual WUPriorityClass getPriority() const { return priority; }
  3060. virtual const char *queryPriorityDesc() const { return getEnumText(priority, priorityClasses); }
  3061. virtual int getPriorityLevel() const { return priorityLevel; }
  3062. virtual bool isProtected() const { return _isProtected; }
  3063. virtual IJlibDateTime & getTimeScheduled(IJlibDateTime & val) const
  3064. {
  3065. if (timeScheduled.length())
  3066. val.setGmtString(timeScheduled.str());
  3067. return val;
  3068. }
  3069. virtual unsigned getTotalThorTime() const { return totalThorTime; };
  3070. virtual IConstWUAppValueIterator & getApplicationValues() const { return *new CArrayIteratorOf<IConstWUAppValue,IConstWUAppValueIterator> (appvalues, 0, (IConstWorkUnitInfo *) this); };
  3071. protected:
  3072. StringAttr wuid, user, jobName, clusterName, timeScheduled, wuscope;
  3073. mutable CachedTags<CLocalWUAppValue,IConstWUAppValue> appvalues;
  3074. unsigned totalThorTime;
  3075. WUState state;
  3076. WUAction action;
  3077. WUPriorityClass priority;
  3078. int priorityLevel;
  3079. bool _isProtected;
  3080. };
  3081. extern IConstWorkUnitInfo *createConstWorkUnitInfo(IPropertyTree &p)
  3082. {
  3083. return new CLightweightWorkunitInfo(p);
  3084. }
  3085. class CDaliWuGraphStats : public CWuGraphStats
  3086. {
  3087. public:
  3088. CDaliWuGraphStats(IRemoteConnection *_conn, StatisticCreatorType _creatorType, const char * _creator, unsigned _wfid, const char * _rootScope, unsigned _id)
  3089. : CWuGraphStats(LINK(_conn->queryRoot()), _creatorType, _creator, _wfid, _rootScope, _id), conn(_conn)
  3090. {
  3091. }
  3092. protected:
  3093. Owned<IRemoteConnection> conn;
  3094. };
  3095. CWorkUnitWatcher::CWorkUnitWatcher(IWorkUnitSubscriber *_subscriber, WUSubscribeOptions flags, const char *wuid) : subscriber(_subscriber)
  3096. {
  3097. abortId = 0;
  3098. stateId = 0;
  3099. actionId = 0;
  3100. assertex((flags & ~SubscribeOptionAbort) == 0);
  3101. if (flags & SubscribeOptionAbort)
  3102. {
  3103. VStringBuffer xpath("/WorkUnitAborts/%s", wuid);
  3104. abortId = querySDS().subscribe(xpath.str(), *this, false, true);
  3105. }
  3106. }
  3107. CWorkUnitWatcher::~CWorkUnitWatcher()
  3108. {
  3109. assertex(abortId==0 && stateId==0 && actionId==0);
  3110. }
  3111. void CWorkUnitWatcher::unsubscribe()
  3112. {
  3113. CriticalBlock b(crit);
  3114. if (abortId)
  3115. querySDS().unsubscribe(abortId);
  3116. if (stateId)
  3117. querySDS().unsubscribe(stateId);
  3118. if (actionId)
  3119. querySDS().unsubscribe(actionId);
  3120. abortId = 0;
  3121. stateId = 0;
  3122. actionId = 0;
  3123. }
  3124. void CWorkUnitWatcher::notify(SubscriptionId id, const char *xpath, SDSNotifyFlags flags, unsigned valueLen, const void *valueData)
  3125. {
  3126. CriticalBlock b(crit);
  3127. if (id==stateId)
  3128. subscriber->notify(SubscribeOptionState, valueLen, valueData);
  3129. else if (id==actionId)
  3130. subscriber->notify(SubscribeOptionAction, valueLen, valueData);
  3131. else if (id==abortId)
  3132. subscriber->notify(SubscribeOptionAbort, valueLen, valueData);
  3133. }
  3134. class CDaliWorkUnitWatcher : public CWorkUnitWatcher
  3135. {
  3136. public:
  3137. CDaliWorkUnitWatcher(IWorkUnitSubscriber *_subscriber, WUSubscribeOptions flags, const char *wuid)
  3138. : CWorkUnitWatcher(_subscriber, (WUSubscribeOptions) (flags & SubscribeOptionAbort), wuid)
  3139. {
  3140. if (flags & SubscribeOptionState)
  3141. {
  3142. VStringBuffer xpath("/WorkUnits/%s/State", wuid);
  3143. stateId = querySDS().subscribe(xpath.str(), *this);
  3144. }
  3145. if (flags & SubscribeOptionAction)
  3146. {
  3147. VStringBuffer xpath("/WorkUnits/%s/Action", wuid);
  3148. actionId = querySDS().subscribe(xpath.str(), *this);
  3149. }
  3150. }
  3151. };
  3152. void CPersistedWorkUnit::subscribe(WUSubscribeOptions options)
  3153. {
  3154. CriticalBlock block(crit);
  3155. assertex(options==SubscribeOptionAbort);
  3156. if (!abortWatcher)
  3157. {
  3158. abortWatcher.setown(new CWorkUnitWatcher(this, SubscribeOptionAbort, p->queryName()));
  3159. abortDirty = true;
  3160. }
  3161. }
  3162. void CPersistedWorkUnit::unsubscribe()
  3163. {
  3164. CriticalBlock block(crit);
  3165. if (abortWatcher)
  3166. {
  3167. abortWatcher->unsubscribe();
  3168. abortWatcher.clear();
  3169. }
  3170. }
  3171. bool CPersistedWorkUnit::aborting() const
  3172. {
  3173. CriticalBlock block(crit);
  3174. if (abortDirty)
  3175. {
  3176. StringBuffer apath;
  3177. apath.append("/WorkUnitAborts/").append(p->queryName());
  3178. Owned<IRemoteConnection> acon = querySDS().connect(apath.str(), myProcessSession(), 0, SDS_LOCK_TIMEOUT);
  3179. if (acon)
  3180. abortState = acon->queryRoot()->getPropInt(NULL) != 0;
  3181. else
  3182. abortState = false;
  3183. abortDirty = false;
  3184. }
  3185. return abortState;
  3186. }
  3187. class CDaliWorkUnit : public CPersistedWorkUnit
  3188. {
  3189. public:
  3190. CDaliWorkUnit(IRemoteConnection *_conn, ISecManager *secmgr, ISecUser *secuser)
  3191. : connection(_conn), CPersistedWorkUnit(secmgr, secuser)
  3192. {
  3193. loadPTree(connection->getRoot());
  3194. }
  3195. ~CDaliWorkUnit()
  3196. {
  3197. // NOTE - order is important - we need to construct connection before p and (especially) destroy after p
  3198. // We use the beforeDispose() in base class to help ensure this
  3199. p.clear();
  3200. }
  3201. IConstWUGraphProgress *getGraphProgress(const char *graphName) const
  3202. {
  3203. Owned<IRemoteConnection> conn = getProgressConnection();
  3204. if (conn)
  3205. {
  3206. IPTree *progress = conn->queryRoot()->queryPropTree(graphName);
  3207. if (progress)
  3208. return new CConstGraphProgress(p->queryName(), graphName, progress);
  3209. }
  3210. return NULL;
  3211. }
  3212. virtual WUGraphState queryGraphState(const char *graphName) const
  3213. {
  3214. Owned<IRemoteConnection> conn = getProgressConnection();
  3215. if (conn)
  3216. {
  3217. IPTree *progress = conn->queryRoot()->queryPropTree(graphName);
  3218. if (progress)
  3219. return (WUGraphState) progress->getPropInt("@_state", (unsigned) WUGraphUnknown);
  3220. }
  3221. return WUGraphUnknown;
  3222. }
  3223. virtual WUGraphState queryNodeState(const char *graphName, WUGraphIDType nodeId) const
  3224. {
  3225. Owned<IRemoteConnection> conn = getProgressConnection();
  3226. if (conn)
  3227. {
  3228. IPTree *progress = conn->queryRoot()->queryPropTree(graphName);
  3229. if (progress)
  3230. {
  3231. StringBuffer path;
  3232. // NOTE - the node state info still uses the old graph layout, even when the stats are using the new...
  3233. path.append("node[@id=\"").append(nodeId).append("\"]/@_state");
  3234. return (WUGraphState) progress->getPropInt(path, (unsigned) WUGraphUnknown);
  3235. }
  3236. }
  3237. return WUGraphUnknown;
  3238. }
  3239. virtual void clearGraphProgress() const
  3240. {
  3241. CriticalBlock block(crit);
  3242. progressConnection.clear(); // Make sure nothing is locking for read or we won't be able to lock for write
  3243. StringBuffer path("/GraphProgress/");
  3244. path.append(p->queryName());
  3245. Owned<IRemoteConnection> delconn = querySDS().connect(path.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
  3246. if (delconn)
  3247. delconn->close(true);
  3248. }
  3249. virtual bool getRunningGraph(IStringVal &graphName, WUGraphIDType &subId) const
  3250. {
  3251. Owned<IRemoteConnection> conn = getProgressConnection();
  3252. if (!conn)
  3253. return false;
  3254. const char *name = conn->queryRoot()->queryProp("Running/@graph");
  3255. if (name)
  3256. {
  3257. graphName.set(name);
  3258. subId = conn->queryRoot()->getPropInt64("Running/@subId");
  3259. return true;
  3260. }
  3261. else
  3262. return false;
  3263. }
  3264. virtual void forceReload()
  3265. {
  3266. synchronized sync(locked); // protect locked workunits (uncommitted writes) from reload
  3267. CriticalBlock block(crit);
  3268. clearCached(true);
  3269. connection->reload();
  3270. progressConnection.clear();
  3271. abortDirty = true;
  3272. p.setown(connection->getRoot());
  3273. }
  3274. virtual void cleanupAndDelete(bool deldll, bool deleteOwned, const StringArray *deleteExclusions)
  3275. {
  3276. CPersistedWorkUnit::cleanupAndDelete(deldll, deleteOwned, deleteExclusions);
  3277. clearGraphProgress();
  3278. connection->close(true);
  3279. connection.clear();
  3280. }
  3281. virtual void commit()
  3282. {
  3283. CPersistedWorkUnit::commit();
  3284. if (connection)
  3285. connection->commit();
  3286. }
  3287. virtual void _lockRemote()
  3288. {
  3289. StringBuffer wuRoot;
  3290. getXPath(wuRoot, p->queryName());
  3291. if (connection)
  3292. connection->changeMode(RTM_LOCK_WRITE,SDS_LOCK_TIMEOUT);
  3293. else
  3294. connection.setown(querySDS().connect(wuRoot.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT));
  3295. if (!connection)
  3296. throw MakeStringException(WUERR_LockFailed, "Failed to get connection for xpath %s", wuRoot.str());
  3297. clearCached(true);
  3298. p.setown(connection->getRoot());
  3299. }
  3300. virtual void _unlockRemote()
  3301. {
  3302. if (connection)
  3303. {
  3304. try
  3305. {
  3306. try
  3307. {
  3308. connection->commit();
  3309. }
  3310. catch (IException *e)
  3311. {
  3312. EXCLOG(e, "Error during workunit commit");
  3313. connection->rollback();
  3314. connection->changeMode(0, SDS_LOCK_TIMEOUT);
  3315. throw;
  3316. }
  3317. connection->changeMode(0, SDS_LOCK_TIMEOUT);
  3318. }
  3319. catch (IException *E)
  3320. {
  3321. StringBuffer s;
  3322. IERRLOG("Failed to release write lock on workunit: %s", E->errorMessage(s).str());
  3323. throw;
  3324. }
  3325. }
  3326. }
  3327. virtual void setGraphState(const char *graphName, unsigned wfid, WUGraphState state) const
  3328. {
  3329. Owned<IRemoteConnection> conn = getWritableProgressConnection(graphName, wfid);
  3330. conn->queryRoot()->setPropInt("@_state", state);
  3331. }
  3332. virtual void setNodeState(const char *graphName, WUGraphIDType nodeId, WUGraphState state) const
  3333. {
  3334. CriticalBlock block(crit);
  3335. progressConnection.clear(); // Make sure nothing is locking for read or we won't be able to lock for write
  3336. VStringBuffer path("/GraphProgress/%s", queryWuid());
  3337. Owned<IRemoteConnection> conn = querySDS().connect(path, myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
  3338. IPTree *progress = ensurePTree(conn->queryRoot(), graphName);
  3339. // NOTE - the node state info still uses the old graph layout, even when the stats are using the new...
  3340. path.clear().append("node[@id=\"").append(nodeId).append("\"]");
  3341. IPropertyTree *node = progress->queryPropTree(path.str());
  3342. if (!node)
  3343. {
  3344. node = progress->addPropTree("node");
  3345. node->setPropInt64("@id", nodeId);
  3346. }
  3347. node->setPropInt("@_state", (unsigned)state);
  3348. switch (state)
  3349. {
  3350. case WUGraphRunning:
  3351. {
  3352. IPropertyTree *running = conn->queryRoot()->setPropTree("Running");
  3353. running->setProp("@graph", graphName);
  3354. running->setPropInt64("@subId", nodeId);
  3355. break;
  3356. }
  3357. case WUGraphComplete:
  3358. {
  3359. conn->queryRoot()->removeProp("Running"); // only one thing running at any given time and one thing with lockWrite access
  3360. break;
  3361. }
  3362. }
  3363. }
  3364. virtual IWUGraphStats *updateStats(const char *graphName, StatisticCreatorType creatorType, const char * creator, unsigned _wfid, unsigned subgraph) const override
  3365. {
  3366. return new CDaliWuGraphStats(getWritableProgressConnection(graphName, _wfid), creatorType, creator, _wfid, graphName, subgraph);
  3367. }
  3368. protected:
  3369. IRemoteConnection *getProgressConnection() const
  3370. {
  3371. CriticalBlock block(crit);
  3372. if (!progressConnection)
  3373. {
  3374. VStringBuffer path("/GraphProgress/%s", queryWuid());
  3375. progressConnection.setown(querySDS().connect(path, myProcessSession(), 0, SDS_LOCK_TIMEOUT)); // Note - we don't lock. The writes are atomic.
  3376. }
  3377. return progressConnection.getLink();
  3378. }
  3379. IRemoteConnection *getWritableProgressConnection(const char *graphName, unsigned wfid) const
  3380. {
  3381. CriticalBlock block(crit);
  3382. progressConnection.clear(); // Make sure subsequent reads from this workunit get the changes I am making
  3383. VStringBuffer path("/GraphProgress/%s/%s", queryWuid(), graphName);
  3384. Owned<IRemoteConnection> conn = querySDS().connect(path, myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
  3385. IPropertyTree * root = conn->queryRoot();
  3386. assertex(wfid);
  3387. if (!root->hasProp("@wfid"))
  3388. {
  3389. root->setPropInt("@wfid", wfid);
  3390. }
  3391. else
  3392. {
  3393. //Ideally the following code would check that the wfids are passed consistently.
  3394. //However there is an obscure problem with out of line functions being called from multiple workflow
  3395. //ids, and possibly library graphs.
  3396. //Stats for library graphs should be nested below the library call activity
  3397. //assertex(root->getPropInt("@wfid", 0) == wfid); // check that wfid is passed consistently
  3398. }
  3399. return conn.getClear();
  3400. }
  3401. IPropertyTree *getGraphProgressTree() const
  3402. {
  3403. Owned<IRemoteConnection> conn = getProgressConnection();
  3404. if (conn)
  3405. {
  3406. Owned<IPropertyTree> tmp = createPTree("GraphProgress");
  3407. mergePTree(tmp,conn->queryRoot());
  3408. return tmp.getClear();
  3409. }
  3410. return NULL;
  3411. }
  3412. Owned<IRemoteConnection> connection;
  3413. mutable Owned<IRemoteConnection> progressConnection;
  3414. };
  3415. class CLockedWorkUnit : implements ILocalWorkUnit, implements IExtendedWUInterface, public CInterface
  3416. {
  3417. public:
  3418. Owned<CLocalWorkUnit> c;
  3419. IMPLEMENT_IINTERFACE;
  3420. CLockedWorkUnit(CLocalWorkUnit *_c) : c(_c) {}
  3421. ~CLockedWorkUnit()
  3422. {
  3423. if (workUnitTraceLevel > 1)
  3424. DBGLOG("Releasing locked workunit %s", queryWuid());
  3425. if (c)
  3426. c->unlockRemote();
  3427. }
  3428. virtual IConstWorkUnit * unlock()
  3429. {
  3430. c->unlockRemote();
  3431. return c.getClear();
  3432. }
  3433. virtual bool aborting() const
  3434. { return c->aborting(); }
  3435. virtual void forceReload()
  3436. { UNIMPLEMENTED; }
  3437. virtual WUAction getAction() const
  3438. { return c->getAction(); }
  3439. virtual const char *queryActionDesc() const
  3440. { return c->queryActionDesc(); }
  3441. virtual IStringVal & getApplicationValue(const char * application, const char * propname, IStringVal & str) const
  3442. { return c->getApplicationValue(application, propname, str); }
  3443. virtual int getApplicationValueInt(const char * application, const char * propname, int defVal) const
  3444. { return c->getApplicationValueInt(application, propname, defVal); }
  3445. virtual IConstWUAppValueIterator & getApplicationValues() const
  3446. { return c->getApplicationValues(); }
  3447. virtual bool hasWorkflow() const
  3448. { return c->hasWorkflow(); }
  3449. virtual unsigned queryEventScheduledCount() const
  3450. { return c->queryEventScheduledCount(); }
  3451. virtual IPropertyTree * queryWorkflowTree() const
  3452. { return c->queryWorkflowTree(); }
  3453. virtual IConstWorkflowItemIterator * getWorkflowItems() const
  3454. { return c->getWorkflowItems(); }
  3455. virtual IWorkflowItemArray * getWorkflowClone() const
  3456. { return c->getWorkflowClone(); }
  3457. virtual bool requiresLocalFileUpload() const
  3458. { return c->requiresLocalFileUpload(); }
  3459. virtual IConstLocalFileUploadIterator * getLocalFileUploads() const
  3460. { return c->getLocalFileUploads(); }
  3461. virtual bool getIsQueryService() const
  3462. { return c->getIsQueryService(); }
  3463. virtual bool getCloneable() const
  3464. { return c->getCloneable(); }
  3465. virtual IUserDescriptor * queryUserDescriptor() const
  3466. { return c->queryUserDescriptor(); }
  3467. virtual const char *queryClusterName() const
  3468. { return c->queryClusterName(); }
  3469. virtual unsigned getCodeVersion() const
  3470. { return c->getCodeVersion(); }
  3471. virtual unsigned getWuidVersion() const
  3472. { return c->getWuidVersion(); }
  3473. virtual void getBuildVersion(IStringVal & buildVersion, IStringVal & eclVersion) const
  3474. { c->getBuildVersion(buildVersion, eclVersion); }
  3475. virtual bool hasDebugValue(const char * propname) const
  3476. { return c->hasDebugValue(propname); }
  3477. virtual IStringVal & getDebugValue(const char * propname, IStringVal & str) const
  3478. { return c->getDebugValue(propname, str); }
  3479. virtual int getDebugValueInt(const char * propname, int defVal) const
  3480. { return c->getDebugValueInt(propname, defVal); }
  3481. virtual __int64 getDebugValueInt64(const char * propname, __int64 defVal) const
  3482. { return c->getDebugValueInt64(propname, defVal); }
  3483. virtual bool getDebugValueBool(const char * propname, bool defVal) const
  3484. { return c->getDebugValueBool(propname, defVal); }
  3485. virtual IStringIterator & getDebugValues() const
  3486. { return c->getDebugValues(NULL); }
  3487. virtual IStringIterator & getDebugValues(const char *prop) const
  3488. { return c->getDebugValues(prop); }
  3489. virtual unsigned getExceptionCount() const
  3490. { return c->getExceptionCount(); }
  3491. virtual IConstWUExceptionIterator & getExceptions() const
  3492. { return c->getExceptions(); }
  3493. virtual unsigned getGraphCount() const
  3494. { return c->getGraphCount(); }
  3495. virtual unsigned getSourceFileCount() const
  3496. { return c->getSourceFileCount(); }
  3497. virtual unsigned getResultCount() const
  3498. { return c->getResultCount(); }
  3499. virtual unsigned getVariableCount() const
  3500. { return c->getVariableCount(); }
  3501. virtual unsigned getApplicationValueCount() const
  3502. { return c->getApplicationValueCount(); }
  3503. virtual IConstWUGraphIterator & getGraphs(WUGraphType type) const
  3504. { return c->getGraphs(type); }
  3505. virtual IConstWUGraphMetaIterator & getGraphsMeta(WUGraphType type) const
  3506. { return c->getGraphsMeta(type); }
  3507. virtual IConstWUGraph * getGraph(const char *name) const
  3508. { return c->getGraph(name); }
  3509. virtual IConstWUGraphProgress * getGraphProgress(const char * name) const
  3510. { return c->getGraphProgress(name); }
  3511. virtual const char *queryJobName() const
  3512. { return c->queryJobName(); }
  3513. virtual IConstWUPlugin * getPluginByName(const char * name) const
  3514. { return c->getPluginByName(name); }
  3515. virtual IConstWUPluginIterator & getPlugins() const
  3516. { return c->getPlugins(); }
  3517. virtual IConstWULibrary* getLibraryByName(const char *name) const
  3518. { return c->getLibraryByName(name); }
  3519. virtual IConstWULibraryIterator & getLibraries() const
  3520. { return c->getLibraries(); }
  3521. virtual WUPriorityClass getPriority() const
  3522. { return c->getPriority(); }
  3523. virtual const char *queryPriorityDesc() const
  3524. { return c->queryPriorityDesc(); }
  3525. virtual int getPriorityLevel() const
  3526. { return c->getPriorityLevel(); }
  3527. virtual int getPriorityValue() const
  3528. { return c->getPriorityValue(); }
  3529. virtual IConstWUQuery * getQuery() const
  3530. { return c->getQuery(); }
  3531. virtual IConstWUWebServicesInfo * getWebServicesInfo() const
  3532. { return c->getWebServicesInfo(); }
  3533. virtual bool getRescheduleFlag() const
  3534. { return c->getRescheduleFlag(); }
  3535. virtual IConstWUResult * getResultByName(const char * name) const
  3536. { return c->getResultByName(name); }
  3537. virtual IConstWUResult * getResultBySequence(unsigned seq) const
  3538. { return c->getResultBySequence(seq); }
  3539. virtual IConstWUResult * getQueryResultByName(const char * name) const
  3540. { return c->getQueryResultByName(name); }
  3541. virtual unsigned getResultLimit() const
  3542. { return c->getResultLimit(); }
  3543. virtual IConstWUResultIterator & getResults() const
  3544. { return c->getResults(); }
  3545. virtual IStringVal & getScope(IStringVal & str) const
  3546. { return c->getScope(str); }
  3547. virtual IStringVal & getWorkunitDistributedAccessToken(IStringVal & str) const
  3548. { return c->getWorkunitDistributedAccessToken(str); }
  3549. virtual WUState getState() const
  3550. { return c->getState(); }
  3551. virtual IStringVal & getStateEx(IStringVal & str) const
  3552. { return c->getStateEx(str); }
  3553. virtual __int64 getAgentSession() const
  3554. { return c->getAgentSession(); }
  3555. virtual unsigned getAgentPID() const
  3556. { return c->getAgentPID(); }
  3557. virtual const char *queryStateDesc() const
  3558. { return c->queryStateDesc(); }
  3559. virtual bool getRunningGraph(IStringVal & graphName, WUGraphIDType & subId) const
  3560. { return c->getRunningGraph(graphName, subId); }
  3561. virtual IConstWUScopeIterator & getScopeIterator(const WuScopeFilter & filter) const override
  3562. { return c->getScopeIterator(filter); }
  3563. virtual bool getStatistic(stat_type & value, const char * scope, StatisticKind kind) const override
  3564. { return c->getStatistic(value, scope, kind); }
  3565. virtual IStringVal & getSnapshot(IStringVal & str) const
  3566. { return c->getSnapshot(str); }
  3567. virtual const char *queryUser() const
  3568. { return c->queryUser(); }
  3569. virtual ErrorSeverity getWarningSeverity(unsigned code, ErrorSeverity defaultSeverity) const
  3570. { return c->getWarningSeverity(code, defaultSeverity); }
  3571. virtual const char *queryWuScope() const
  3572. { return c->queryWuScope(); }
  3573. virtual const char *queryWuid() const
  3574. { return c->queryWuid(); }
  3575. virtual IConstWUResult * getGlobalByName(const char * name) const
  3576. { return c->getGlobalByName(name); }
  3577. virtual IConstWUResult * getTemporaryByName(const char * name) const
  3578. { return c->getTemporaryByName(name); }
  3579. virtual IConstWUResultIterator & getTemporaries() const
  3580. { return c->getTemporaries(); }
  3581. virtual IConstWUResult * getVariableByName(const char * name) const
  3582. { return c->getVariableByName(name); }
  3583. virtual IConstWUResultIterator & getVariables() const
  3584. { return c->getVariables(); }
  3585. virtual bool isProtected() const
  3586. { return c->isProtected(); }
  3587. virtual bool isPausing() const
  3588. { return c->isPausing(); }
  3589. virtual IWorkUnit & lock()
  3590. { ((CInterface *)this)->Link(); return (IWorkUnit &) *this; }
  3591. virtual bool reload()
  3592. { UNIMPLEMENTED; }
  3593. virtual void subscribe(WUSubscribeOptions options)
  3594. { c->subscribe(options); }
  3595. virtual void requestAbort()
  3596. { c->requestAbort(); }
  3597. virtual unsigned calculateHash(unsigned prevHash)
  3598. { return queryExtendedWU(c)->calculateHash(prevHash); }
  3599. virtual void copyWorkUnit(IConstWorkUnit *cached, bool copyStats, bool all)
  3600. { queryExtendedWU(c)->copyWorkUnit(cached, copyStats, all); }
  3601. virtual IPropertyTree *queryPTree() const
  3602. { return queryExtendedWU(c)->queryPTree(); }
  3603. virtual IPropertyTree *getUnpackedTree(bool includeProgress) const
  3604. { return queryExtendedWU(c)->getUnpackedTree(includeProgress); }
  3605. virtual bool archiveWorkUnit(const char *base,bool del,bool deldll,bool deleteOwned,bool exportAssociatedFiles)
  3606. { return queryExtendedWU(c)->archiveWorkUnit(base,del,deldll,deleteOwned,exportAssociatedFiles); }
  3607. virtual unsigned queryFileUsage(const char *filename) const
  3608. { return c->queryFileUsage(filename); }
  3609. virtual IConstWUFileUsageIterator * getFieldUsage() const
  3610. { return c->getFieldUsage(); }
  3611. virtual bool getFieldUsageArray(StringArray & filenames, StringArray & columnnames, const char * clusterName) const
  3612. { return c->getFieldUsageArray(filenames, columnnames, clusterName); }
  3613. virtual IJlibDateTime & getTimeScheduled(IJlibDateTime &val) const
  3614. { return c->getTimeScheduled(val); }
  3615. virtual unsigned getDebugAgentListenerPort() const
  3616. { return c->getDebugAgentListenerPort(); }
  3617. virtual IStringVal & getDebugAgentListenerIP(IStringVal &ip) const
  3618. { return c->getDebugAgentListenerIP(ip); }
  3619. virtual IStringVal & getXmlParams(IStringVal & params, bool hidePasswords) const
  3620. { return c->getXmlParams(params, hidePasswords); }
  3621. virtual const IPropertyTree *getXmlParams() const
  3622. { return c->getXmlParams(); }
  3623. virtual unsigned __int64 getHash() const
  3624. { return c->getHash(); }
  3625. virtual IStringIterator *getLogs(const char *type, const char *instance) const
  3626. { return c->getLogs(type, instance); }
  3627. virtual IStringIterator *getProcesses(const char *type) const
  3628. { return c->getProcesses(type); }
  3629. virtual IPropertyTreeIterator* getProcesses(const char *type, const char *instance) const
  3630. { return c->getProcesses(type, instance); }
  3631. virtual unsigned getTotalThorTime() const
  3632. { return c->getTotalThorTime(); }
  3633. virtual WUGraphState queryGraphState(const char *graphName) const
  3634. { return c->queryGraphState(graphName); }
  3635. virtual WUGraphState queryNodeState(const char *graphName, WUGraphIDType nodeId) const
  3636. { return c->queryNodeState(graphName, nodeId); }
  3637. virtual void setGraphState(const char *graphName, unsigned wfid, WUGraphState state) const
  3638. { c->setGraphState(graphName, wfid, state); }
  3639. virtual void setNodeState(const char *graphName, WUGraphIDType nodeId, WUGraphState state) const
  3640. { c->setNodeState(graphName, nodeId, state); }
  3641. virtual IWUGraphStats *updateStats(const char *graphName, StatisticCreatorType creatorType, const char * creator, unsigned _wfid, unsigned subgraph) const override
  3642. { return c->updateStats(graphName, creatorType, creator, _wfid, subgraph); }
  3643. virtual void clearGraphProgress() const
  3644. { c->clearGraphProgress(); }
  3645. virtual IStringVal & getAbortBy(IStringVal & str) const
  3646. { return c->getAbortBy(str); }
  3647. virtual unsigned __int64 getAbortTimeStamp() const
  3648. { return c->getAbortTimeStamp(); }
  3649. virtual void clearExceptions(const char *source=nullptr)
  3650. { c->clearExceptions(source); }
  3651. virtual void commit()
  3652. { c->commit(); }
  3653. virtual IWUException * createException()
  3654. { return c->createException(); }
  3655. virtual void addProcess(const char *type, const char *instance, unsigned pid, const char *log)
  3656. { c->addProcess(type, instance, pid, log); }
  3657. virtual void protect(bool protectMode)
  3658. { c->protect(protectMode); }
  3659. virtual void setAction(WUAction action)
  3660. { c->setAction(action); }
  3661. virtual void setApplicationValue(const char * application, const char * propname, const char * value, bool overwrite)
  3662. { c->setApplicationValue(application, propname, value, overwrite); }
  3663. virtual void setApplicationValueInt(const char * application, const char * propname, int value, bool overwrite)
  3664. { c->setApplicationValueInt(application, propname, value, overwrite); }
  3665. virtual void incEventScheduledCount()
  3666. { c->incEventScheduledCount(); }
  3667. virtual void setIsQueryService(bool value)
  3668. { c->setIsQueryService(value); }
  3669. virtual void setCloneable(bool value)
  3670. { c->setCloneable(value); }
  3671. virtual void setIsClone(bool value)
  3672. { c->setIsClone(value); }
  3673. virtual void setClusterName(const char * value)
  3674. { c->setClusterName(value); }
  3675. virtual void setCodeVersion(unsigned version, const char * buildVersion, const char * eclVersion)
  3676. { c->setCodeVersion(version, buildVersion, eclVersion); }
  3677. virtual void setDebugValue(const char * propname, const char * value, bool overwrite)
  3678. { c->setDebugValue(propname, value, overwrite); }
  3679. virtual void setDebugValueInt(const char * propname, int value, bool overwrite)
  3680. { c->setDebugValueInt(propname, value, overwrite); }
  3681. virtual void setJobName(const char * value)
  3682. { c->setJobName(value); }
  3683. virtual void setPriority(WUPriorityClass cls)
  3684. { c->setPriority(cls); }
  3685. virtual void setPriorityLevel(int level)
  3686. { c->setPriorityLevel(level); }
  3687. virtual void setRescheduleFlag(bool value)
  3688. { c->setRescheduleFlag(value); }
  3689. virtual void setResultLimit(unsigned value)
  3690. { c->setResultLimit(value); }
  3691. virtual void setState(WUState state)
  3692. { c->setState(state); }
  3693. virtual void setStateEx(const char * text)
  3694. { c->setStateEx(text); }
  3695. virtual void setAgentSession(__int64 sessionId)
  3696. { c->setAgentSession(sessionId); }
  3697. virtual void setStatistic(StatisticCreatorType creatorType, const char * creator, StatisticScopeType scopeType, const char * scope, StatisticKind kind, const char * optDescription, unsigned __int64 value, unsigned __int64 count, unsigned __int64 maxValue, StatsMergeAction mergeAction)
  3698. { c->setStatistic(creatorType, creator, scopeType, scope, kind, optDescription, value, count, maxValue, mergeAction); }
  3699. virtual void setTracingValue(const char * propname, const char * value)
  3700. { c->setTracingValue(propname, value); }
  3701. virtual void setTracingValueInt(const char * propname, int value)
  3702. { c->setTracingValueInt(propname, value); }
  3703. virtual void setTracingValueInt64(const char * propname, __int64 value)
  3704. { c->setTracingValueInt64(propname, value); }
  3705. virtual void setUser(const char * value)
  3706. { c->setUser(value); }
  3707. virtual void setWuScope(const char * value)
  3708. { c->setWuScope(value); }
  3709. virtual IWorkflowItem* addWorkflowItem(unsigned wfid, WFType type, WFMode mode, unsigned success, unsigned failure, unsigned recovery, unsigned retriesAllowed, unsigned contingencyFor)
  3710. { return c->addWorkflowItem(wfid, type, mode, success, failure, recovery, retriesAllowed, contingencyFor); }
  3711. virtual void syncRuntimeWorkflow(IWorkflowItemArray * array)
  3712. { c->syncRuntimeWorkflow(array); }
  3713. virtual IWorkflowItemIterator * updateWorkflowItems()
  3714. { return c->updateWorkflowItems(); }
  3715. virtual void resetWorkflow()
  3716. { c->resetWorkflow(); }
  3717. virtual void schedule()
  3718. { c->schedule(); }
  3719. virtual void deschedule()
  3720. { c->deschedule(); }
  3721. virtual unsigned addLocalFileUpload(LocalFileUploadType type, char const * source, char const * destination, char const * eventTag)
  3722. { return c->addLocalFileUpload(type, source, destination, eventTag); }
  3723. virtual IWUResult * updateGlobalByName(const char * name)
  3724. { return c->updateGlobalByName(name); }
  3725. virtual void createGraph(const char * name, const char *label, WUGraphType type, IPropertyTree *xgmml, unsigned wfid)
  3726. { c->createGraph(name, label, type, xgmml, wfid); }
  3727. virtual IWUQuery * updateQuery()
  3728. { return c->updateQuery(); }
  3729. virtual IWUWebServicesInfo * updateWebServicesInfo(bool create)
  3730. { return c->updateWebServicesInfo(create); }
  3731. virtual IWUPlugin * updatePluginByName(const char * name)
  3732. { return c->updatePluginByName(name); }
  3733. virtual IWULibrary * updateLibraryByName(const char * name)
  3734. { return c->updateLibraryByName(name); }
  3735. virtual IWUResult * updateResultByName(const char * name)
  3736. { return c->updateResultByName(name); }
  3737. virtual IWUResult * updateResultBySequence(unsigned seq)
  3738. { return c->updateResultBySequence(seq); }
  3739. virtual IWUResult * updateTemporaryByName(const char * name)
  3740. { return c->updateTemporaryByName(name); }
  3741. virtual IWUResult * updateVariableByName(const char * name)
  3742. { return c->updateVariableByName(name); }
  3743. virtual void addFile(const char *fileName, StringArray *clusters, unsigned usageCount, WUFileKind fileKind, const char *graphOwner)
  3744. { c->addFile(fileName, clusters, usageCount, fileKind, graphOwner); }
  3745. virtual void noteFileRead(IDistributedFile *file)
  3746. { c->noteFileRead(file); }
  3747. virtual void noteFieldUsage(IPropertyTree * usage)
  3748. { c->noteFieldUsage(usage); }
  3749. virtual void releaseFile(const char *fileName)
  3750. { c->releaseFile(fileName); }
  3751. virtual void resetBeforeGeneration()
  3752. { c->resetBeforeGeneration(); }
  3753. virtual void deleteTempFiles(const char *graph, bool deleteOwned, bool deleteJobOwned)
  3754. { c->deleteTempFiles(graph, deleteOwned, deleteJobOwned); }
  3755. virtual void deleteTemporaries()
  3756. { c->deleteTemporaries(); }
  3757. virtual void addDiskUsageStats(__int64 avgNodeUsage, unsigned minNode, __int64 minNodeUsage, unsigned maxNode, __int64 maxNodeUsage, __int64 graphId)
  3758. { c->addDiskUsageStats(avgNodeUsage, minNode, minNodeUsage, maxNode, maxNodeUsage, graphId); }
  3759. virtual IPropertyTree * getDiskUsageStats()
  3760. { return c->getDiskUsageStats(); }
  3761. virtual IPropertyTreeIterator & getFileIterator() const
  3762. { return c->getFileIterator(); }
  3763. virtual IPropertyTreeIterator & getFilesReadIterator() const
  3764. { return c->getFilesReadIterator(); }
  3765. virtual void setSnapshot(const char * value)
  3766. { c->setSnapshot(value); }
  3767. virtual void setWarningSeverity(unsigned code, ErrorSeverity severity)
  3768. { c->setWarningSeverity(code, severity); }
  3769. virtual void setTimeScheduled(const IJlibDateTime &val)
  3770. { c->setTimeScheduled(val); }
  3771. virtual void setDebugAgentListenerPort(unsigned port)
  3772. { c->setDebugAgentListenerPort(port); }
  3773. virtual void setDebugAgentListenerIP(const char * ip)
  3774. { c->setDebugAgentListenerIP(ip); }
  3775. virtual void setXmlParams(const char *params)
  3776. { c->setXmlParams(params); }
  3777. virtual void setXmlParams(IPropertyTree *tree)
  3778. { c->setXmlParams(tree); }
  3779. virtual void setHash(unsigned __int64 hash)
  3780. { c->setHash(hash); }
  3781. // ILocalWorkUnit - used for debugging etc
  3782. virtual void serialize(MemoryBuffer &tgt)
  3783. { c->serialize(tgt); }
  3784. virtual void deserialize(MemoryBuffer &src)
  3785. { c->deserialize(src); }
  3786. virtual bool switchThorQueue(const char *cluster, IQueueSwitcher *qs)
  3787. { return c->switchThorQueue(cluster,qs); }
  3788. virtual void setAllowedClusters(const char *value)
  3789. { c->setAllowedClusters(value); }
  3790. virtual IStringVal& getAllowedClusters(IStringVal &str) const
  3791. { return c->getAllowedClusters(str); }
  3792. virtual void remoteCheckAccess(IUserDescriptor *user, bool writeaccess) const
  3793. { c->remoteCheckAccess(user,writeaccess); }
  3794. virtual void setAllowAutoQueueSwitch(bool val)
  3795. { c->setAllowAutoQueueSwitch(val); }
  3796. virtual bool getAllowAutoQueueSwitch() const
  3797. { return c->getAllowAutoQueueSwitch(); }
  3798. virtual void setLibraryInformation(const char * name, unsigned interfaceHash, unsigned definitionHash)
  3799. { c->setLibraryInformation(name, interfaceHash, definitionHash); }
  3800. virtual void setResultInt(const char * name, unsigned sequence, __int64 val)
  3801. { c->setResultInt(name, sequence, val); }
  3802. virtual void setResultUInt(const char * name, unsigned sequence, unsigned __int64 val)
  3803. { c->setResultUInt(name, sequence, val); }
  3804. virtual void setResultReal(const char *name, unsigned sequence, double val)
  3805. { c->setResultReal(name, sequence, val); }
  3806. virtual void setResultVarString(const char * stepname, unsigned sequence, const char *val)
  3807. { c->setResultVarString(stepname, sequence, val); }
  3808. virtual void setResultVarUnicode(const char * stepname, unsigned sequence, UChar const *val)
  3809. { c->setResultVarUnicode(stepname, sequence, val); }
  3810. virtual void setResultString(const char * stepname, unsigned sequence, int len, const char *val)
  3811. { c->setResultString(stepname, sequence, len, val); }
  3812. virtual void setResultData(const char * stepname, unsigned sequence, int len, const void *val)
  3813. { c->setResultData(stepname, sequence, len, val); }
  3814. virtual void setResultRaw(const char * name, unsigned sequence, int len, const void *val)
  3815. { c->setResultRaw(name, sequence, len, val); }
  3816. virtual void setResultSet(const char * name, unsigned sequence, bool isAll, size32_t len, const void *val, ISetToXmlTransformer *xform)
  3817. { c->setResultSet(name, sequence, isAll, len, val, xform); }
  3818. virtual void setResultUnicode(const char * name, unsigned sequence, int len, UChar const * val)
  3819. { c->setResultUnicode(name, sequence, len, val); }
  3820. virtual void setResultBool(const char *name, unsigned sequence, bool val)
  3821. { c->setResultBool(name, sequence, val); }
  3822. virtual void setResultDecimal(const char *name, unsigned sequence, int len, int precision, bool isSigned, const void *val)
  3823. { c->setResultDecimal(name, sequence, len, precision, isSigned, val); }
  3824. virtual void setResultDataset(const char * name, unsigned sequence, size32_t len, const void *val, unsigned numRows, bool extend)
  3825. { c->setResultDataset(name, sequence, len, val, numRows, extend); }
  3826. };
  3827. class CLocalWUAssociated : implements IConstWUAssociatedFile, public CInterface
  3828. {
  3829. Owned<IPropertyTree> p;
  3830. public:
  3831. IMPLEMENT_IINTERFACE;
  3832. CLocalWUAssociated(IPropertyTree *p);
  3833. virtual WUFileType getType() const;
  3834. virtual IStringVal & getDescription(IStringVal & ret) const;
  3835. virtual IStringVal & getIp(IStringVal & ret) const;
  3836. virtual IStringVal & getName(IStringVal & ret) const;
  3837. virtual IStringVal & getNameTail(IStringVal & ret) const;
  3838. virtual unsigned getCrc() const;
  3839. virtual unsigned getMinActivityId() const;
  3840. virtual unsigned getMaxActivityId() const;
  3841. };
  3842. class CLocalWUQuery : implements IWUQuery, public CInterface
  3843. {
  3844. Owned<IPropertyTree> p;
  3845. mutable IArrayOf<IConstWUAssociatedFile> associated;
  3846. mutable CriticalSection crit;
  3847. mutable bool associatedCached;
  3848. private:
  3849. void addSpecialCaseAssociated(WUFileType type, const char * propname, unsigned crc) const;
  3850. void loadAssociated() const;
  3851. public:
  3852. IMPLEMENT_IINTERFACE;
  3853. CLocalWUQuery(IPropertyTree *p);
  3854. virtual WUQueryType getQueryType() const;
  3855. virtual IStringVal& getQueryText(IStringVal &str) const;
  3856. virtual IStringVal& getQueryShortText(IStringVal &str) const;
  3857. virtual IStringVal& getQueryName(IStringVal &str) const;
  3858. virtual IStringVal & getQueryMainDefinition(IStringVal & str) const;
  3859. virtual IStringVal& getQueryDllName(IStringVal &str) const;
  3860. virtual unsigned getQueryDllCrc() const;
  3861. virtual IStringVal& getQueryCppName(IStringVal &str) const;
  3862. virtual IStringVal& getQueryResTxtName(IStringVal &str) const;
  3863. virtual IConstWUAssociatedFile * getAssociatedFile(WUFileType type, unsigned index) const;
  3864. virtual IConstWUAssociatedFileIterator& getAssociatedFiles() const;
  3865. virtual bool isArchive() const;
  3866. virtual bool hasArchive() const
  3867. {
  3868. return p->getPropBool("@hasArchive");
  3869. }
  3870. virtual void setQueryType(WUQueryType qt);
  3871. virtual void setQueryText(const char *pstr);
  3872. virtual void setQueryName(const char *);
  3873. virtual void setQueryMainDefinition(const char * str);
  3874. virtual void addAssociatedFile(WUFileType type, const char * name, const char * ip, const char * desc, unsigned crc, unsigned minActivity, unsigned maxActivity);
  3875. virtual void removeAssociatedFiles();
  3876. virtual void removeAssociatedFile(WUFileType type, const char * name, const char * desc);
  3877. };
  3878. class CLocalWUWebServicesInfo : implements IWUWebServicesInfo, public CInterface
  3879. {
  3880. Owned<IPropertyTree> p;
  3881. mutable CriticalSection crit;
  3882. private:
  3883. public:
  3884. IMPLEMENT_IINTERFACE;
  3885. CLocalWUWebServicesInfo(IPropertyTree *p);
  3886. virtual IStringVal& getModuleName(IStringVal &str) const;
  3887. virtual IStringVal& getAttributeName(IStringVal &str) const;
  3888. virtual IStringVal& getDefaultName(IStringVal &str) const;
  3889. virtual IStringVal& getInfo(const char *name, IStringVal &str) const;
  3890. virtual IStringVal& getText(const char *name, IStringVal &str) const;
  3891. virtual unsigned getWebServicesCRC() const;
  3892. virtual void setModuleName(const char *);
  3893. virtual void setAttributeName(const char *);
  3894. virtual void setDefaultName(const char *);
  3895. virtual void setInfo(const char *name, const char *info);
  3896. virtual void setText(const char *name, const char *info);
  3897. virtual void setWebServicesCRC(unsigned);
  3898. };
  3899. class CLocalWUResult : implements IWUResult, public CInterface
  3900. {
  3901. friend class CLocalWorkUnit;
  3902. mutable CriticalSection crit;
  3903. Owned<IPropertyTree> p;
  3904. Owned<IProperties> xmlns;
  3905. public:
  3906. IMPLEMENT_IINTERFACE;
  3907. CLocalWUResult(IPropertyTree *props);
  3908. ~CLocalWUResult() { try { p.clear(); } catch (IException *E) {E->Release();}}
  3909. virtual WUResultStatus getResultStatus() const;
  3910. virtual IStringVal& getResultName(IStringVal &str) const;
  3911. virtual int getResultSequence() const;
  3912. virtual bool isResultScalar() const;
  3913. virtual IStringVal& getResultXml(IStringVal &str, bool hidePasswords) const;
  3914. virtual unsigned getResultFetchSize() const;
  3915. virtual __int64 getResultTotalRowCount() const;
  3916. virtual __int64 getResultRowCount() const;
  3917. virtual void getResultDataset(IStringVal & ecl, IStringVal & defs) const;
  3918. virtual IStringVal& getResultLogicalName(IStringVal &ecl) const;
  3919. virtual IStringVal& getResultKeyField(IStringVal& ecl) const;
  3920. virtual unsigned getResultRequestedRows() const;
  3921. virtual __int64 getResultInt() const;
  3922. virtual bool getResultBool() const;
  3923. virtual double getResultReal() const;
  3924. virtual IStringVal& getResultString(IStringVal & str, bool hidePassword) const;
  3925. virtual IDataVal& getResultRaw(IDataVal & data, IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const;
  3926. virtual IDataVal& getResultUnicode(IDataVal & data) const;
  3927. virtual void getResultDecimal(void * val, unsigned length, unsigned precision, bool isSigned) const;
  3928. virtual IStringVal& getResultEclSchema(IStringVal & str) const;
  3929. virtual __int64 getResultRawSize(IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const;
  3930. virtual IDataVal& getResultRaw(IDataVal & data, __int64 from, __int64 length, IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const;
  3931. virtual IStringVal& getResultRecordSizeEntry(IStringVal & str) const;
  3932. virtual IStringVal& getResultTransformerEntry(IStringVal & str) const;
  3933. virtual __int64 getResultRowLimit() const;
  3934. virtual IStringVal& getResultFilename(IStringVal & str) const;
  3935. virtual WUResultFormat getResultFormat() const;
  3936. virtual unsigned getResultHash() const;
  3937. virtual bool getResultIsAll() const;
  3938. virtual IProperties *queryResultXmlns();
  3939. virtual IStringVal& getResultFieldOpt(const char *name, IStringVal &str) const;
  3940. virtual void getSchema(IArrayOf<ITypeInfo> &types, StringAttrArray &names, IStringVal * ecl=NULL) const;
  3941. virtual void getResultWriteLocation(IStringVal & _graph, unsigned & _activityId) const;
  3942. // interface IWUResult
  3943. virtual void setResultStatus(WUResultStatus status);
  3944. virtual void setResultName(const char *name);
  3945. virtual void setResultSequence(unsigned seq);
  3946. virtual void setResultSchemaRaw(unsigned len, const void *schema);
  3947. virtual void setResultScalar(bool isScalar);
  3948. virtual void setResultRaw(unsigned len, const void *xml, WUResultFormat format);
  3949. virtual void setResultFetchSize(unsigned rows); // 0 means file-loaded
  3950. virtual void setResultTotalRowCount(__int64 rows); // -1 means unknown
  3951. virtual void setResultRowCount(__int64 rows);
  3952. virtual void setResultDataset(const char *ecl, const char *defs);
  3953. virtual void setResultLogicalName(const char *logicalName);
  3954. virtual void setResultKeyField(const char * name);
  3955. virtual void setResultRequestedRows(unsigned req);
  3956. virtual void setResultRecordSizeEntry(const char * val);
  3957. virtual void setResultTransformerEntry(const char * val);
  3958. virtual void setResultInt(__int64 val);
  3959. virtual void setResultReal(double val);
  3960. virtual void setResultBool(bool val);
  3961. virtual void setResultString(const char * val, unsigned length);
  3962. virtual void setResultUnicode(const void * val, unsigned length);
  3963. virtual void setResultData(const void * val, unsigned length);
  3964. virtual void setResultDecimal(const void * val, unsigned length);
  3965. virtual void addResultRaw(unsigned len, const void * data, WUResultFormat format);
  3966. virtual void setResultRowLimit(__int64 value);
  3967. virtual void setResultFilename(const char * name);
  3968. virtual void setResultUInt(unsigned __int64 val);
  3969. virtual void setResultIsAll(bool value);
  3970. virtual void setResultFormat(WUResultFormat format);
  3971. virtual void setResultXML(const char *val);
  3972. virtual void setResultRow(unsigned len, const void * data);
  3973. virtual void setResultXmlns(const char *prefix, const char *uri);
  3974. virtual void setResultFieldOpt(const char *name, const char *value);
  3975. virtual void setResultWriteLocation(const char * _graph, unsigned _activityId);
  3976. virtual IPropertyTree *queryPTree() { return p; }
  3977. };
  3978. class CLocalWUPlugin : implements IWUPlugin, public CInterface
  3979. {
  3980. Owned<IPropertyTree> p;
  3981. public:
  3982. IMPLEMENT_IINTERFACE;
  3983. CLocalWUPlugin(IPropertyTree *p);
  3984. virtual IStringVal& getPluginName(IStringVal &str) const;
  3985. virtual IStringVal& getPluginVersion(IStringVal &str) const;
  3986. virtual void setPluginName(const char *str);
  3987. virtual void setPluginVersion(const char *str);
  3988. };
  3989. class CLocalWULibrary : implements IWULibrary, public CInterface
  3990. {
  3991. Owned<IPropertyTree> p;
  3992. public:
  3993. IMPLEMENT_IINTERFACE;
  3994. CLocalWULibrary(IPropertyTree *p);
  3995. virtual IStringVal & getName(IStringVal & str) const;
  3996. virtual void setName(const char * str);
  3997. };
  3998. class CLocalWUException : implements IWUException, public CInterface
  3999. {
  4000. Owned<IPropertyTree> p;
  4001. public:
  4002. IMPLEMENT_IINTERFACE;
  4003. CLocalWUException(IPropertyTree *p);
  4004. virtual IStringVal& getExceptionSource(IStringVal &str) const override;
  4005. virtual IStringVal& getExceptionMessage(IStringVal &str) const override;
  4006. virtual unsigned getExceptionCode() const override;
  4007. virtual ErrorSeverity getSeverity() const override;
  4008. virtual IStringVal & getTimeStamp(IStringVal & dt) const override;
  4009. virtual IStringVal & getExceptionFileName(IStringVal & str) const override;
  4010. virtual unsigned getExceptionLineNo() const override;
  4011. virtual unsigned getExceptionColumn() const override;
  4012. virtual unsigned getActivityId() const override;
  4013. virtual unsigned getSequence() const override;
  4014. virtual const char * queryScope() const override;
  4015. virtual unsigned getPriority() const override;
  4016. virtual void setExceptionSource(const char *str) override;
  4017. virtual void setExceptionMessage(const char *str) override;
  4018. virtual void setExceptionCode(unsigned code) override;
  4019. virtual void setSeverity(ErrorSeverity level) override;
  4020. virtual void setTimeStamp(const char * dt) override;
  4021. virtual void setExceptionFileName(const char *str) override;
  4022. virtual void setExceptionLineNo(unsigned r) override;
  4023. virtual void setExceptionColumn(unsigned c) override;
  4024. virtual void setActivityId(unsigned _id) override;
  4025. virtual void setScope(const char * _scope) override;
  4026. virtual void setPriority(unsigned _priority) override;
  4027. };
  4028. //==========================================================================================
  4029. extern WORKUNIT_API bool isSpecialResultSequence(unsigned sequence)
  4030. {
  4031. switch ((int) sequence)
  4032. {
  4033. case ResultSequenceInternal:
  4034. case ResultSequenceOnce:
  4035. case ResultSequencePersist:
  4036. case ResultSequenceStored:
  4037. return true;
  4038. default:
  4039. assertex(sequence <= INT_MAX);
  4040. if ((int) sequence >= LibraryBaseSequence)
  4041. return true;
  4042. return false;
  4043. }
  4044. }
  4045. class CConstWUArrayIterator : implements IConstWorkUnitIterator, public CInterface
  4046. {
  4047. unsigned curTreeNum;
  4048. IArrayOf<IPropertyTree> trees;
  4049. Owned<IConstWorkUnitInfo> cur;
  4050. void setCurrent()
  4051. {
  4052. cur.setown(new CLightweightWorkunitInfo(trees.item(curTreeNum)));
  4053. }
  4054. public:
  4055. IMPLEMENT_IINTERFACE;
  4056. CConstWUArrayIterator(IArrayOf<IPropertyTree> &_trees)
  4057. {
  4058. ForEachItemIn(t, _trees)
  4059. trees.append(*LINK(&_trees.item(t)));
  4060. curTreeNum = 0;
  4061. }
  4062. bool first()
  4063. {
  4064. curTreeNum = 0;
  4065. return next();
  4066. }
  4067. bool isValid()
  4068. {
  4069. return (NULL != cur.get());
  4070. }
  4071. bool next()
  4072. {
  4073. if (curTreeNum >= trees.ordinality())
  4074. {
  4075. cur.clear();
  4076. return false;
  4077. }
  4078. setCurrent();
  4079. ++curTreeNum;
  4080. return true;
  4081. }
  4082. IConstWorkUnitInfo & query() { return *cur; }
  4083. };
  4084. class CLocalWUFieldUsage : public CInterface, implements IConstWUFieldUsage
  4085. {
  4086. Owned<IPropertyTree> p;
  4087. public:
  4088. IMPLEMENT_IINTERFACE;
  4089. CLocalWUFieldUsage(IPropertyTree& _p) { p.setown(&_p); }
  4090. virtual const char * queryName() const { return p->queryProp("@name"); }
  4091. };
  4092. class CConstWUFieldUsageIterator : public CInterface, implements IConstWUFieldUsageIterator
  4093. {
  4094. public:
  4095. IMPLEMENT_IINTERFACE;
  4096. CConstWUFieldUsageIterator(IPropertyTreeIterator * tree) { iter.setown(tree); }
  4097. bool first() override { return iter->first(); }
  4098. bool isValid() override { return iter->isValid(); }
  4099. bool next() override { return iter->next(); }
  4100. IConstWUFieldUsage * get() const override { return new CLocalWUFieldUsage(iter->get()); }
  4101. private:
  4102. Owned<IPropertyTreeIterator> iter;
  4103. };
  4104. class CLocalWUFileUsage : public CInterface, implements IConstWUFileUsage
  4105. {
  4106. Owned<IPropertyTree> p;
  4107. public:
  4108. IMPLEMENT_IINTERFACE;
  4109. CLocalWUFileUsage(IPropertyTree& _p) { p.setown(&_p); }
  4110. virtual const char * queryName() const { return p->queryProp("@name"); }
  4111. virtual const char * queryType() const { return p->queryProp("@type"); }
  4112. virtual unsigned getNumFields() const { return p->getPropInt("@numFields"); }
  4113. virtual unsigned getNumFieldsUsed() const { return p->getPropInt("@numFieldsUsed"); }
  4114. virtual IConstWUFieldUsageIterator * getFields() const { return new CConstWUFieldUsageIterator(p->getElements("fields/field")); }
  4115. };
  4116. class CConstWUFileUsageIterator : public CInterface, implements IConstWUFileUsageIterator
  4117. {
  4118. public:
  4119. IMPLEMENT_IINTERFACE;
  4120. CConstWUFileUsageIterator(IPropertyTreeIterator * tree) { iter.setown(tree); }
  4121. bool first() override { return iter->first(); }
  4122. bool isValid() override { return iter->isValid(); }
  4123. bool next() override { return iter->next(); }
  4124. IConstWUFileUsage * get() const override { return new CLocalWUFileUsage(iter->get()); }
  4125. private:
  4126. Owned<IPropertyTreeIterator> iter;
  4127. };
  4128. //==========================================================================================
  4129. class CStringArrayIterator : implements IStringIterator, public CInterface
  4130. {
  4131. unsigned idx;
  4132. StringArray strings;
  4133. public:
  4134. IMPLEMENT_IINTERFACE;
  4135. CStringArrayIterator() { idx = 0; };
  4136. void append(const char *str) { strings.append(str); }
  4137. virtual bool first() { idx = 0; return strings.isItem(idx); }
  4138. virtual bool next() { idx ++; return strings.isItem(idx); }
  4139. virtual bool isValid() { return strings.isItem(idx); }
  4140. virtual IStringVal & str(IStringVal &s) { s.set(strings.item(idx)); return s; }
  4141. };
  4142. class CCachedJobNameIterator : implements IStringIterator, public CInterface
  4143. {
  4144. Owned<IPropertyTreeIterator> it;
  4145. public:
  4146. IMPLEMENT_IINTERFACE;
  4147. CCachedJobNameIterator(IPropertyTreeIterator *p) : it(p) {};
  4148. virtual bool first() { return it->first(); }
  4149. virtual bool next() { return it->next(); }
  4150. virtual bool isValid() { return it->isValid(); }
  4151. virtual IStringVal & str(IStringVal &s) { s.set(it->query().queryName()+1); return s; }
  4152. };
  4153. class CEmptyStringIterator : implements IStringIterator, public CInterface
  4154. {
  4155. public:
  4156. IMPLEMENT_IINTERFACE;
  4157. virtual bool first() { return false; }
  4158. virtual bool next() { return false; }
  4159. virtual bool isValid() { return false; }
  4160. virtual IStringVal & str(IStringVal &s) { s.clear(); return s; }
  4161. };
  4162. EnumMapping workunitSortFields[] =
  4163. {
  4164. { WUSFuser, "@submitID" },
  4165. { WUSFcluster, "@clusterName" },
  4166. { WUSFjob, "@jobName" },
  4167. { WUSFstate, "@state" },
  4168. { WUSFpriority, "@priorityClass" },
  4169. { WUSFprotected, "@protected" },
  4170. { WUSFwuid, "@" },
  4171. { WUSFecl, "Query/ShortText" },
  4172. { WUSFfileread, "FilesRead/File/@name" },
  4173. { WUSFtotalthortime, "@totalThorTime|"
  4174. "Statistics/Statistic[@c='summary'][@creator='thor'][@kind='TimeElapsed']/@value|"
  4175. "Statistics/Statistic[@c='summary'][@creator='hthor'][@kind='TimeElapsed']/@value|"
  4176. "Statistics/Statistic[@c='summary'][@creator='roxie'][@kind='TimeElapsed']/@value|"
  4177. "Statistics/Statistic[@desc='Total thor time']/@value|"
  4178. "Timings/Timing[@name='Total thor time']/@duration" //Use Statistics first. If not found, use Timings
  4179. },
  4180. { WUSFwuidhigh, "@" },
  4181. { WUSFwildwuid, "@" },
  4182. { WUSFappvalue, "Application" },
  4183. { WUSFfilewritten, "Files/File/@name" },
  4184. { WUSFterm, NULL }
  4185. };
  4186. extern const char *queryFilterXPath(WUSortField field)
  4187. {
  4188. return getEnumText(field, workunitSortFields);
  4189. }
  4190. EnumMapping querySortFields[] =
  4191. {
  4192. { WUQSFId, "@id" },
  4193. { WUQSFwuid, "@wuid" },
  4194. { WUQSFname, "@name" },
  4195. { WUQSFdll, "@dll" },
  4196. { WUQSFmemoryLimit, "@memoryLimit" },
  4197. { WUQSFmemoryLimitHi, "@memoryLimit" },
  4198. { WUQSFtimeLimit, "@timeLimit" },
  4199. { WUQSFtimeLimitHi, "@timeLimit" },
  4200. { WUQSFwarnTimeLimit, "@warnTimeLimit" },
  4201. { WUQSFwarnTimeLimitHi, "@warnTimeLimit" },
  4202. { WUQSFpriority, "@priority" },
  4203. { WUQSFpriorityHi, "@priority" },
  4204. { WUQSFQuerySet, "@querySetId" },
  4205. { WUQSFActivited, "@activated" },
  4206. { WUQSFSuspendedByUser, "@suspended" },
  4207. { WUQSFLibrary, "Library"},
  4208. { WUQSFPublishedBy, "@publishedBy" },
  4209. { WUQSFterm, NULL }
  4210. };
  4211. class asyncRemoveDllWorkItem: public CInterface, implements IWorkQueueItem // class only used in asyncRemoveDll
  4212. {
  4213. StringAttr name;
  4214. public:
  4215. IMPLEMENT_IINTERFACE;
  4216. asyncRemoveDllWorkItem(const char * _name) : name(_name)
  4217. {
  4218. }
  4219. void execute()
  4220. {
  4221. PROGLOG("WU removeDll %s", name.get());
  4222. queryDllServer().removeDll(name, true, true); // <name>, removeDlls=true, removeDirectory=true
  4223. }
  4224. };
  4225. class asyncRemoveRemoteFileWorkItem: public CInterface, implements IWorkQueueItem // class only used in asyncRemoveFile
  4226. {
  4227. RemoteFilename name;
  4228. public:
  4229. IMPLEMENT_IINTERFACE;
  4230. asyncRemoveRemoteFileWorkItem(const char * _ip, const char * _name)
  4231. {
  4232. SocketEndpoint ep(_ip);
  4233. name.setPath(ep, _name);
  4234. }
  4235. void execute()
  4236. {
  4237. Owned<IFile> file = createIFile(name);
  4238. PROGLOG("WU removeDll %s",file->queryFilename());
  4239. file->remove();
  4240. }
  4241. };
  4242. //==========================================================================================
  4243. class CConstQuerySetQueryIterator : implements IConstQuerySetQueryIterator, public CInterface
  4244. {
  4245. unsigned index;
  4246. IArrayOf<IPropertyTree> trees;
  4247. public:
  4248. IMPLEMENT_IINTERFACE;
  4249. CConstQuerySetQueryIterator(IArrayOf<IPropertyTree> &_trees)
  4250. {
  4251. ForEachItemIn(t, _trees)
  4252. trees.append(*LINK(&_trees.item(t)));
  4253. index = 0;
  4254. }
  4255. ~CConstQuerySetQueryIterator()
  4256. {
  4257. trees.kill();
  4258. }
  4259. bool first()
  4260. {
  4261. index = 0;
  4262. return (trees.ordinality()!=0);
  4263. }
  4264. bool next()
  4265. {
  4266. index++;
  4267. return (index<trees.ordinality());
  4268. }
  4269. bool isValid()
  4270. {
  4271. return (index<trees.ordinality());
  4272. }
  4273. IPropertyTree &query()
  4274. {
  4275. return trees.item(index);
  4276. }
  4277. };
  4278. class CSecurityCache
  4279. {
  4280. };
  4281. class CConstWUIterator : implements IConstWorkUnitIterator, public CInterface
  4282. {
  4283. public:
  4284. IMPLEMENT_IINTERFACE;
  4285. CConstWUIterator(IPropertyTreeIterator *_ptreeIter)
  4286. : ptreeIter(_ptreeIter)
  4287. {
  4288. }
  4289. bool first()
  4290. {
  4291. if (!ptreeIter->first())
  4292. {
  4293. cur.clear();
  4294. return false;
  4295. }
  4296. cur.setown(new CLightweightWorkunitInfo(ptreeIter->query()));
  4297. return true;
  4298. }
  4299. bool isValid()
  4300. {
  4301. return (NULL != cur.get());
  4302. }
  4303. bool next()
  4304. {
  4305. if (!ptreeIter->next())
  4306. {
  4307. cur.clear();
  4308. return false;
  4309. }
  4310. cur.setown(new CLightweightWorkunitInfo(ptreeIter->query()));
  4311. return true;
  4312. }
  4313. IConstWorkUnitInfo & query() { return *cur; }
  4314. private:
  4315. Owned<IConstWorkUnitInfo> cur;
  4316. Owned<IPropertyTreeIterator> ptreeIter;
  4317. };
  4318. class CSecureConstWUIterator : public CInterfaceOf<IConstWorkUnitIterator>
  4319. {
  4320. public:
  4321. CSecureConstWUIterator(IConstWorkUnitIterator *_parent, ISecManager *_secmgr=NULL, ISecUser *_secuser=NULL)
  4322. : parent(_parent), secmgr(_secmgr), secuser(_secuser)
  4323. {
  4324. assertex(_secuser && _secmgr);
  4325. }
  4326. bool first()
  4327. {
  4328. if (!parent->first())
  4329. return false;
  4330. return getNext();
  4331. }
  4332. bool next()
  4333. {
  4334. if (!parent->next())
  4335. return false;
  4336. return getNext();
  4337. }
  4338. virtual bool isValid()
  4339. {
  4340. return parent->isValid();
  4341. }
  4342. virtual IConstWorkUnitInfo &query()
  4343. {
  4344. return parent->query();
  4345. }
  4346. private:
  4347. Owned<IConstWorkUnitIterator> parent;
  4348. MapStringTo<int> scopePermissions;
  4349. Linked<ISecManager> secmgr;
  4350. Linked<ISecUser> secuser;
  4351. bool getNext() // scan for a workunit with permissions
  4352. {
  4353. do
  4354. {
  4355. const char *scopeName = parent->query().queryWuScope();
  4356. if (!scopeName || !*scopeName || checkScope(scopeName))
  4357. return true;
  4358. } while (parent->next());
  4359. return false;
  4360. }
  4361. bool checkScope(const char *scopeName)
  4362. {
  4363. int *perms = scopePermissions.getValue(scopeName);
  4364. SecAccessFlags perm;
  4365. if (!perms)
  4366. {
  4367. perm = secuser.get() ? secmgr->authorizeWorkunitScope(*secuser, scopeName) : SecAccess_Unavailable;
  4368. scopePermissions.setValue(scopeName, perm);
  4369. }
  4370. else
  4371. perm = (SecAccessFlags)*perms;
  4372. return perm >= SecAccess_Read;
  4373. }
  4374. };
  4375. CWorkUnitFactory::CWorkUnitFactory()
  4376. {
  4377. }
  4378. CWorkUnitFactory::~CWorkUnitFactory()
  4379. {
  4380. }
  4381. IWorkUnit* CWorkUnitFactory::createNamedWorkUnit(const char *wuid, const char *app, const char *scope, ISecManager *secmgr, ISecUser *secuser)
  4382. {
  4383. checkWuScopeSecAccess(scope, secmgr, secuser, SecAccess_Write, "Create", true, true);
  4384. Owned<CLocalWorkUnit> cw = _createWorkUnit(wuid, secmgr, secuser);
  4385. if (scope)
  4386. cw->setWuScope(scope); // Note - this may check access rights and throw exception. Is that correct? We might prefer to only check access once, and this will check on the lock too...
  4387. cw->setDistributedAccessToken(secuser ? secuser->getName() : "");//create and sign the workunit distributed access token
  4388. IWorkUnit* ret = &cw->lockRemote(false); // Note - this may throw exception if user does not have rights.
  4389. ret->setDebugValue("CREATED_BY", app, true);
  4390. ret->setDebugValue("CREATED_FOR", scope, true);
  4391. return ret;
  4392. }
  4393. IWorkUnit* CWorkUnitFactory::createWorkUnit(const char *app, const char *scope, ISecManager *secmgr, ISecUser *secuser)
  4394. {
  4395. StringBuffer wuid("W");
  4396. char result[32];
  4397. time_t ltime;
  4398. time( &ltime );
  4399. tm *today = localtime( &ltime ); // MORE - this is not threadsafe. But I probably don't care that much!
  4400. strftime(result, sizeof(result), "%Y%m%d-%H%M%S", today);
  4401. wuid.append(result);
  4402. if (workUnitTraceLevel > 1)
  4403. DBGLOG("createWorkUnit created %s", wuid.str());
  4404. IWorkUnit* ret = createNamedWorkUnit(wuid.str(), app, scope, secmgr, secuser);
  4405. if (workUnitTraceLevel > 1)
  4406. DBGLOG("createWorkUnit created %s", ret->queryWuid());
  4407. addTimeStamp(ret, SSTglobal, NULL, StWhenCreated);
  4408. return ret;
  4409. }
  4410. bool CWorkUnitFactory::deleteWorkUnitEx(const char * wuid, bool throwException, ISecManager *secmgr, ISecUser *secuser)
  4411. {
  4412. if (workUnitTraceLevel > 1)
  4413. DBGLOG("deleteWorkUnit %s", wuid);
  4414. StringBuffer wuRoot;
  4415. getXPath(wuRoot, wuid);
  4416. Owned<CLocalWorkUnit> cw = _updateWorkUnit(wuid, secmgr, secuser);
  4417. if (!checkWuSecAccess(*cw.get(), secmgr, secuser, SecAccess_Full, "delete", true, true))
  4418. return false;
  4419. if (throwException)
  4420. cw->cleanupAndDelete(true, true);
  4421. else
  4422. {
  4423. try
  4424. {
  4425. cw->cleanupAndDelete(true, true);
  4426. }
  4427. catch (IException *E)
  4428. {
  4429. StringBuffer s;
  4430. LOG(MCexception(E, MSGCLS_warning), E, s.append("Exception during deleteWorkUnit: ").append(wuid).str());
  4431. E->Release();
  4432. return false;
  4433. }
  4434. }
  4435. removeWorkUnitFromAllQueues(wuid); //known active workunits wouldn't make it this far
  4436. return true;
  4437. }
  4438. bool CWorkUnitFactory::deleteWorkUnit(const char * wuid, ISecManager *secmgr, ISecUser *secuser)
  4439. {
  4440. return deleteWorkUnitEx(wuid, false, secmgr, secuser);
  4441. }
  4442. IConstWorkUnit* CWorkUnitFactory::openWorkUnit(const char *wuid, ISecManager *secmgr, ISecUser *secuser)
  4443. {
  4444. StringBuffer wuidStr(wuid);
  4445. wuidStr.trim();
  4446. if (wuidStr.length() && ('w' == wuidStr.charAt(0)))
  4447. wuidStr.setCharAt(0, 'W');
  4448. if (!wuidStr.length() || ('W' != wuidStr.charAt(0)))
  4449. {
  4450. if (workUnitTraceLevel > 1)
  4451. DBGLOG("openWorkUnit %s invalid WUID", nullText(wuidStr.str()));
  4452. return NULL;
  4453. }
  4454. if (workUnitTraceLevel > 1)
  4455. DBGLOG("openWorkUnit %s", wuidStr.str());
  4456. Owned<IConstWorkUnit> wu = _openWorkUnit(wuid, secmgr, secuser);
  4457. if (wu)
  4458. {
  4459. if (!checkWuSecAccess(*wu, secmgr, secuser, SecAccess_Read, "opening", true, true))
  4460. return NULL; // Actually throws exception on failure, so won't reach here
  4461. return wu.getClear();
  4462. }
  4463. else
  4464. {
  4465. if (workUnitTraceLevel > 0)
  4466. IERRLOG("openWorkUnit %s not found", wuidStr.str());
  4467. return NULL;
  4468. }
  4469. }
  4470. IWorkUnit* CWorkUnitFactory::updateWorkUnit(const char *wuid, ISecManager *secmgr, ISecUser *secuser)
  4471. {
  4472. if (workUnitTraceLevel > 1)
  4473. DBGLOG("updateWorkUnit %s", wuid);
  4474. Owned<CLocalWorkUnit> wu = _updateWorkUnit(wuid, secmgr, secuser);
  4475. if (wu)
  4476. {
  4477. if (!checkWuSecAccess(*wu.get(), secmgr, secuser, SecAccess_Write, "updating", true, true))
  4478. return NULL;
  4479. return &wu->lockRemote(false);
  4480. }
  4481. else
  4482. {
  4483. if (workUnitTraceLevel > 0)
  4484. IERRLOG("updateWorkUnit %s not found", wuid);
  4485. return NULL;
  4486. }
  4487. }
  4488. IPropertyTree * pruneBranch(IPropertyTree * from, char const * xpath)
  4489. {
  4490. Owned<IPropertyTree> ret;
  4491. IPropertyTree * branch = from->queryPropTree(xpath);
  4492. if(branch)
  4493. {
  4494. ret.setown(createPTreeFromIPT(branch));
  4495. from->removeTree(branch);
  4496. }
  4497. return ret.getClear();
  4498. }
  4499. bool CWorkUnitFactory::restoreWorkUnit(const char *base, const char *wuid, bool restoreAssociated)
  4500. {
  4501. StringBuffer path(base);
  4502. addPathSepChar(path).append(wuid).append(".xml");
  4503. Owned<IPTree> pt = createPTreeFromXMLFile(path);
  4504. if (!pt)
  4505. return false;
  4506. CDateTime dt;
  4507. dt.setNow();
  4508. StringBuffer dts;
  4509. dt.getString(dts);
  4510. pt->setProp("@restoredDate", dts.str());
  4511. Owned<IPropertyTree> generatedDlls = pruneBranch(pt, "GeneratedDlls[1]");
  4512. Owned<IPropertyTree> associatedFiles;
  4513. IPropertyTree *srcAssociated = pt->queryPropTree("Query/Associated");
  4514. if (srcAssociated)
  4515. associatedFiles.setown(createPTreeFromIPT(srcAssociated));
  4516. // The updating of the repo is implementation specific...
  4517. if (!_restoreWorkUnit(pt.getClear(), wuid))
  4518. return false;
  4519. // now kludge back GeneratedDlls
  4520. if (generatedDlls)
  4521. {
  4522. Owned<IPropertyTreeIterator> dlls = generatedDlls->getElements("GeneratedDll");
  4523. for(dlls->first(); dlls->isValid(); dlls->next())
  4524. {
  4525. IPropertyTree & dll = dlls->query();
  4526. char const * name = dll.queryProp("@name");
  4527. char const * kind = dll.queryProp("@kind");
  4528. char const * location = dll.queryProp("@location");
  4529. Owned<IDllEntry> got = queryDllServer().getEntry(name);
  4530. if (!got)
  4531. {
  4532. RemoteFilename dstRfn;
  4533. dstRfn.setRemotePath(location);
  4534. StringBuffer srcPath(base);
  4535. addPathSepChar(srcPath);
  4536. dstRfn.getTail(srcPath);
  4537. OwnedIFile srcFile = createIFile(srcPath);
  4538. OwnedIFile dstFile = createIFile(dstRfn);
  4539. copyFile(dstFile, srcFile);
  4540. queryDllServer().registerDll(name, kind, location);
  4541. }
  4542. }
  4543. }
  4544. if (associatedFiles)
  4545. {
  4546. Owned<IPropertyTreeIterator> associated = associatedFiles->getElements("*");
  4547. ForEach(*associated)
  4548. {
  4549. IPropertyTree &file = associated->query();
  4550. const char *filename = file.queryProp("@filename");
  4551. SocketEndpoint ep(file.queryProp("@ip"));
  4552. RemoteFilename rfn;
  4553. rfn.setPath(ep, filename);
  4554. OwnedIFile dstFile = createIFile(rfn);
  4555. StringBuffer srcPath(base), name;
  4556. addPathSepChar(srcPath);
  4557. rfn.getTail(name);
  4558. srcPath.append(name);
  4559. if (generatedDlls)
  4560. {
  4561. VStringBuffer gDllPath("GeneratedDll[@name=\"%s\"]", name.str());
  4562. if (generatedDlls->hasProp(gDllPath))
  4563. continue; // generated dlls handled separately - see above
  4564. }
  4565. OwnedIFile srcFile = createIFile(srcPath);
  4566. if (srcFile->exists())
  4567. {
  4568. try
  4569. {
  4570. copyFile(dstFile, srcFile);
  4571. }
  4572. catch (IException *e)
  4573. {
  4574. VStringBuffer msg("Failed to restore associated file '%s' to destination '%s'", srcFile->queryFilename(), dstFile->queryFilename());
  4575. EXCLOG(e, msg.str());
  4576. e->Release();
  4577. }
  4578. }
  4579. }
  4580. }
  4581. return true;
  4582. }
  4583. int CWorkUnitFactory::setTracingLevel(int newLevel)
  4584. {
  4585. if (newLevel)
  4586. DBGLOG("Setting workunit trace level to %d", newLevel);
  4587. int level = workUnitTraceLevel;
  4588. workUnitTraceLevel = newLevel;
  4589. return level;
  4590. }
  4591. void CWorkUnitFactory::descheduleAllWorkUnits(ISecManager *secmgr, ISecUser *secuser)
  4592. {
  4593. Owned<IRemoteConnection> conn = querySDS().connect("/Schedule", myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
  4594. if(!conn) return;
  4595. Owned<IPropertyTree> root(conn->queryRoot()->getBranch("."));
  4596. KeptAtomTable entries;
  4597. Owned<IPropertyTreeIterator> iter(root->getElements("*/*/*/*"));
  4598. StringBuffer wuid;
  4599. for(iter->first(); iter->isValid(); iter->next())
  4600. {
  4601. char const * entry = iter->query().queryName();
  4602. if(!entries.find(entry))
  4603. {
  4604. entries.addAtom(entry);
  4605. ncnameUnescape(entry, wuid.clear());
  4606. Owned<IWorkUnit> wu = updateWorkUnit(wuid, secmgr, secuser);
  4607. if(wu && (wu->getState() == WUStateWait))
  4608. wu->setState(WUStateCompleted);
  4609. }
  4610. }
  4611. bool more;
  4612. do more = root->removeProp("*"); while(more);
  4613. }
  4614. IConstQuerySetQueryIterator* CWorkUnitFactory::getQuerySetQueriesSorted( WUQuerySortField *sortorder, // list of fields to sort by (terminated by WUSFterm)
  4615. WUQuerySortField *filters, // NULL or list of fields to filter on (terminated by WUSFterm)
  4616. const void *filterbuf, // (appended) string values for filters
  4617. unsigned startoffset,
  4618. unsigned maxnum,
  4619. __int64 *cachehint,
  4620. unsigned *total,
  4621. const MapStringTo<bool> *_subset)
  4622. {
  4623. struct PostFilters
  4624. {
  4625. WUQueryFilterBoolean activatedFilter;
  4626. WUQueryFilterBoolean suspendedByUserFilter;
  4627. PostFilters()
  4628. {
  4629. activatedFilter = WUQFSAll;
  4630. suspendedByUserFilter = WUQFSAll;
  4631. };
  4632. } postFilters;
  4633. class CQuerySetQueriesPager : public CSimpleInterface, implements IElementsPager
  4634. {
  4635. StringAttr querySet;
  4636. StringAttr xPath;
  4637. StringAttr sortOrder;
  4638. PostFilters postFilters;
  4639. StringArray unknownAttributes;
  4640. const MapStringTo<bool> *subset;
  4641. void populateQueryTree(const IPropertyTree* querySetTree, IPropertyTree* queryTree)
  4642. {
  4643. const char* querySetId = querySetTree->queryProp("@id");
  4644. VStringBuffer path("Query%s", xPath.get());
  4645. Owned<IPropertyTreeIterator> iter = querySetTree->getElements(path.str());
  4646. ForEach(*iter)
  4647. {
  4648. IPropertyTree &query = iter->query();
  4649. bool activated = false;
  4650. const char* queryId = query.queryProp("@id");
  4651. if (queryId && *queryId)
  4652. {
  4653. if (subset)
  4654. {
  4655. VStringBuffer match("%s/%s", querySetId, queryId);
  4656. if (!subset->getValue(match))
  4657. continue;
  4658. }
  4659. VStringBuffer aliasXPath("Alias[@id='%s']", queryId);
  4660. IPropertyTree *alias = querySetTree->queryPropTree(aliasXPath.str());
  4661. if (alias)
  4662. activated = true;
  4663. }
  4664. if (activated && (postFilters.activatedFilter == WUQFSNo))
  4665. continue;
  4666. if (!activated && (postFilters.activatedFilter == WUQFSYes))
  4667. continue;
  4668. if ((postFilters.suspendedByUserFilter == WUQFSNo) && query.hasProp(getEnumText(WUQSFSuspendedByUser,querySortFields)))
  4669. continue;
  4670. if ((postFilters.suspendedByUserFilter == WUQFSYes) && !query.hasProp(getEnumText(WUQSFSuspendedByUser,querySortFields)))
  4671. continue;
  4672. IPropertyTree *queryWithSetId = queryTree->addPropTree("Query", createPTreeFromIPT(&query));
  4673. queryWithSetId->setProp("@querySetId", querySetId);
  4674. queryWithSetId->setPropBool("@activated", activated);
  4675. }
  4676. }
  4677. IRemoteConnection* populateQueryTree(IPropertyTree* queryTree)
  4678. {
  4679. StringBuffer querySetXPath("QuerySets");
  4680. if (!querySet.isEmpty())
  4681. querySetXPath.appendf("/QuerySet[@id=\"%s\"]", querySet.get());
  4682. Owned<IRemoteConnection> conn = querySDS().connect(querySetXPath.str(), myProcessSession(), 0, SDS_LOCK_TIMEOUT);
  4683. if (!conn)
  4684. return NULL;
  4685. if (querySet.isEmpty())
  4686. {
  4687. Owned<IPropertyTreeIterator> querySetIter = conn->queryRoot()->getElements("*");
  4688. ForEach(*querySetIter)
  4689. populateQueryTree(&querySetIter->query(), queryTree);
  4690. }
  4691. else
  4692. populateQueryTree(conn->queryRoot(), queryTree);
  4693. return conn.getClear();
  4694. }
  4695. public:
  4696. IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
  4697. CQuerySetQueriesPager(const char* _querySet, const char* _xPath, const char *_sortOrder, PostFilters& _postFilters, StringArray& _unknownAttributes, const MapStringTo<bool> *_subset)
  4698. : querySet(_querySet), xPath(_xPath), sortOrder(_sortOrder), subset(_subset)
  4699. {
  4700. postFilters.activatedFilter = _postFilters.activatedFilter;
  4701. postFilters.suspendedByUserFilter = _postFilters.suspendedByUserFilter;
  4702. ForEachItemIn(x, _unknownAttributes)
  4703. unknownAttributes.append(_unknownAttributes.item(x));
  4704. }
  4705. virtual IRemoteConnection* getElements(IArrayOf<IPropertyTree> &elements)
  4706. {
  4707. Owned<IPropertyTree> elementTree = createPTree("Queries");
  4708. Owned<IRemoteConnection> conn = populateQueryTree(elementTree);
  4709. if (!conn)
  4710. return NULL;
  4711. Owned<IPropertyTreeIterator> iter = elementTree->getElements("*");
  4712. if (!iter)
  4713. return NULL;
  4714. sortElements(iter, sortOrder.get(), NULL, NULL, unknownAttributes, elements);
  4715. return conn.getClear();
  4716. }
  4717. virtual bool allMatchingElementsReceived() { return true; } //For now, dali always returns all of matched Queries.
  4718. };
  4719. StringAttr querySet;
  4720. StringBuffer xPath;
  4721. StringBuffer so;
  4722. StringArray unknownAttributes;
  4723. if (filters)
  4724. {
  4725. const char *fv = (const char *)filterbuf;
  4726. for (unsigned i=0;filters[i]!=WUQSFterm;i++) {
  4727. int fmt = filters[i];
  4728. int subfmt = (fmt&0xff);
  4729. if (subfmt==WUQSFQuerySet)
  4730. querySet.set(fv);
  4731. else if ((subfmt==WUQSFmemoryLimit) || (subfmt==WUQSFtimeLimit) || (subfmt==WUQSFwarnTimeLimit) || (subfmt==WUQSFpriority))
  4732. xPath.append('[').append(getEnumText(subfmt,querySortFields)).append(">=").append(fv).append("]");
  4733. else if ((subfmt==WUQSFmemoryLimitHi) || (subfmt==WUQSFtimeLimitHi) || (subfmt==WUQSFwarnTimeLimitHi) || (subfmt==WUQSFpriorityHi))
  4734. xPath.append('[').append(getEnumText(subfmt,querySortFields)).append("<=").append(fv).append("]");
  4735. else if (subfmt==WUQSFActivited)
  4736. postFilters.activatedFilter = (WUQueryFilterBoolean) atoi(fv);
  4737. else if (subfmt==WUQSFSuspendedByUser)
  4738. postFilters.suspendedByUserFilter = (WUQueryFilterBoolean) atoi(fv);
  4739. else if (!fv || !*fv)
  4740. unknownAttributes.append(getEnumText(subfmt,querySortFields));
  4741. else {
  4742. xPath.append('[').append(getEnumText(subfmt,querySortFields)).append('=');
  4743. if (fmt&WUQSFnocase)
  4744. xPath.append('?');
  4745. if (fmt&WUQSFnumeric)
  4746. xPath.append('#');
  4747. if (fmt&WUQSFwild)
  4748. xPath.append('~');
  4749. xPath.append('"').append(fv).append("\"]");
  4750. }
  4751. fv = fv + strlen(fv)+1;
  4752. }
  4753. }
  4754. if (sortorder) {
  4755. for (unsigned i=0;sortorder[i]!=WUQSFterm;i++) {
  4756. if (so.length())
  4757. so.append(',');
  4758. int fmt = sortorder[i];
  4759. if (fmt&WUQSFreverse)
  4760. so.append('-');
  4761. if (fmt&WUQSFnocase)
  4762. so.append('?');
  4763. if (fmt&WUQSFnumeric)
  4764. so.append('#');
  4765. so.append(getEnumText(fmt&0xff,querySortFields));
  4766. }
  4767. }
  4768. IArrayOf<IPropertyTree> results;
  4769. Owned<IElementsPager> elementsPager = new CQuerySetQueriesPager(querySet.get(), xPath.str(), so.length()?so.str():NULL, postFilters, unknownAttributes, _subset);
  4770. Owned<IRemoteConnection> conn=getElementsPaged(elementsPager,startoffset,maxnum,NULL,"",cachehint,results,total,NULL);
  4771. return new CConstQuerySetQueryIterator(results);
  4772. }
  4773. bool CWorkUnitFactory::isAborting(const char *wuid) const
  4774. {
  4775. VStringBuffer apath("/WorkUnitAborts/%s", wuid);
  4776. try
  4777. {
  4778. Owned<IRemoteConnection> acon = querySDS().connect(apath.str(), myProcessSession(), 0, SDS_LOCK_TIMEOUT);
  4779. if (acon)
  4780. return acon->queryRoot()->getPropInt(NULL) != 0;
  4781. }
  4782. catch (IException *E)
  4783. {
  4784. EXCLOG(E);
  4785. E->Release();
  4786. }
  4787. return false;
  4788. }
  4789. void CWorkUnitFactory::clearAborting(const char *wuid)
  4790. {
  4791. VStringBuffer apath("/WorkUnitAborts/%s", wuid);
  4792. try
  4793. {
  4794. Owned<IRemoteConnection> acon = querySDS().connect(apath.str(), myProcessSession(), RTM_LOCK_WRITE|RTM_LOCK_SUB, SDS_LOCK_TIMEOUT);
  4795. if (acon)
  4796. acon->close(true);
  4797. }
  4798. catch (IException *E)
  4799. {
  4800. EXCLOG(E);
  4801. E->Release();
  4802. }
  4803. }
  4804. void CWorkUnitFactory::reportAbnormalTermination(const char *wuid, WUState &state, SessionId agent)
  4805. {
  4806. WARNLOG("reportAbnormalTermination: session stopped unexpectedly: %" I64F "d state: %d", (__int64) agent, (int) state);
  4807. bool isEcl = false;
  4808. switch (state)
  4809. {
  4810. case WUStateAborting:
  4811. state = WUStateAborted;
  4812. break;
  4813. case WUStateCompiling:
  4814. isEcl = true;
  4815. // drop into
  4816. default:
  4817. state = WUStateFailed;
  4818. }
  4819. Owned<IWorkUnit> wu = updateWorkUnit(wuid, NULL, NULL);
  4820. wu->setState(state);
  4821. Owned<IWUException> e = wu->createException();
  4822. e->setExceptionCode(isEcl ? 1001 : 1000);
  4823. e->setExceptionMessage(isEcl ? "EclServer terminated unexpectedly" : "Workunit terminated unexpectedly");
  4824. }
  4825. static CriticalSection deleteDllLock;
  4826. static IWorkQueueThread *deleteDllWorkQ = nullptr;
  4827. MODULE_INIT(INIT_PRIORITY_STANDARD)
  4828. {
  4829. return true;
  4830. }
  4831. MODULE_EXIT()
  4832. {
  4833. CriticalBlock b(deleteDllLock);
  4834. if (deleteDllWorkQ)
  4835. ::Release(deleteDllWorkQ);
  4836. deleteDllWorkQ = nullptr;
  4837. }
  4838. static void asyncRemoveDll(const char * name)
  4839. {
  4840. CriticalBlock b(deleteDllLock);
  4841. if (!deleteDllWorkQ)
  4842. deleteDllWorkQ = createWorkQueueThread();
  4843. deleteDllWorkQ->post(new asyncRemoveDllWorkItem(name));
  4844. }
  4845. static void asyncRemoveFile(const char * ip, const char * name)
  4846. {
  4847. CriticalBlock b(deleteDllLock);
  4848. if (!deleteDllWorkQ)
  4849. deleteDllWorkQ = createWorkQueueThread();
  4850. deleteDllWorkQ->post(new asyncRemoveRemoteFileWorkItem(ip, name));
  4851. }
  4852. class CDaliWorkUnitFactory : public CWorkUnitFactory, implements IDaliClientShutdown
  4853. {
  4854. public:
  4855. IMPLEMENT_IINTERFACE_USING(CWorkUnitFactory);
  4856. CDaliWorkUnitFactory()
  4857. {
  4858. // Assumes dali client configuration has already been done
  4859. sdsManager = &querySDS();
  4860. session = myProcessSession();
  4861. addShutdownHook(*this);
  4862. }
  4863. ~CDaliWorkUnitFactory()
  4864. {
  4865. removeShutdownHook(*this);
  4866. }
  4867. virtual bool initializeStore()
  4868. {
  4869. throwUnexpected(); // Used when loading a plugin factory - not applicable here
  4870. }
  4871. virtual IWorkUnitWatcher *getWatcher(IWorkUnitSubscriber *subscriber, WUSubscribeOptions options, const char *wuid) const
  4872. {
  4873. return new CDaliWorkUnitWatcher(subscriber, options, wuid);
  4874. }
  4875. virtual unsigned validateRepository(bool fixErrors)
  4876. {
  4877. return 0;
  4878. }
  4879. virtual void deleteRepository(bool recreate)
  4880. {
  4881. Owned<IRemoteConnection> conn = sdsManager->connect("/WorkUnits", session, RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
  4882. if (conn)
  4883. conn->close(true);
  4884. conn.setown(sdsManager->connect("/GraphProgress", session, RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT));
  4885. if (conn)
  4886. conn->close(true);
  4887. }
  4888. virtual void createRepository()
  4889. {
  4890. // Nothing to do
  4891. }
  4892. virtual const char *queryStoreType() const
  4893. {
  4894. return "Dali";
  4895. }
  4896. virtual CLocalWorkUnit *_createWorkUnit(const char *wuid, ISecManager *secmgr, ISecUser *secuser)
  4897. {
  4898. StringBuffer wuRoot;
  4899. getXPath(wuRoot, wuid);
  4900. IRemoteConnection *conn;
  4901. conn = sdsManager->connect(wuRoot.str(), session, RTM_LOCK_WRITE|RTM_CREATE_UNIQUE, SDS_LOCK_TIMEOUT);
  4902. conn->queryRoot()->setProp("@xmlns:xsi", "http://www.w3.org/1999/XMLSchema-instance");
  4903. conn->queryRoot()->setPropInt("@wuidVersion", WUID_VERSION);
  4904. conn->queryRoot()->setProp("@totalThorTime", "");
  4905. return new CDaliWorkUnit(conn, secmgr, secuser);
  4906. }
  4907. virtual CLocalWorkUnit* _openWorkUnit(const char *wuid, ISecManager *secmgr, ISecUser *secuser)
  4908. {
  4909. StringBuffer wuRoot;
  4910. getXPath(wuRoot, wuid);
  4911. IRemoteConnection* conn = sdsManager->connect(wuRoot.str(), session, 0, SDS_LOCK_TIMEOUT);
  4912. if (conn)
  4913. return new CDaliWorkUnit(conn, secmgr, secuser);
  4914. else
  4915. return NULL;
  4916. }
  4917. virtual bool _restoreWorkUnit(IPTree *_pt, const char *wuid)
  4918. {
  4919. Owned<IPTree> pt(_pt);
  4920. Owned<IPropertyTree> gprogress = pruneBranch(pt, "GraphProgress[1]");
  4921. StringBuffer wuRoot;
  4922. getXPath(wuRoot, wuid);
  4923. Owned<IRemoteConnection> conn = sdsManager->connect(wuRoot.str(), myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
  4924. if (!conn)
  4925. {
  4926. ERRLOG("restoreWorkUnit could not create to %s", wuRoot.str());
  4927. return false;
  4928. }
  4929. IPropertyTree *root = conn->queryRoot();
  4930. if (root->hasChildren())
  4931. {
  4932. ERRLOG("restoreWorkUnit WUID %s already exists", wuid);
  4933. return false;
  4934. }
  4935. root->setPropTree(NULL, pt.getClear());
  4936. conn.clear();
  4937. // now kludge back GraphProgress
  4938. if (gprogress)
  4939. {
  4940. VStringBuffer xpath("/GraphProgress/%s", wuid);
  4941. conn.setown(querySDS().connect(xpath, myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT));
  4942. if (conn)
  4943. {
  4944. IPropertyTree *groot = conn->queryRoot();
  4945. if (groot->hasChildren())
  4946. WARNLOG("restoreWorkUnit WUID %s graphprogress already exists, replacing",wuid);
  4947. groot->setPropTree(NULL, gprogress.getClear());
  4948. }
  4949. }
  4950. return true;
  4951. }
  4952. virtual CLocalWorkUnit* _updateWorkUnit(const char *wuid, ISecManager *secmgr, ISecUser *secuser)
  4953. {
  4954. StringBuffer wuRoot;
  4955. getXPath(wuRoot, wuid);
  4956. IRemoteConnection* conn = sdsManager->connect(wuRoot.str(), session, RTM_LOCK_WRITE|RTM_LOCK_SUB, SDS_LOCK_TIMEOUT);
  4957. if (conn)
  4958. return new CDaliWorkUnit(conn, secmgr, secuser);
  4959. else
  4960. return NULL;
  4961. }
  4962. virtual IWorkUnit* getGlobalWorkUnit(ISecManager *secmgr, ISecUser *secuser)
  4963. {
  4964. // MORE - should it check security?
  4965. StringBuffer wuRoot;
  4966. getXPath(wuRoot, GLOBAL_WORKUNIT);
  4967. IRemoteConnection* conn = sdsManager->connect(wuRoot.str(), session, RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
  4968. conn->queryRoot()->setProp("@xmlns:xsi", "http://www.w3.org/1999/XMLSchema-instance");
  4969. Owned<CLocalWorkUnit> cw = new CDaliWorkUnit(conn, (ISecManager *) NULL, NULL);
  4970. return &cw->lockRemote(false);
  4971. }
  4972. virtual IConstWorkUnitIterator* getWorkUnitsByOwner(const char * owner, ISecManager *secmgr, ISecUser *secuser)
  4973. {
  4974. StringBuffer path("*");
  4975. if (owner && *owner)
  4976. path.append("[@submitID=?~\"").append(owner).append("\"]");
  4977. return _getWorkUnitsByXPath(path.str(), secmgr, secuser);
  4978. }
  4979. IConstWorkUnitIterator* getScheduledWorkUnits(ISecManager *secmgr, ISecUser *secuser)
  4980. {
  4981. StringBuffer path("*");
  4982. path.append("[@state=\"").append(getEnumText(WUStateScheduled, states)).append("\"]");
  4983. return _getWorkUnitsByXPath(path.str(), secmgr, secuser);
  4984. }
  4985. virtual void clientShutdown();
  4986. virtual unsigned numWorkUnits()
  4987. {
  4988. Owned<IRemoteConnection> conn = sdsManager->connect("/WorkUnits", session, 0, SDS_LOCK_TIMEOUT);
  4989. if (!conn)
  4990. return 0;
  4991. IPropertyTree *root = conn->queryRoot();
  4992. return root->numChildren();
  4993. }
  4994. /**
  4995. * Add a filter to an xpath query, with the appropriate filter flags
  4996. */
  4997. static void appendFilterToQueryString(StringBuffer& query, int flags, const char* name, const char* value)
  4998. {
  4999. query.append('[').append(name).append('=');
  5000. if (flags & WUSFnocase)
  5001. query.append('?');
  5002. if (flags & WUSFwild)
  5003. query.append('~');
  5004. query.append('"').append(value).append("\"]");
  5005. };
  5006. IConstWorkUnitIterator* getWorkUnitsSorted( WUSortField sortorder, // field to sort by (and flags for desc sort etc)
  5007. WUSortField *filters, // NULL or list of fields to filter on (terminated by WUSFterm)
  5008. const void *filterbuf, // (appended) string values for filters
  5009. unsigned startoffset,
  5010. unsigned maxnum,
  5011. __int64 *cachehint,
  5012. unsigned *total,
  5013. ISecManager *secmgr,
  5014. ISecUser *secuser)
  5015. {
  5016. class CQueryOrFilter : public CInterface
  5017. {
  5018. unsigned flags;
  5019. StringAttr name;
  5020. StringArray values;
  5021. public:
  5022. IMPLEMENT_IINTERFACE;
  5023. CQueryOrFilter(unsigned _flags, const char *_name, const char *_value)
  5024. : flags(_flags), name(_name)
  5025. {
  5026. values.appendListUniq(_value, "|");
  5027. };
  5028. const char* queryName() { return name.get(); };
  5029. unsigned querySearchFlags() { return flags; };
  5030. const StringArray& queryValues() const { return values; };
  5031. };
  5032. class CMultiPTreeIterator : public CInterfaceOf<IPropertyTreeIterator>
  5033. {
  5034. public:
  5035. virtual bool first() override
  5036. {
  5037. curSource = 0;
  5038. while (sources.isItem(curSource))
  5039. {
  5040. if (sources.item(curSource).first())
  5041. return true;
  5042. curSource++;
  5043. }
  5044. return false;
  5045. }
  5046. virtual bool next() override
  5047. {
  5048. if (sources.isItem(curSource))
  5049. {
  5050. if (sources.item(curSource).next())
  5051. return true;
  5052. curSource++;
  5053. while (sources.isItem(curSource))
  5054. {
  5055. if (sources.item(curSource).first())
  5056. return true;
  5057. curSource++;
  5058. }
  5059. }
  5060. return false;
  5061. }
  5062. virtual bool isValid() override
  5063. {
  5064. return sources.isItem(curSource);
  5065. }
  5066. virtual IPropertyTree & query() override
  5067. {
  5068. return sources.item(curSource).query();
  5069. }
  5070. void addSource(IPropertyTreeIterator &source)
  5071. {
  5072. sources.append(source);
  5073. }
  5074. private:
  5075. IArrayOf<IPropertyTreeIterator> sources;
  5076. unsigned curSource = 0;
  5077. };
  5078. class CWorkUnitsPager : public CSimpleInterface, implements IElementsPager
  5079. {
  5080. StringAttr xPath;
  5081. StringAttr sortOrder;
  5082. StringAttr nameFilterLo;
  5083. StringAttr nameFilterHi;
  5084. StringArray unknownAttributes;
  5085. Owned<CQueryOrFilter> orFilter;
  5086. public:
  5087. IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
  5088. CWorkUnitsPager(const char* _xPath, CQueryOrFilter* _orFilter, const char *_sortOrder, const char* _nameFilterLo, const char* _nameFilterHi, StringArray& _unknownAttributes)
  5089. : xPath(_xPath), orFilter(_orFilter), sortOrder(_sortOrder), nameFilterLo(_nameFilterLo), nameFilterHi(_nameFilterHi)
  5090. {
  5091. ForEachItemIn(x, _unknownAttributes)
  5092. unknownAttributes.append(_unknownAttributes.item(x));
  5093. }
  5094. virtual IRemoteConnection* getElements(IArrayOf<IPropertyTree> &elements)
  5095. {
  5096. Owned<IRemoteConnection> conn = querySDS().connect("WorkUnits", myProcessSession(), 0, SDS_LOCK_TIMEOUT);
  5097. if (!conn)
  5098. return NULL;
  5099. Owned<IPropertyTreeIterator> iter;
  5100. if (!orFilter)
  5101. {
  5102. iter.setown(conn->getElements(xPath.get()));
  5103. }
  5104. else
  5105. {
  5106. Owned <CMultiPTreeIterator> multi = new CMultiPTreeIterator;
  5107. bool added = false;
  5108. const char* fieldName = orFilter->queryName();
  5109. unsigned flags = orFilter->querySearchFlags();
  5110. const StringArray& values = orFilter->queryValues();
  5111. ForEachItemIn(i, values)
  5112. {
  5113. StringBuffer path(xPath.get());
  5114. const char* value = values.item(i);
  5115. if (!isEmptyString(value))
  5116. {
  5117. appendFilterToQueryString(path, flags, fieldName, value);
  5118. IPropertyTreeIterator *itr = conn->getElements(path.str());
  5119. if (itr)
  5120. {
  5121. multi->addSource(*itr);
  5122. added = true;
  5123. }
  5124. }
  5125. }
  5126. if (added)
  5127. iter.setown(multi.getClear());
  5128. }
  5129. if (!iter)
  5130. return NULL;
  5131. sortElements(iter, sortOrder.get(), nameFilterLo.get(), nameFilterHi.get(), unknownAttributes, elements);
  5132. return conn.getClear();
  5133. }
  5134. virtual bool allMatchingElementsReceived() { return true; }//For now, dali always returns all of matched WUs.
  5135. };
  5136. class CScopeChecker : public CSimpleInterface, implements ISortedElementsTreeFilter
  5137. {
  5138. UniqueScopes done;
  5139. ISecManager *secmgr;
  5140. ISecUser *secuser;
  5141. CriticalSection crit;
  5142. public:
  5143. IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
  5144. CScopeChecker(ISecManager *_secmgr,ISecUser *_secuser)
  5145. {
  5146. secmgr = _secmgr;
  5147. secuser = _secuser;
  5148. }
  5149. bool isOK(IPropertyTree &tree)
  5150. {
  5151. const char *scopename = tree.queryProp("@scope");
  5152. if (!scopename||!*scopename)
  5153. return true;
  5154. {
  5155. CriticalBlock block(crit);
  5156. const bool *b = done.getValue(scopename);
  5157. if (b)
  5158. return *b;
  5159. }
  5160. bool ret = checkWuScopeSecAccess(scopename,secmgr,secuser,SecAccess_Read,"iterating",false,false);
  5161. {
  5162. // conceivably could have already been checked and added, but ok.
  5163. CriticalBlock block(crit);
  5164. done.setValue(scopename,ret);
  5165. }
  5166. return ret;
  5167. }
  5168. };
  5169. Owned<CQueryOrFilter> orFilter;
  5170. Owned<ISortedElementsTreeFilter> sc = new CScopeChecker(secmgr,secuser);
  5171. StringBuffer query;
  5172. StringBuffer so;
  5173. StringAttr namefilter("*");
  5174. StringAttr namefilterlo;
  5175. StringAttr namefilterhi;
  5176. StringArray unknownAttributes;
  5177. if (filters)
  5178. {
  5179. const char *fv = (const char *) filterbuf;
  5180. for (unsigned i=0;filters[i]!=WUSFterm;i++)
  5181. {
  5182. assertex(fv);
  5183. int fmt = filters[i];
  5184. int subfmt = (fmt&0xff);
  5185. if (subfmt==WUSFwuid)
  5186. namefilterlo.set(fv);
  5187. else if (subfmt==WUSFwuidhigh)
  5188. namefilterhi.set(fv);
  5189. else if (subfmt==WUSFwildwuid)
  5190. namefilter.set(fv);
  5191. else if (subfmt==WUSFappvalue)
  5192. {
  5193. const char *app = fv;
  5194. fv = fv + strlen(fv)+1;
  5195. query.append("[Application/").append(app);
  5196. if (*fv)
  5197. query.append("=?~\"").append(fv).append('\"');
  5198. query.append("]");
  5199. }
  5200. else if (!*fv)
  5201. {
  5202. unknownAttributes.append(getEnumText(subfmt,workunitSortFields));
  5203. if (subfmt==WUSFtotalthortime)
  5204. sortorder = (WUSortField) (sortorder & ~WUSFnumeric);
  5205. }
  5206. else
  5207. {
  5208. const char* fieldName = getEnumText(subfmt,workunitSortFields);
  5209. if (!strchr(fv, '|'))
  5210. appendFilterToQueryString(query, fmt, fieldName, fv);
  5211. else if (orFilter)
  5212. throw MakeStringException(WUERR_InvalidUserInput, "Multiple OR filters not allowed");
  5213. else
  5214. {
  5215. if (!strieq(fieldName, getEnumText(WUSFstate,workunitSortFields)) &&
  5216. !strieq(fieldName, getEnumText(WUSFuser,workunitSortFields)) &&
  5217. !strieq(fieldName, getEnumText(WUSFcluster,workunitSortFields)))
  5218. throw MakeStringException(WUERR_InvalidUserInput, "OR filters not allowed for %s", fieldName);
  5219. orFilter.setown(new CQueryOrFilter(fmt, fieldName, fv));
  5220. }
  5221. }
  5222. fv = fv + strlen(fv)+1;
  5223. }
  5224. }
  5225. if ((sortorder&0xff)==WUSFtotalthortime)
  5226. sortorder = (WUSortField) (sortorder & ~WUSFnumeric);
  5227. query.insert(0, namefilter.get());
  5228. if (sortorder)
  5229. {
  5230. if (so.length())
  5231. so.append(',');
  5232. if (sortorder & WUSFreverse)
  5233. so.append('-');
  5234. if (sortorder & WUSFnocase)
  5235. so.append('?');
  5236. if (sortorder & WUSFnumeric)
  5237. so.append('#');
  5238. so.append(getEnumText(sortorder&0xff,workunitSortFields));
  5239. }
  5240. IArrayOf<IPropertyTree> results;
  5241. Owned<IElementsPager> elementsPager = new CWorkUnitsPager(query.str(), orFilter.getClear(), so.length()?so.str():NULL, namefilterlo.get(), namefilterhi.get(), unknownAttributes);
  5242. Owned<IRemoteConnection> conn=getElementsPaged(elementsPager,startoffset,maxnum,secmgr?sc:NULL,"",cachehint,results,total,NULL);
  5243. return new CConstWUArrayIterator(results);
  5244. }
  5245. virtual WUState waitForWorkUnit(const char * wuid, unsigned timeout, bool compiled, bool returnOnWaitState)
  5246. {
  5247. WUState ret = WUStateUnknown;
  5248. StringBuffer wuRoot;
  5249. getXPath(wuRoot, wuid);
  5250. Owned<IRemoteConnection> conn = sdsManager->connect(wuRoot.str(), session, 0, SDS_LOCK_TIMEOUT);
  5251. if (timeout == 0) //no need to subscribe
  5252. {
  5253. ret = (WUState) getEnum(conn->queryRoot(), "@state", states);
  5254. switch (ret)
  5255. {
  5256. case WUStateCompiled:
  5257. case WUStateUploadingFiles:
  5258. if (!compiled)
  5259. break;
  5260. //fall through
  5261. case WUStateCompleted:
  5262. case WUStateFailed:
  5263. case WUStateAborted:
  5264. return ret;
  5265. case WUStateWait:
  5266. if(returnOnWaitState)
  5267. return ret;
  5268. default:
  5269. break;
  5270. }
  5271. return WUStateUnknown;
  5272. }
  5273. Owned<WorkUnitWaiter> waiter = new WorkUnitWaiter(wuid, SubscribeOptionState);
  5274. LocalIAbortHandler abortHandler(*waiter);
  5275. if (conn)
  5276. {
  5277. SessionId agent = -1;
  5278. bool agentSessionStopped = false;
  5279. unsigned start = msTick();
  5280. for (;;)
  5281. {
  5282. ret = (WUState) getEnum(conn->queryRoot(), "@state", states);
  5283. switch (ret)
  5284. {
  5285. case WUStateCompiled:
  5286. case WUStateUploadingFiles:
  5287. if (!compiled)
  5288. break;
  5289. // fall into
  5290. case WUStateCompleted:
  5291. case WUStateFailed:
  5292. case WUStateAborted:
  5293. return ret;
  5294. case WUStateWait:
  5295. if(returnOnWaitState)
  5296. {
  5297. return ret;
  5298. }
  5299. break;
  5300. case WUStateCompiling:
  5301. case WUStateRunning:
  5302. case WUStateDebugPaused:
  5303. case WUStateDebugRunning:
  5304. case WUStateBlocked:
  5305. case WUStateAborting:
  5306. if (agentSessionStopped)
  5307. {
  5308. reportAbnormalTermination(wuid, ret, agent);
  5309. return ret;
  5310. }
  5311. if (queryDaliServerVersion().compare("2.1")>=0)
  5312. {
  5313. agent = conn->queryRoot()->getPropInt64("@agentSession", -1);
  5314. if((agent>0) && querySessionManager().sessionStopped(agent, 0))
  5315. {
  5316. agentSessionStopped = true;
  5317. conn->reload();
  5318. continue;
  5319. }
  5320. }
  5321. break;
  5322. }
  5323. agentSessionStopped = false; // reset for state changes such as WUStateWait then WUStateRunning again
  5324. unsigned waited = msTick() - start;
  5325. if (timeout==-1 || waited + 20000 < timeout)
  5326. {
  5327. waiter->wait(20000); // recheck state every 20 seconds, in case eclagent has crashed.
  5328. if (waiter->isAborted())
  5329. {
  5330. ret = WUStateUnknown; // MORE - throw an exception?
  5331. break;
  5332. }
  5333. }
  5334. else if (waited > timeout || !waiter->wait(timeout-waited))
  5335. {
  5336. ret = WUStateUnknown; // MORE - throw an exception?
  5337. break;
  5338. }
  5339. conn->reload();
  5340. }
  5341. }
  5342. return ret;
  5343. }
  5344. virtual WUAction waitForWorkUnitAction(const char * wuid, WUAction original)
  5345. {
  5346. Owned<WorkUnitWaiter> waiter = new WorkUnitWaiter(wuid, SubscribeOptionAction);
  5347. LocalIAbortHandler abortHandler(*waiter);
  5348. WUAction ret = WUActionUnknown;
  5349. StringBuffer wuRoot;
  5350. getXPath(wuRoot, wuid);
  5351. Owned<IRemoteConnection> conn = sdsManager->connect(wuRoot.str(), session, 0, SDS_LOCK_TIMEOUT);
  5352. if (conn)
  5353. {
  5354. unsigned start = msTick();
  5355. for (;;)
  5356. {
  5357. ret = (WUAction) getEnum(conn->queryRoot(), "Action", actions);
  5358. if (ret != original)
  5359. break;
  5360. unsigned waited = msTick() - start;
  5361. waiter->wait(20000); // recheck state every 20 seconds even if no timeout, in case eclagent has crashed.
  5362. if (waiter->isAborted())
  5363. {
  5364. ret = WUActionUnknown; // MORE - throw an exception?
  5365. break;
  5366. }
  5367. conn->reload();
  5368. }
  5369. }
  5370. waiter->unsubscribe();
  5371. return ret;
  5372. }
  5373. protected:
  5374. IConstWorkUnitIterator * _getWorkUnitsByXPath(const char *xpath, ISecManager *secmgr, ISecUser *secuser)
  5375. {
  5376. Owned<IRemoteConnection> conn = sdsManager->connect("/WorkUnits", session, 0, SDS_LOCK_TIMEOUT);
  5377. if (conn)
  5378. {
  5379. CDaliVersion serverVersionNeeded("3.2");
  5380. Owned<IPropertyTreeIterator> iter(queryDaliServerVersion().compare(serverVersionNeeded) < 0 ?
  5381. conn->queryRoot()->getElements(xpath) :
  5382. conn->getElements(xpath));
  5383. return createSecureConstWUIterator(iter.getClear(), secmgr, secuser);
  5384. }
  5385. else
  5386. return NULL;
  5387. }
  5388. ISDSManager *sdsManager;
  5389. SessionId session;
  5390. };
  5391. extern WORKUNIT_API IConstWorkUnitIterator *createSecureConstWUIterator(IConstWorkUnitIterator *iter, ISecManager *secmgr, ISecUser *secuser)
  5392. {
  5393. if (secmgr)
  5394. return new CSecureConstWUIterator(iter, secmgr, secuser);
  5395. else
  5396. return iter;
  5397. }
  5398. extern WORKUNIT_API IConstWorkUnitIterator *createSecureConstWUIterator(IPropertyTreeIterator *iter, ISecManager *secmgr, ISecUser *secuser)
  5399. {
  5400. if (secmgr)
  5401. return new CSecureConstWUIterator(new CConstWUIterator(iter), secmgr, secuser);
  5402. else
  5403. return new CConstWUIterator(iter);
  5404. }
  5405. static CriticalSection factoryCrit;
  5406. static Owned<IWorkUnitFactory> factory;
  5407. void CDaliWorkUnitFactory::clientShutdown()
  5408. {
  5409. CriticalBlock b(factoryCrit);
  5410. factory.clear();
  5411. }
  5412. void clientShutdownWorkUnit()
  5413. {
  5414. CriticalBlock b(factoryCrit);
  5415. factory.clear();
  5416. }
  5417. extern WORKUNIT_API void setWorkUnitFactory(IWorkUnitFactory * _factory)
  5418. {
  5419. CriticalBlock b(factoryCrit);
  5420. factory.setown(_factory);
  5421. }
  5422. extern WORKUNIT_API IWorkUnitFactory * getWorkUnitFactory()
  5423. {
  5424. if (!factory)
  5425. {
  5426. CriticalBlock b(factoryCrit);
  5427. if (!factory) // NOTE - this "double test" paradigm is not guaranteed threadsafe on modern systems/compilers - I think in this instance that is harmless even in the (extremely) unlikely event that it resulted in the setown being called twice.
  5428. {
  5429. const char *forceEnv = getenv("FORCE_DALI_WORKUNITS");
  5430. bool forceDali = forceEnv && !strieq(forceEnv, "off") && !strieq(forceEnv, "0");
  5431. Owned<IRemoteConnection> env = querySDS().connect("/Environment", myProcessSession(), 0, SDS_LOCK_TIMEOUT);
  5432. IPropertyTree *pluginInfo = NULL;
  5433. if (env)
  5434. {
  5435. SocketEndpoint targetDali = queryCoven().queryGroup().queryNode(0).endpoint();
  5436. IPropertyTree *daliInfo = findDaliProcess(env->queryRoot(), targetDali);
  5437. if (daliInfo)
  5438. {
  5439. const char *daliName = daliInfo->queryProp("@name");
  5440. if (daliName)
  5441. {
  5442. VStringBuffer xpath("Software/DaliServerPlugin[@type='WorkunitServer'][@daliServers='%s']", daliName);
  5443. pluginInfo = env->queryRoot()->queryPropTree(xpath);
  5444. }
  5445. if (!pluginInfo)
  5446. pluginInfo = daliInfo->queryPropTree("Plugin[@type='WorkunitServer']"); // Compatibility with early betas of 6.0 ...
  5447. }
  5448. }
  5449. if (pluginInfo && !forceDali)
  5450. factory.setown( (IWorkUnitFactory *) loadPlugin(pluginInfo));
  5451. else
  5452. factory.setown(new CDaliWorkUnitFactory());
  5453. }
  5454. }
  5455. return factory.getLink();
  5456. }
  5457. extern WORKUNIT_API IWorkUnitFactory * getDaliWorkUnitFactory()
  5458. {
  5459. if (!factory)
  5460. {
  5461. CriticalBlock b(factoryCrit);
  5462. if (!factory) // NOTE - this "double test" paradigm is not guaranteed threadsafe on modern systems/compilers - I think in this instance that is harmless even in the (extremely) unlikely event that it resulted in the setown being called twice.
  5463. factory.setown(new CDaliWorkUnitFactory());
  5464. }
  5465. return factory.getLink();
  5466. }
  5467. // A SecureWorkUnitFactory allows the security params to be supplied once to the factory rather than being supplied to each call.
  5468. // They can still be supplied if you want...
  5469. class CSecureWorkUnitFactory : implements IWorkUnitFactory, public CInterface
  5470. {
  5471. public:
  5472. IMPLEMENT_IINTERFACE;
  5473. CSecureWorkUnitFactory(IWorkUnitFactory *_baseFactory, ISecManager *_secMgr, ISecUser *_secUser)
  5474. : baseFactory(_baseFactory), defaultSecMgr(_secMgr), defaultSecUser(_secUser)
  5475. {
  5476. }
  5477. virtual bool initializeStore()
  5478. {
  5479. throwUnexpected(); // Used when loading a plugin factory - not applicable here
  5480. }
  5481. virtual IWorkUnitWatcher *getWatcher(IWorkUnitSubscriber *subscriber, WUSubscribeOptions options, const char *wuid) const
  5482. {
  5483. return baseFactory->getWatcher(subscriber, options, wuid);
  5484. }
  5485. virtual unsigned validateRepository(bool fix)
  5486. {
  5487. return baseFactory->validateRepository(fix);
  5488. }
  5489. virtual void deleteRepository(bool recreate)
  5490. {
  5491. return baseFactory->deleteRepository(recreate);
  5492. }
  5493. virtual void createRepository()
  5494. {
  5495. return baseFactory->createRepository();
  5496. }
  5497. virtual const char *queryStoreType() const
  5498. {
  5499. return baseFactory->queryStoreType();
  5500. }
  5501. virtual StringArray &getUniqueValues(WUSortField field, const char *prefix, StringArray &result) const
  5502. {
  5503. return baseFactory->getUniqueValues(field, prefix, result);
  5504. }
  5505. virtual IWorkUnit* createNamedWorkUnit(const char *wuid, const char *app, const char *user, ISecManager *secMgr, ISecUser *secUser)
  5506. {
  5507. if (!secMgr) secMgr = defaultSecMgr.get();
  5508. if (!secUser) secUser = defaultSecUser.get();
  5509. return baseFactory->createNamedWorkUnit(wuid, app, user, secMgr, secUser);
  5510. }
  5511. virtual IWorkUnit* createWorkUnit(const char *app, const char *user, ISecManager *secMgr, ISecUser *secUser)
  5512. {
  5513. if (!secMgr) secMgr = defaultSecMgr.get();
  5514. if (!secUser) secUser = defaultSecUser.get();
  5515. return baseFactory->createWorkUnit(app, user, secMgr, secUser);
  5516. }
  5517. virtual bool deleteWorkUnit(const char * wuid, ISecManager *secMgr, ISecUser *secUser)
  5518. {
  5519. if (!secMgr) secMgr = defaultSecMgr.get();
  5520. if (!secUser) secUser = defaultSecUser.get();
  5521. return baseFactory->deleteWorkUnit(wuid, secMgr, secUser);
  5522. }
  5523. virtual bool deleteWorkUnitEx(const char * wuid, bool throwException, ISecManager *secMgr, ISecUser *secUser)
  5524. {
  5525. if (!secMgr) secMgr = defaultSecMgr.get();
  5526. if (!secUser) secUser = defaultSecUser.get();
  5527. return baseFactory->deleteWorkUnitEx(wuid, throwException, secMgr, secUser);
  5528. }
  5529. virtual IConstWorkUnit* openWorkUnit(const char *wuid, ISecManager *secMgr, ISecUser *secUser)
  5530. {
  5531. if (!secMgr) secMgr = defaultSecMgr.get();
  5532. if (!secUser) secUser = defaultSecUser.get();
  5533. return baseFactory->openWorkUnit(wuid, secMgr, secUser);
  5534. }
  5535. virtual IWorkUnit* updateWorkUnit(const char *wuid, ISecManager *secMgr, ISecUser *secUser)
  5536. {
  5537. if (!secMgr) secMgr = defaultSecMgr.get();
  5538. if (!secUser) secUser = defaultSecUser.get();
  5539. return baseFactory->updateWorkUnit(wuid, secMgr, secUser);
  5540. }
  5541. virtual bool restoreWorkUnit(const char *base, const char *wuid, bool restoreAssociated)
  5542. {
  5543. return baseFactory->restoreWorkUnit(base, wuid, restoreAssociated);
  5544. }
  5545. virtual IWorkUnit * getGlobalWorkUnit(ISecManager *secMgr, ISecUser *secUser)
  5546. {
  5547. if (!secMgr) secMgr = defaultSecMgr.get();
  5548. if (!secUser) secUser = defaultSecUser.get();
  5549. return baseFactory->getGlobalWorkUnit(secMgr, secUser);
  5550. }
  5551. virtual IConstWorkUnitIterator * getWorkUnitsByOwner(const char * owner, ISecManager *secMgr, ISecUser *secUser)
  5552. {
  5553. if (!secMgr) secMgr = defaultSecMgr.get();
  5554. if (!secUser) secUser = defaultSecUser.get();
  5555. return baseFactory->getWorkUnitsByOwner(owner, secMgr, secUser);
  5556. }
  5557. virtual IConstWorkUnitIterator * getScheduledWorkUnits(ISecManager *secMgr, ISecUser *secUser)
  5558. {
  5559. if (!secMgr) secMgr = defaultSecMgr.get();
  5560. if (!secUser) secUser = defaultSecUser.get();
  5561. return baseFactory->getScheduledWorkUnits(secMgr, secUser);
  5562. }
  5563. virtual void descheduleAllWorkUnits(ISecManager *secMgr, ISecUser *secUser)
  5564. {
  5565. if (!secMgr) secMgr = defaultSecMgr.get();
  5566. if (!secUser) secUser = defaultSecUser.get();
  5567. baseFactory->descheduleAllWorkUnits(secMgr, secUser);
  5568. }
  5569. virtual int setTracingLevel(int newLevel)
  5570. {
  5571. return baseFactory->setTracingLevel(newLevel);
  5572. }
  5573. virtual IConstWorkUnitIterator* getWorkUnitsSorted( WUSortField sortorder, // field to sort by
  5574. WUSortField *filters, // NULL or list of fields to filter on (terminated by WUSFterm)
  5575. const void *filterbuf, // (appended) string values for filters
  5576. unsigned startoffset,
  5577. unsigned maxnum,
  5578. __int64 *cachehint,
  5579. unsigned *total,
  5580. ISecManager *secMgr, ISecUser *secUser)
  5581. {
  5582. if (!secMgr) secMgr = defaultSecMgr.get();
  5583. if (!secUser) secUser = defaultSecUser.get();
  5584. return baseFactory->getWorkUnitsSorted(sortorder,filters,filterbuf,startoffset,maxnum,cachehint, total, secMgr, secUser);
  5585. }
  5586. virtual IConstQuerySetQueryIterator* getQuerySetQueriesSorted( WUQuerySortField *sortorder,
  5587. WUQuerySortField *filters,
  5588. const void *filterbuf,
  5589. unsigned startoffset,
  5590. unsigned maxnum,
  5591. __int64 *cachehint,
  5592. unsigned *total,
  5593. const MapStringTo<bool> *subset)
  5594. {
  5595. // MORE - why no security?
  5596. return baseFactory->getQuerySetQueriesSorted(sortorder,filters,filterbuf,startoffset,maxnum,cachehint,total,subset);
  5597. }
  5598. virtual unsigned numWorkUnits()
  5599. {
  5600. return baseFactory->numWorkUnits();
  5601. }
  5602. virtual bool isAborting(const char *wuid) const
  5603. {
  5604. return baseFactory->isAborting(wuid);
  5605. }
  5606. virtual void clearAborting(const char *wuid)
  5607. {
  5608. baseFactory->clearAborting(wuid);
  5609. }
  5610. virtual WUState waitForWorkUnit(const char * wuid, unsigned timeout, bool compiled, bool returnOnWaitState)
  5611. {
  5612. return baseFactory->waitForWorkUnit(wuid, timeout, compiled, returnOnWaitState);
  5613. }
  5614. virtual WUAction waitForWorkUnitAction(const char * wuid, WUAction original)
  5615. {
  5616. return baseFactory->waitForWorkUnitAction(wuid, original);
  5617. }
  5618. private:
  5619. Owned<IWorkUnitFactory> baseFactory;
  5620. Linked<ISecManager> defaultSecMgr;
  5621. Linked<ISecUser> defaultSecUser;
  5622. };
  5623. extern WORKUNIT_API IWorkUnitFactory * getWorkUnitFactory(ISecManager *secmgr, ISecUser *secuser)
  5624. {
  5625. if (secmgr && secuser)
  5626. return new CSecureWorkUnitFactory(getWorkUnitFactory(), secmgr, secuser);
  5627. else
  5628. return getWorkUnitFactory();
  5629. }
  5630. //==========================================================================================
  5631. class CStringPTreeIterator : implements IStringIterator, public CInterface
  5632. {
  5633. Owned<IPropertyTreeIterator> it;
  5634. public:
  5635. IMPLEMENT_IINTERFACE;
  5636. CStringPTreeIterator(IPropertyTreeIterator *p) : it(p) {};
  5637. virtual bool first() { return it->first(); }
  5638. virtual bool next() { return it->next(); }
  5639. virtual bool isValid() { return it->isValid(); }
  5640. virtual IStringVal & str(IStringVal &s) { s.set(it->query().queryProp(NULL)); return s; }
  5641. };
  5642. class CStringPTreeTagIterator : implements IStringIterator, public CInterface
  5643. {
  5644. Owned<IPropertyTreeIterator> it;
  5645. public:
  5646. IMPLEMENT_IINTERFACE;
  5647. CStringPTreeTagIterator(IPropertyTreeIterator *p) : it(p) {};
  5648. virtual bool first() { return it->first(); }
  5649. virtual bool next() { return it->next(); }
  5650. virtual bool isValid() { return it->isValid(); }
  5651. virtual IStringVal & str(IStringVal &s) { s.set(it->query().queryName()); return s; }
  5652. };
  5653. class CStringPTreeAttrIterator : implements IStringIterator, public CInterface
  5654. {
  5655. Owned<IPropertyTreeIterator> it;
  5656. StringAttr name;
  5657. public:
  5658. IMPLEMENT_IINTERFACE;
  5659. CStringPTreeAttrIterator(IPropertyTreeIterator *p, const char *_name) : it(p), name(_name) {};
  5660. virtual bool first() { return it->first(); }
  5661. virtual bool next() { return it->next(); }
  5662. virtual bool isValid() { return it->isValid(); }
  5663. virtual IStringVal & str(IStringVal &s) { s.set(it->query().queryProp(name)); return s; }
  5664. };
  5665. //==========================================================================================
  5666. CLocalWorkUnit::CLocalWorkUnit(ISecManager *secmgr, ISecUser *secuser)
  5667. {
  5668. clearCached(false);
  5669. secMgr.set(secmgr);
  5670. secUser.set(secuser);
  5671. workflowIteratorCached = false;
  5672. resultsCached = false;
  5673. graphsCached = false;
  5674. temporariesCached = false;
  5675. variablesCached = false;
  5676. exceptionsCached = false;
  5677. pluginsCached = false;
  5678. librariesCached = false;
  5679. activitiesCached = false;
  5680. webServicesInfoCached = false;
  5681. roxieQueryInfoCached = false;
  5682. }
  5683. void CLocalWorkUnit::clearCached(bool clearTree)
  5684. {
  5685. query.clear();
  5686. webServicesInfo.clear();
  5687. workflowIterator.clear();
  5688. graphs.kill();
  5689. results.kill();
  5690. variables.kill();
  5691. plugins.kill();
  5692. libraries.kill();
  5693. exceptions.kill();
  5694. temporaries.kill();
  5695. statistics.kill();
  5696. appvalues.kill();
  5697. if (clearTree)
  5698. p.clear();
  5699. workflowIteratorCached = false;
  5700. resultsCached = false;
  5701. graphsCached = false;
  5702. temporariesCached = false;
  5703. variablesCached = false;
  5704. exceptionsCached = false;
  5705. pluginsCached = false;
  5706. librariesCached = false;
  5707. activitiesCached = false;
  5708. webServicesInfoCached = false;
  5709. roxieQueryInfoCached = false;
  5710. }
  5711. void CLocalWorkUnit::loadPTree(IPropertyTree *ptree)
  5712. {
  5713. clearCached(false);
  5714. p.setown(ptree);
  5715. }
  5716. void CLocalWorkUnit::beforeDispose()
  5717. {
  5718. try
  5719. {
  5720. unsubscribe();
  5721. clearCached(true);
  5722. userDesc.clear();
  5723. secMgr.clear();
  5724. secUser.clear();
  5725. }
  5726. catch (IException *E) { LOG(MCexception(E, MSGCLS_warning), E, "Exception during ~CLocalWorkUnit"); E->Release(); }
  5727. }
  5728. void CLocalWorkUnit::cleanupAndDelete(bool deldll, bool deleteOwned, const StringArray *deleteExclusions)
  5729. {
  5730. MTIME_SECTION(queryActiveTimer(), "WUDELETE cleanupAndDelete total");
  5731. // Delete any related things in SDS etc that might otherwise be forgotten
  5732. if (p->getPropBool("@protected", false))
  5733. throw MakeStringException(WUERR_WorkunitProtected, "%s: Workunit is protected",p->queryName());
  5734. switch (getState())
  5735. {
  5736. case WUStateAborted:
  5737. case WUStateCompleted:
  5738. case WUStateFailed:
  5739. case WUStateArchived:
  5740. break;
  5741. case WUStateCompiled:
  5742. if (getAction()==WUActionRun || getAction()==WUActionUnknown)
  5743. throw MakeStringException(WUERR_WorkunitActive, "%s: Workunit is active. Please abort before deleting this workunit.",p->queryName());
  5744. break;
  5745. case WUStateWait:
  5746. throw MakeStringException(WUERR_WorkunitScheduled, "%s: Workunit is scheduled",p->queryName());
  5747. default:
  5748. throw MakeStringException(WUERR_WorkunitActive, "%s: Workunit is active. Please abort before deleting this workunit.",p->queryName());
  5749. break;
  5750. }
  5751. if (getIsQueryService())
  5752. {
  5753. Owned<IPropertyTree> registry = getQueryRegistryRoot();
  5754. if (registry)
  5755. {
  5756. VStringBuffer xpath("QuerySet/Query[@wuid='%s']", p->queryName());
  5757. if (registry->hasProp(xpath.str()))
  5758. throw MakeStringException(WUERR_WorkunitPublished, "%s: Workunit is published",p->queryName());
  5759. }
  5760. }
  5761. try
  5762. {
  5763. if (deldll && !p->getPropBool("@isClone", false))
  5764. {
  5765. Owned<IConstWUQuery> q = getQuery();
  5766. if (q)
  5767. {
  5768. Owned<IConstWUAssociatedFileIterator> iter = &q->getAssociatedFiles();
  5769. SCMStringBuffer name;
  5770. ForEach(*iter)
  5771. {
  5772. IConstWUAssociatedFile & cur = iter->query();
  5773. cur.getNameTail(name);
  5774. if (!deleteExclusions || (NotFound == deleteExclusions->find(name.str())))
  5775. {
  5776. Owned<IDllEntry> entry = queryDllServer().getEntry(name.str());
  5777. if (entry.get())
  5778. asyncRemoveDll(name.str());
  5779. else
  5780. {
  5781. SCMStringBuffer ip, localPath;
  5782. cur.getName(localPath);
  5783. cur.getIp(ip);
  5784. asyncRemoveFile(ip.str(), localPath.str());
  5785. }
  5786. }
  5787. }
  5788. }
  5789. }
  5790. factory->clearAborting(queryWuid());
  5791. deleteTempFiles(NULL, deleteOwned, true); // all, any remaining.
  5792. }
  5793. catch(IException *E)
  5794. {
  5795. StringBuffer s;
  5796. LOG(MCexception(E, MSGCLS_warning), E, s.append("Exception during cleanupAndDelete: ").append(p->queryName()).str());
  5797. E->Release();
  5798. }
  5799. catch (...)
  5800. {
  5801. WARNLOG("Unknown exception during cleanupAndDelete: %s", p->queryName());
  5802. }
  5803. }
  5804. void CLocalWorkUnit::setTimeScheduled(const IJlibDateTime &val)
  5805. {
  5806. SCMStringBuffer strval;
  5807. val.getGmtString(strval);
  5808. p->setProp("@timescheduled",strval.str());
  5809. }
  5810. IJlibDateTime & CLocalWorkUnit::getTimeScheduled(IJlibDateTime &val) const
  5811. {
  5812. StringBuffer str;
  5813. p->getProp("@timescheduled",str);
  5814. if(str.length())
  5815. val.setGmtString(str.str());
  5816. return val;
  5817. }
  5818. bool modifyAndWriteWorkUnitXML(char const * wuid, StringBuffer & buf, StringBuffer & extra, IFileIO * fileio)
  5819. {
  5820. // kludge in extra chunks of XML such as GraphProgress and GeneratedDlls
  5821. if (extra.length())
  5822. {
  5823. const char *bufStart = buf.str();
  5824. const char *bufPtr = bufStart+buf.length();
  5825. while (bufStart != bufPtr)
  5826. {
  5827. --bufPtr;
  5828. if (!isspace(*bufPtr))
  5829. break;
  5830. }
  5831. assertex('>' == *bufPtr);
  5832. size_t l = strlen(wuid);
  5833. assertex((size_t)(bufPtr-bufStart) > l+2); // e.g. at least </W20171111-111111>
  5834. bufPtr -= l+2; // skip back over </wuid
  5835. assertex(0 == memcmp(bufPtr, "</", 2) );
  5836. assertex(0 == memcmp(bufPtr+2, wuid, l));
  5837. buf.insert(bufPtr-bufStart, extra);
  5838. }
  5839. return (fileio->write(0,buf.length(),buf.str()) == buf.length());
  5840. }
  5841. bool CLocalWorkUnit::archiveWorkUnit(const char *base, bool del, bool ignoredllerrors, bool deleteOwned, bool exportAssociatedFiles)
  5842. {
  5843. CriticalBlock block(crit);
  5844. StringBuffer path(base);
  5845. if (!p)
  5846. return false;
  5847. const char *wuid = p->queryName();
  5848. if (!wuid||!*wuid)
  5849. return false;
  5850. addPathSepChar(path).append(wuid).append(".xml");
  5851. Owned<IFile> file = createIFile(path.str());
  5852. if (!file)
  5853. return false;
  5854. Owned<IFileIO> fileio = file->open(IFOcreate);
  5855. if (!fileio)
  5856. return false;
  5857. StringBuffer buf;
  5858. exportWorkUnitToXML(this, buf, false, false, true);
  5859. StringBuffer extraWorkUnitXML;
  5860. Owned<IPTree> graphProgress = getGraphProgressTree();
  5861. if (graphProgress)
  5862. {
  5863. toXML(graphProgress,extraWorkUnitXML,1,XML_Format);
  5864. graphProgress.clear();
  5865. }
  5866. Owned<IConstWUQuery> q = getQuery();
  5867. if (!q)
  5868. {
  5869. if (!modifyAndWriteWorkUnitXML(wuid, buf, extraWorkUnitXML, fileio))
  5870. return false;
  5871. if (del)
  5872. {
  5873. if (getState()==WUStateUnknown)
  5874. setState(WUStateArchived); // to allow delete
  5875. cleanupAndDelete(false,deleteOwned); // no query, may as well delete
  5876. }
  5877. return false;
  5878. }
  5879. StringArray deleteExclusions; // associated files not to delete, added if failure to copy
  5880. Owned<IConstWUAssociatedFileIterator> iter = &q->getAssociatedFiles();
  5881. Owned<IPropertyTree> generatedDlls = createPTree("GeneratedDlls");
  5882. ForEach(*iter)
  5883. {
  5884. IConstWUAssociatedFile & cur = iter->query();
  5885. SCMStringBuffer name;
  5886. cur.getNameTail(name);
  5887. if (name.length())
  5888. {
  5889. Owned<IDllEntry> entry = queryDllServer().getEntry(name.str());
  5890. SCMStringBuffer curPath, curIp;
  5891. cur.getName(curPath);
  5892. cur.getIp(curIp);
  5893. SocketEndpoint curEp(curIp.str());
  5894. RemoteFilename curRfn;
  5895. curRfn.setPath(curEp, curPath.str());
  5896. StringBuffer dst(base);
  5897. addPathSepChar(dst);
  5898. curRfn.getTail(dst);
  5899. Owned<IFile> dstFile = createIFile(dst.str());
  5900. if (entry.get())
  5901. {
  5902. Owned<IException> exception;
  5903. Owned<IDllLocation> loc;
  5904. Owned<IPropertyTree> generatedDllBranch = createPTree();
  5905. generatedDllBranch->setProp("@name", entry->queryName());
  5906. generatedDllBranch->setProp("@kind", entry->queryKind());
  5907. if (exportAssociatedFiles)
  5908. {
  5909. try
  5910. {
  5911. loc.setown(entry->getBestLocation()); //throws exception if no readable locations
  5912. }
  5913. catch(IException * e)
  5914. {
  5915. exception.setown(e);
  5916. loc.setown(entry->getBestLocationCandidate()); //this will be closest of the unreadable locations
  5917. }
  5918. RemoteFilename filename;
  5919. loc->getDllFilename(filename);
  5920. if (!exception)
  5921. {
  5922. Owned<IFile> srcfile = createIFile(filename);
  5923. try
  5924. {
  5925. if (dstFile->exists())
  5926. {
  5927. if (streq(srcfile->queryFilename(), dstFile->queryFilename()))
  5928. deleteExclusions.append(name.str()); // restored workunit, referencing archive location for query dll (no longer true post HPCC-11191 fix)
  5929. // still want to delete if already archived but there are source file copies
  5930. }
  5931. else
  5932. copyFile(dstFile, srcfile);
  5933. }
  5934. catch(IException * e)
  5935. {
  5936. exception.setown(e);
  5937. }
  5938. }
  5939. if (exception)
  5940. {
  5941. if (ignoredllerrors)
  5942. {
  5943. EXCLOG(exception.get(), "archiveWorkUnit (copying associated file)");
  5944. //copy failed, so don't delete the registered dll files
  5945. deleteExclusions.append(name.str());
  5946. }
  5947. else
  5948. throw exception.getClear();
  5949. }
  5950. }
  5951. // Record Associated path to restore back to
  5952. StringBuffer restorePath;
  5953. curRfn.getRemotePath(restorePath);
  5954. generatedDllBranch->setProp("@location", restorePath.str());
  5955. generatedDlls->addPropTree("GeneratedDll", generatedDllBranch.getClear());
  5956. }
  5957. else if (exportAssociatedFiles) // no generated dll entry
  5958. {
  5959. Owned<IFile> srcFile = createIFile(curRfn);
  5960. try
  5961. {
  5962. copyFile(dstFile, srcFile);
  5963. }
  5964. catch (IException *e)
  5965. {
  5966. VStringBuffer msg("Failed to archive associated file '%s' to destination '%s'", srcFile->queryFilename(), dstFile->queryFilename());
  5967. EXCLOG(e, msg.str());
  5968. e->Release();
  5969. deleteExclusions.append(name.str());
  5970. }
  5971. }
  5972. }
  5973. }
  5974. iter.clear();
  5975. if (generatedDlls->numChildren())
  5976. toXML(generatedDlls, extraWorkUnitXML, 1, XML_Format);
  5977. if (!modifyAndWriteWorkUnitXML(wuid, buf, extraWorkUnitXML, fileio))
  5978. return false;
  5979. if (del)
  5980. {
  5981. //setState(WUStateArchived); // this isn't useful as about to delete it!
  5982. q.clear();
  5983. cleanupAndDelete(true, deleteOwned, &deleteExclusions);
  5984. }
  5985. return true;
  5986. }
  5987. void CLocalWorkUnit::loadXML(const char *xml)
  5988. {
  5989. CriticalBlock block(crit);
  5990. clearCached(true);
  5991. assertex(xml);
  5992. p.setown(createPTreeFromXMLString(xml,ipt_lowmem));
  5993. }
  5994. void CLocalWorkUnit::serialize(MemoryBuffer &tgt)
  5995. {
  5996. CriticalBlock block(crit);
  5997. StringBuffer x;
  5998. tgt.append(exportWorkUnitToXML(this, x, false, false, false).str());
  5999. }
  6000. void CLocalWorkUnit::deserialize(MemoryBuffer &src)
  6001. {
  6002. CriticalBlock block(crit);
  6003. StringAttr value;
  6004. src.read(value);
  6005. loadXML(value);
  6006. }
  6007. void CLocalWorkUnit::requestAbort()
  6008. {
  6009. CriticalBlock block(crit);
  6010. abortWorkUnit(p->queryName());
  6011. }
  6012. void CLocalWorkUnit::unlockRemote()
  6013. {
  6014. CriticalBlock block(crit);
  6015. locked.unlock();
  6016. _unlockRemote();
  6017. }
  6018. IWorkUnit &CLocalWorkUnit::lockRemote(bool commit)
  6019. {
  6020. if (secMgr)
  6021. checkWuSecAccess(*this, secMgr.get(), secUser.get(), SecAccess_Write, "write lock", true, true);
  6022. locked.lock();
  6023. CriticalBlock block(crit);
  6024. if (commit)
  6025. {
  6026. try
  6027. {
  6028. _lockRemote();
  6029. }
  6030. catch (IException *E)
  6031. {
  6032. StringBuffer s;
  6033. IERRLOG("Failed to get write lock on workunit: %s", E->errorMessage(s).str());
  6034. locked.unlock();
  6035. throw;
  6036. }
  6037. }
  6038. return *new CLockedWorkUnit(LINK(this));
  6039. }
  6040. void CLocalWorkUnit::commit()
  6041. {
  6042. // Nothing to do if not backed by a persistent store
  6043. }
  6044. IWorkUnit& CLocalWorkUnit::lock()
  6045. {
  6046. return lockRemote(true);
  6047. }
  6048. const char *CLocalWorkUnit::queryWuid() const
  6049. {
  6050. CriticalBlock block(crit);
  6051. return p->queryName();
  6052. }
  6053. unsigned CLocalWorkUnit::getDebugAgentListenerPort() const
  6054. {
  6055. CriticalBlock block(crit);
  6056. return p->getPropInt("@DebugListenerPort", 0);
  6057. }
  6058. unsigned CLocalWorkUnit::getTotalThorTime() const
  6059. {
  6060. CriticalBlock block(crit);
  6061. return (unsigned)nanoToMilli(extractTimeCollatable(p->queryProp("@totalThorTime"), false));
  6062. }
  6063. void CLocalWorkUnit::setDebugAgentListenerPort(unsigned port)
  6064. {
  6065. CriticalBlock block(crit);
  6066. p->setPropInt("@DebugListenerPort", port);
  6067. }
  6068. IStringVal& CLocalWorkUnit::getDebugAgentListenerIP(IStringVal &ip) const
  6069. {
  6070. CriticalBlock block(crit);
  6071. ip.set(p->queryProp("@DebugListenerIP"));
  6072. return ip;
  6073. }
  6074. void CLocalWorkUnit::setDebugAgentListenerIP(const char * ip)
  6075. {
  6076. CriticalBlock block(crit);
  6077. p->setProp("@DebugListenerIP", ip);
  6078. }
  6079. IStringVal & CLocalWorkUnit::getWorkunitDistributedAccessToken(IStringVal & datoken) const
  6080. {
  6081. CriticalBlock block(crit);
  6082. datoken.set(p->queryProp("@distributedAccessToken"));
  6083. return datoken;
  6084. }
  6085. bool CLocalWorkUnit::setDistributedAccessToken(const char * user)
  6086. {
  6087. CriticalBlock block(crit);
  6088. //If this format changes, you must update the isWorkunitDAToken() and extractFromWorkunitDAToken() methods
  6089. VStringBuffer datoken("HPCC[u=%s,w=%s]", user, queryWuid());
  6090. IDigitalSignatureManager * pDSM = queryDigitalSignatureManagerInstanceFromEnv();
  6091. if (pDSM && pDSM->isDigiSignerConfigured())
  6092. {
  6093. datoken.append(pDSM->queryKeyName()).append(';');
  6094. StringBuffer b64Signature;
  6095. if (pDSM->digiSign(b64Signature, datoken))
  6096. {
  6097. datoken.append(b64Signature.str());
  6098. }
  6099. else
  6100. {
  6101. ERRLOG("Cannot create workunit Distributed Access Token, digisign failed");
  6102. return false;
  6103. }
  6104. }
  6105. else
  6106. {
  6107. WARNLOG("Cannot sign Distributed Access Token, digisign signing not configured");
  6108. datoken.append(";");
  6109. }
  6110. p->setProp("@distributedAccessToken", datoken);
  6111. return true;
  6112. }
  6113. bool CLocalWorkUnit::getRunningGraph(IStringVal &graphName, WUGraphIDType &subId) const
  6114. {
  6115. // Only implemented in derived classes
  6116. return false;
  6117. }
  6118. void CLocalWorkUnit::setJobName(const char *value)
  6119. {
  6120. CriticalBlock block(crit);
  6121. p->setProp("@jobName", value);
  6122. }
  6123. const char *CLocalWorkUnit::queryJobName() const
  6124. {
  6125. CriticalBlock block(crit);
  6126. const char *ret = p->queryProp("@jobName");
  6127. if (!ret)
  6128. ret = "";
  6129. return ret;
  6130. }
  6131. void CLocalWorkUnit::setClusterName(const char *value)
  6132. {
  6133. CriticalBlock block(crit);
  6134. p->setProp("@clusterName", value);
  6135. }
  6136. const char *CLocalWorkUnit::queryClusterName() const
  6137. {
  6138. CriticalBlock block(crit);
  6139. const char *ret = p->queryProp("@clusterName");
  6140. if (!ret)
  6141. ret = "";
  6142. return ret;
  6143. }
  6144. void CLocalWorkUnit::setAllowedClusters(const char *value)
  6145. {
  6146. setDebugValue("allowedclusters",value, true);
  6147. }
  6148. IStringVal& CLocalWorkUnit::getAllowedClusters(IStringVal &str) const
  6149. {
  6150. CriticalBlock block(crit);
  6151. getDebugValue("allowedclusters",str);
  6152. if (str.length()!=0)
  6153. return str;
  6154. str.set(p->queryProp("@clusterName"));
  6155. return str;
  6156. }
  6157. void CLocalWorkUnit::setAllowAutoQueueSwitch(bool val)
  6158. {
  6159. setDebugValueInt("allowautoqueueswitch",val?1:0,true);
  6160. }
  6161. bool CLocalWorkUnit::getAllowAutoQueueSwitch() const
  6162. {
  6163. CriticalBlock block(crit);
  6164. return getDebugValueBool("allowautoqueueswitch",false);
  6165. }
  6166. void CLocalWorkUnit::setLibraryInformation(const char * name, unsigned interfaceHash, unsigned definitionHash)
  6167. {
  6168. StringBuffer suffix;
  6169. if (name && *name)
  6170. setApplicationValue("LibraryModule", "name", name, true);
  6171. setApplicationValueInt("LibraryModule", "interfaceHash", interfaceHash, true);
  6172. setApplicationValueInt("LibraryModule", "definitionHash", definitionHash, true);
  6173. setApplicationValue("LibraryModule", "platform", appendLibrarySuffix(suffix).str(), true);
  6174. }
  6175. void CLocalWorkUnit::remoteCheckAccess(IUserDescriptor *user, bool writeaccess) const
  6176. {
  6177. unsigned auditflags = DALI_LDAP_AUDIT_REPORT|DALI_LDAP_READ_WANTED;
  6178. if (writeaccess)
  6179. auditflags |= DALI_LDAP_WRITE_WANTED;
  6180. SecAccessFlags perm = SecAccess_Full;
  6181. const char *scopename = p->queryProp("@scope");
  6182. if (scopename&&*scopename) {
  6183. if (!user)
  6184. user = queryUserDescriptor();
  6185. perm = querySessionManager().getPermissionsLDAP("workunit",scopename,user,auditflags);
  6186. if (perm<0) {
  6187. if (perm == SecAccess_Unavailable)
  6188. perm = SecAccess_Full;
  6189. else
  6190. perm = SecAccess_None;
  6191. }
  6192. }
  6193. if (!HASREADPERMISSION(perm))
  6194. throw MakeStringException(WUERR_WorkunitAccessDenied, "Read access denied for workunit %s", queryWuid());
  6195. if (writeaccess && !HASWRITEPERMISSION(perm))
  6196. throw MakeStringException(WUERR_WorkunitAccessDenied, "Write access denied for workunit %s", queryWuid());
  6197. }
  6198. void CLocalWorkUnit::setUser(const char * value)
  6199. {
  6200. CriticalBlock block(crit);
  6201. p->setProp("@submitID", value);
  6202. }
  6203. const char *CLocalWorkUnit::queryUser() const
  6204. {
  6205. CriticalBlock block(crit);
  6206. const char *ret = p->queryProp("@submitID");
  6207. if (!ret)
  6208. ret = "";
  6209. return ret;
  6210. }
  6211. void CLocalWorkUnit::setWuScope(const char * value)
  6212. {
  6213. if (value && *value)
  6214. {
  6215. if (checkWuScopeSecAccess(value, secMgr.get(), secUser.get(), SecAccess_Write, "Change Scope", true, true))
  6216. {
  6217. CriticalBlock block(crit);
  6218. p->setProp("@scope", value);
  6219. }
  6220. }
  6221. }
  6222. const char *CLocalWorkUnit::queryWuScope() const
  6223. {
  6224. CriticalBlock block(crit);
  6225. const char *ret = p->queryProp("@scope");
  6226. if (!ret)
  6227. ret = "";
  6228. return ret;
  6229. }
  6230. void CLocalWorkUnit::setPriority(WUPriorityClass cls)
  6231. {
  6232. CriticalBlock block(crit);
  6233. setEnum(p, "@priorityClass", cls, priorityClasses);
  6234. }
  6235. WUPriorityClass CLocalWorkUnit::getPriority() const
  6236. {
  6237. CriticalBlock block(crit);
  6238. return (WUPriorityClass) getEnum(p, "@priorityClass", priorityClasses);
  6239. }
  6240. const char *CLocalWorkUnit::queryPriorityDesc() const
  6241. {
  6242. return getEnumText(getPriority(), priorityClasses);
  6243. }
  6244. void CLocalWorkUnit::setState(WUState value)
  6245. {
  6246. if (value==WUStateAborted || value==WUStatePaused || value==WUStateCompleted || value==WUStateFailed || value==WUStateSubmitted || value==WUStateWait)
  6247. {
  6248. if (factory)
  6249. factory->clearAborting(queryWuid());
  6250. }
  6251. CriticalBlock block(crit);
  6252. setEnum(p, "@state", value, states); // For historical reasons, we use state to store the state
  6253. setEnum(p, "State", value, states); // But we can only subscribe to elements, not attributes
  6254. if (getDebugValueBool("monitorWorkunit", false))
  6255. {
  6256. switch(value)
  6257. {
  6258. case WUStateAborted:
  6259. FLLOG(MCoperatorWarning, "Workunit %s aborted", p->queryName());
  6260. break;
  6261. case WUStateCompleted:
  6262. FLLOG(MCoperatorProgress, "Workunit %s completed", p->queryName());
  6263. break;
  6264. case WUStateFailed:
  6265. FLLOG(MCoperatorProgress, "Workunit %s failed", p->queryName());
  6266. break;
  6267. }
  6268. }
  6269. p->removeProp("@stateEx");
  6270. }
  6271. void CLocalWorkUnit::setStateEx(const char * text)
  6272. {
  6273. CriticalBlock block(crit);
  6274. p->setProp("@stateEx", text);
  6275. }
  6276. void CLocalWorkUnit::setAgentSession(__int64 sessionId)
  6277. {
  6278. CriticalBlock block(crit);
  6279. p->setPropInt64("@agentSession", sessionId);
  6280. }
  6281. bool CLocalWorkUnit::getIsQueryService() const
  6282. {
  6283. CriticalBlock block(crit);
  6284. return p->getPropBool("@isQueryService", false);
  6285. }
  6286. void CLocalWorkUnit::setIsQueryService(bool value)
  6287. {
  6288. CriticalBlock block(crit);
  6289. p->setPropBool("@isQueryService", value);
  6290. }
  6291. void CLocalWorkUnit::checkAgentRunning(WUState & state)
  6292. {
  6293. if (queryDaliServerVersion().compare("2.1")<0)
  6294. return;
  6295. switch(state)
  6296. {
  6297. case WUStateRunning:
  6298. case WUStateDebugPaused:
  6299. case WUStateDebugRunning:
  6300. case WUStateBlocked:
  6301. case WUStateAborting:
  6302. case WUStateCompiling:
  6303. case WUStatePaused:
  6304. {
  6305. SessionId agent = getAgentSession();
  6306. if((agent>0) && querySessionManager().sessionStopped(agent, 0))
  6307. {
  6308. forceReload();
  6309. state = (WUState) getEnum(p, "@state", states);
  6310. bool isecl=state==WUStateCompiling;
  6311. if (aborting())
  6312. state = WUStateAborted;
  6313. else if (state==WUStateRunning || state==WUStatePaused || state==WUStateDebugPaused || state==WUStateDebugRunning || state==WUStateBlocked || state==WUStateCompiling)
  6314. state = WUStateFailed;
  6315. else
  6316. return;
  6317. WARNLOG("checkAgentRunning terminated: %" I64F "d state = %d",(__int64)agent,(int)state);
  6318. Owned<IWorkUnit> w = &lock();
  6319. w->setState(state);
  6320. Owned<IWUException> e = w->createException();
  6321. WUAction action = w->getAction();
  6322. switch (action)
  6323. {
  6324. case WUActionPause:
  6325. case WUActionPauseNow:
  6326. case WUActionResume:
  6327. w->setAction(WUActionUnknown);
  6328. }
  6329. if(isecl)
  6330. {
  6331. e->setExceptionCode(1001);
  6332. e->setExceptionMessage("EclServer terminated unexpectedly");
  6333. }
  6334. else
  6335. {
  6336. e->setExceptionCode(1000);
  6337. e->setExceptionMessage("Workunit terminated unexpectedly");
  6338. }
  6339. }
  6340. }
  6341. }
  6342. }
  6343. WUState CLocalWorkUnit::getState() const
  6344. {
  6345. CriticalBlock block(crit);
  6346. WUState state = (WUState) getEnum(p, "@state", states);
  6347. switch (state)
  6348. {
  6349. case WUStateRunning:
  6350. case WUStateDebugPaused:
  6351. case WUStateDebugRunning:
  6352. case WUStateBlocked:
  6353. case WUStateCompiling:
  6354. if (aborting())
  6355. state = WUStateAborting;
  6356. break;
  6357. case WUStateSubmitted:
  6358. if (aborting())
  6359. state = WUStateAborted;
  6360. break;
  6361. }
  6362. const_cast<CLocalWorkUnit *>(this)->checkAgentRunning(state); //need const_cast as will change state if agent has died
  6363. return state;
  6364. }
  6365. IStringVal& CLocalWorkUnit::getStateEx(IStringVal & str) const
  6366. {
  6367. CriticalBlock block(crit);
  6368. str.set(p->queryProp("@stateEx"));
  6369. return str;
  6370. }
  6371. __int64 CLocalWorkUnit::getAgentSession() const
  6372. {
  6373. CriticalBlock block(crit);
  6374. return p->getPropInt64("@agentSession", -1);
  6375. }
  6376. unsigned CLocalWorkUnit::getAgentPID() const
  6377. {
  6378. CriticalBlock block(crit);
  6379. return p->getPropInt("@agentPID", -1);
  6380. }
  6381. const char * CLocalWorkUnit::queryStateDesc() const
  6382. {
  6383. // MORE - not sure about this - may prefer a separate interface
  6384. CriticalBlock block(crit);
  6385. try
  6386. {
  6387. return getEnumText(getState(), states);
  6388. }
  6389. catch (...)
  6390. {
  6391. return "???";
  6392. }
  6393. }
  6394. void CLocalWorkUnit::setAction(WUAction value)
  6395. {
  6396. CriticalBlock block(crit);
  6397. setEnum(p, "Action", value, actions);
  6398. }
  6399. WUAction CLocalWorkUnit::getAction() const
  6400. {
  6401. CriticalBlock block(crit);
  6402. return (WUAction) getEnum(p, "Action", actions);
  6403. }
  6404. const char *CLocalWorkUnit::queryActionDesc() const
  6405. {
  6406. CriticalBlock block(crit);
  6407. return p->queryProp("Action");
  6408. }
  6409. IStringVal& CLocalWorkUnit::getApplicationValue(const char *app, const char *propname, IStringVal &str) const
  6410. {
  6411. CriticalBlock block(crit);
  6412. StringBuffer prop("Application/");
  6413. prop.append(app).append('/').append(propname);
  6414. str.set(p->queryProp(prop.str()));
  6415. return str;
  6416. }
  6417. int CLocalWorkUnit::getApplicationValueInt(const char *app, const char *propname, int defVal) const
  6418. {
  6419. CriticalBlock block(crit);
  6420. StringBuffer prop("Application/");
  6421. prop.append(app).append('/').append(propname);
  6422. return p->getPropInt(prop.str(), defVal);
  6423. }
  6424. IConstWUAppValueIterator& CLocalWorkUnit::getApplicationValues() const
  6425. {
  6426. CriticalBlock block(crit);
  6427. appvalues.load(p,"Application/*");
  6428. return *new CArrayIteratorOf<IConstWUAppValue,IConstWUAppValueIterator> (appvalues, 0, (IConstWorkUnit *) this);
  6429. }
  6430. void CLocalWorkUnit::setApplicationValue(const char *app, const char *propname, const char *value, bool overwrite)
  6431. {
  6432. CriticalBlock block(crit);
  6433. StringBuffer prop("Application/");
  6434. prop.append(app).append('/').append(propname);
  6435. if (overwrite || !p->hasProp(prop.str()))
  6436. {
  6437. StringBuffer sp;
  6438. p->setProp(sp.append("Application").str(), "");
  6439. p->setProp(sp.append('/').append(app).str(), "");
  6440. p->setProp(prop.str(), value);
  6441. }
  6442. }
  6443. void CLocalWorkUnit::setApplicationValueInt(const char *app, const char *propname, int value, bool overwrite)
  6444. {
  6445. VStringBuffer s("%d", value);
  6446. setApplicationValue(app, propname, s, overwrite);
  6447. }
  6448. void CLocalWorkUnit::setPriorityLevel(int level)
  6449. {
  6450. CriticalBlock block(crit);
  6451. p->setPropInt("PriorityFlag", level);
  6452. }
  6453. int CLocalWorkUnit::getPriorityLevel() const
  6454. {
  6455. CriticalBlock block(crit);
  6456. return p->getPropInt("PriorityFlag");
  6457. }
  6458. int calcPriorityValue(const IPropertyTree * p)
  6459. {
  6460. int priority = p->getPropInt("PriorityFlag");
  6461. switch((WUPriorityClass) getEnum(p, "@priorityClass", priorityClasses))
  6462. {
  6463. case PriorityClassLow:
  6464. priority -= 100;
  6465. break;
  6466. case PriorityClassHigh:
  6467. priority += 100;
  6468. break;
  6469. }
  6470. return priority;
  6471. }
  6472. int CLocalWorkUnit::getPriorityValue() const
  6473. {
  6474. CriticalBlock block(crit);
  6475. return calcPriorityValue(p);
  6476. }
  6477. void CLocalWorkUnit::setRescheduleFlag(bool value)
  6478. {
  6479. CriticalBlock block(crit);
  6480. p->setPropInt("RescheduleFlag", (int) value);
  6481. }
  6482. bool CLocalWorkUnit::getRescheduleFlag() const
  6483. {
  6484. CriticalBlock block(crit);
  6485. return p->getPropInt("RescheduleFlag") != 0;
  6486. }
  6487. class NullIStringIterator : implements IStringIterator, public CInterface
  6488. {
  6489. public:
  6490. IMPLEMENT_IINTERFACE;
  6491. bool first() { return false; }
  6492. bool next() { return false; }
  6493. bool isValid() { return false; }
  6494. IStringVal & str(IStringVal & str) { return str; }
  6495. };
  6496. ClusterType getClusterType(const char * platform, ClusterType dft)
  6497. {
  6498. if (stricmp(platform, "thor") == 0)
  6499. return ThorLCRCluster;
  6500. if (stricmp(platform, "thorlcr") == 0)
  6501. return ThorLCRCluster;
  6502. if (stricmp(platform, "hthor") == 0)
  6503. return HThorCluster;
  6504. if (stricmp(platform, "roxie") == 0)
  6505. return RoxieCluster;
  6506. return dft;
  6507. }
  6508. const char *clusterTypeString(ClusterType clusterType, bool lcrSensitive)
  6509. {
  6510. switch (clusterType)
  6511. {
  6512. case ThorLCRCluster:
  6513. if (lcrSensitive)
  6514. return "thorlcr";
  6515. return "thor";
  6516. case RoxieCluster:
  6517. return "roxie";
  6518. case HThorCluster:
  6519. return "hthor";
  6520. }
  6521. throwUnexpected();
  6522. }
  6523. bool extractFromWorkunitDAToken(const char * distributedAccessToken, StringBuffer * wuid, StringBuffer * user, StringBuffer * privKey)
  6524. {
  6525. if (!isWorkunitDAToken(distributedAccessToken))
  6526. {
  6527. DBGLOG("Not a valid workunit distributed access token");
  6528. return false;
  6529. }
  6530. //Extract the string between [ and ]
  6531. const char * pEndBracket = strchr(distributedAccessToken + 5, ']');
  6532. StringBuffer tokenValues;
  6533. tokenValues.append(pEndBracket - distributedAccessToken - 5, distributedAccessToken + 5);
  6534. StringArray nameValues;
  6535. nameValues.appendList(tokenValues.str(), ",",true);
  6536. if (user || wuid)
  6537. {
  6538. for (int x = 0; x < nameValues.ordinality(); x++)
  6539. {
  6540. if (user && 0==strncmp(nameValues[x],"u=",2))//Extract user
  6541. user->append(nameValues[x] + 2);
  6542. if (wuid && 0==strncmp(nameValues[x],"w=",2))//Extract wuid
  6543. wuid->append(nameValues[x] + 2);
  6544. }
  6545. }
  6546. if (privKey)
  6547. {
  6548. const char * finger = pEndBracket;
  6549. ++finger;
  6550. while (*finger && *finger != ';')
  6551. privKey->append(1, finger++);
  6552. }
  6553. return true;
  6554. }
  6555. //Verifies the given signed workunit distributed access token was created
  6556. //and signed by the given wuid and user, and that the workunit is still active
  6557. //Returns:
  6558. // 0 : Success, token is valid and workunit is active
  6559. // 1 : Signature does not verify (wuid/username don't match, or signature does not verify)
  6560. // 2 : Workunit not active
  6561. // Throws if unable to open workunit
  6562. wuTokenStates verifyWorkunitDAToken(const char * ctxUser, const char * daToken)
  6563. {
  6564. if (isEmptyString(daToken))
  6565. {
  6566. ERRLOG("verifyWorkunitDAToken : Token must be provided");
  6567. return wuTokenInvalid;
  6568. }
  6569. StringBuffer tokWuid;
  6570. StringBuffer tokUser;
  6571. if (!extractFromWorkunitDAToken(daToken, &tokWuid, &tokUser, nullptr))//get the wuid and user
  6572. {
  6573. //Not a valid workunit distributed access token
  6574. return wuTokenInvalid;
  6575. }
  6576. //Validate signature
  6577. IDigitalSignatureManager * pDSM = queryDigitalSignatureManagerInstanceFromEnv();
  6578. if (pDSM && pDSM->isDigiVerifierConfigured())
  6579. {
  6580. const char * finger;
  6581. StringBuffer token;//receives copy of everything up until signature
  6582. for (finger = daToken; *finger && *finger != ';'; finger++)
  6583. token.append(1, finger);
  6584. token.append(1, finger);//append ;
  6585. StringBuffer sig(++finger);
  6586. if (!pDSM->digiVerify(sig, token))
  6587. {
  6588. ERRLOG("verifyWorkunitDAToken : workunit distributed access token does not verify");
  6589. return wuTokenInvalid;
  6590. }
  6591. }
  6592. Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
  6593. Owned<IConstWorkUnit> cw = factory->openWorkUnit(tokWuid.str());
  6594. if(!cw)
  6595. {
  6596. throw MakeStringException(WUERR_WorkunitAccessDenied,"verifyWorkunitDAToken : Cannot open workunit %s",tokWuid.str());
  6597. }
  6598. //Verify user matches
  6599. bool wuUserExist = !isEmptyString(cw->queryUser());
  6600. bool tokUserExist = !isEmptyString(tokUser.str());
  6601. bool ctxUserExist = !isEmptyString(ctxUser);
  6602. if (wuUserExist && tokUserExist)
  6603. {
  6604. //if both users are found, they must match
  6605. if (!streq(tokUser.str(), cw->queryUser()))
  6606. {
  6607. ERRLOG("verifyWorkunitDAToken : Token user (%s) does not match WU user (%s)", tokUser.str(), cw->queryUser());
  6608. return wuTokenInvalid;//Possible Internal error
  6609. }
  6610. else if (ctxUserExist && !streq(tokUser.str(), ctxUser))//ctxUser will be empty if security not enabled
  6611. {
  6612. ERRLOG("verifyWorkunitDAToken : Token user (%s) does not match Context user (%s)", cw->queryUser(), ctxUser);
  6613. return wuTokenInvalid;
  6614. }
  6615. }
  6616. else if (!wuUserExist && !tokUserExist)//both users will be empty if no security enabled
  6617. {
  6618. if (ctxUserExist)
  6619. {
  6620. ERRLOG("verifyWorkunitDAToken : Security enabled but WU user and Token user not specified");
  6621. return wuTokenInvalid;
  6622. }
  6623. //both users empty and no context user means if no security enabled
  6624. }
  6625. else
  6626. {
  6627. //one user found, but not the other, treat as an error
  6628. ERRLOG("verifyWorkunitDAToken : WU user %s and Token user %s must be provided", wuUserExist ? cw->queryUser() : "(NULL)", tokUserExist ? tokUser.str() : "(NULL)");
  6629. return wuTokenInvalid;
  6630. }
  6631. // no need to compare tokWuid with workunit wuid, because it will always match
  6632. bool wuActive;
  6633. switch (cw->getState())
  6634. {
  6635. case WUStateRunning:
  6636. case WUStateDebugRunning:
  6637. case WUStateBlocked:
  6638. case WUStateAborting:
  6639. case WUStateUploadingFiles:
  6640. #ifdef _DEBUG
  6641. DBGLOG("verifyWorkunitDAToken : Workunit token validated for %s %s, state is '%s'", cw->queryWuid(), cw->queryUser(), getWorkunitStateStr(cw->getState()));
  6642. #endif
  6643. wuActive = true;
  6644. break;
  6645. default:
  6646. ERRLOG("verifyWorkunitDAToken : Workunit %s not active, state is '%s'", cw->queryWuid(), getWorkunitStateStr(cw->getState()));
  6647. wuActive = false;
  6648. break;
  6649. }
  6650. return wuActive ? wuTokenValid : wuTokenWorkunitInactive;
  6651. }
  6652. IPropertyTree *queryRoxieProcessTree(IPropertyTree *environment, const char *process)
  6653. {
  6654. if (!process || !*process)
  6655. return NULL;
  6656. VStringBuffer xpath("Software/RoxieCluster[@name=\"%s\"]", process);
  6657. return environment->queryPropTree(xpath.str());
  6658. }
  6659. void getRoxieProcessServers(IPropertyTree *roxie, SocketEndpointArray &endpoints)
  6660. {
  6661. if (!roxie)
  6662. return;
  6663. Owned<IPropertyTreeIterator> servers = roxie->getElements("RoxieServerProcess");
  6664. ForEach(*servers)
  6665. {
  6666. IPropertyTree &server = servers->query();
  6667. const char *netAddress = server.queryProp("@netAddress");
  6668. if (netAddress && *netAddress)
  6669. {
  6670. SocketEndpoint ep(netAddress, server.getPropInt("@port", 9876));
  6671. endpoints.append(ep);
  6672. }
  6673. }
  6674. }
  6675. void getRoxieProcessServers(const char *process, SocketEndpointArray &servers)
  6676. {
  6677. Owned<IEnvironmentFactory> factory = getEnvironmentFactory(true);
  6678. Owned<IConstEnvironment> env = factory->openEnvironment();
  6679. Owned<IPropertyTree> root = &env->getPTree();
  6680. getRoxieProcessServers(queryRoxieProcessTree(root, process), servers);
  6681. }
  6682. class CEnvironmentClusterInfo: implements IConstWUClusterInfo, public CInterface
  6683. {
  6684. StringAttr name;
  6685. StringAttr alias;
  6686. StringAttr serverQueue;
  6687. StringAttr agentQueue;
  6688. StringAttr agentName;
  6689. StringAttr roxieProcess;
  6690. SocketEndpointArray roxieServers;
  6691. StringAttr thorQueue;
  6692. StringArray thorProcesses;
  6693. StringArray primaryThorProcesses;
  6694. StringAttr prefix;
  6695. StringAttr ldapUser;
  6696. StringBuffer ldapPassword;
  6697. ClusterType platform;
  6698. unsigned clusterWidth;
  6699. unsigned roxieRedundancy;
  6700. unsigned channelsPerNode;
  6701. unsigned numberOfSlaveLogs;
  6702. int roxieReplicateOffset;
  6703. public:
  6704. IMPLEMENT_IINTERFACE;
  6705. CEnvironmentClusterInfo(const char *_name, const char *_prefix, const char *_alias, IPropertyTree *agent, IArrayOf<IPropertyTree> &thors, IPropertyTree *roxie)
  6706. : name(_name), prefix(_prefix), alias(_alias), roxieRedundancy(0), channelsPerNode(0), numberOfSlaveLogs(0), roxieReplicateOffset(1)
  6707. {
  6708. StringBuffer queue;
  6709. if (thors.ordinality())
  6710. {
  6711. thorQueue.set(getClusterThorQueueName(queue.clear(), name));
  6712. clusterWidth = 0;
  6713. bool isMultiThor = (thors.length() > 1);
  6714. ForEachItemIn(i,thors)
  6715. {
  6716. IPropertyTree &thor = thors.item(i);
  6717. const char* thorName = thor.queryProp("@name");
  6718. thorProcesses.append(thorName);
  6719. if (!isMultiThor)
  6720. primaryThorProcesses.append(thorName);
  6721. else
  6722. {
  6723. const char *nodeGroup = thor.queryProp("@nodeGroup");
  6724. if (!nodeGroup || strieq(nodeGroup, thorName))
  6725. primaryThorProcesses.append(thorName);
  6726. }
  6727. unsigned nodes = thor.getCount("ThorSlaveProcess");
  6728. if (!nodes)
  6729. throw MakeStringException(WUERR_MismatchClusterSize,"CEnvironmentClusterInfo: Thor cluster can not have 0 slave processes");
  6730. unsigned slavesPerNode = thor.getPropInt("@slavesPerNode", 1);
  6731. unsigned channelsPerSlave = thor.getPropInt("@channelsPerSlave", 1);
  6732. unsigned ts = nodes * slavesPerNode * channelsPerSlave;
  6733. numberOfSlaveLogs = nodes * slavesPerNode;
  6734. if (clusterWidth && (ts!=clusterWidth))
  6735. throw MakeStringException(WUERR_MismatchClusterSize,"CEnvironmentClusterInfo: mismatched thor sizes in cluster");
  6736. clusterWidth = ts;
  6737. bool islcr = !thor.getPropBool("@Legacy");
  6738. if (!islcr)
  6739. throw MakeStringException(WUERR_MismatchThorType,"CEnvironmentClusterInfo: Legacy Thor no longer supported");
  6740. }
  6741. platform = ThorLCRCluster;
  6742. }
  6743. else if (roxie)
  6744. {
  6745. roxieProcess.set(roxie->queryProp("@name"));
  6746. platform = RoxieCluster;
  6747. getRoxieProcessServers(roxie, roxieServers);
  6748. clusterWidth = roxieServers.length();
  6749. ldapUser.set(roxie->queryProp("@ldapUser"));
  6750. StringBuffer encPassword (roxie->queryProp("@ldapPassword"));
  6751. if (encPassword.length())
  6752. decrypt(ldapPassword, encPassword);
  6753. const char *redundancyMode = roxie->queryProp("@slaveConfig");
  6754. if (redundancyMode && *redundancyMode)
  6755. {
  6756. unsigned dataCopies = roxie->getPropInt("@numDataCopies", 1);
  6757. if (strieq(redundancyMode, "overloaded"))
  6758. channelsPerNode = roxie->getPropInt("@channelsPernode", 1);
  6759. else if (strieq(redundancyMode, "full redundancy"))
  6760. {
  6761. roxieRedundancy = dataCopies-1;
  6762. roxieReplicateOffset = 0;
  6763. }
  6764. else if (strieq(redundancyMode, "cyclic redundancy"))
  6765. {
  6766. roxieRedundancy = dataCopies-1;
  6767. channelsPerNode = dataCopies;
  6768. roxieReplicateOffset = roxie->getPropInt("@cyclicOffset", 1);
  6769. }
  6770. }
  6771. }
  6772. else
  6773. {
  6774. clusterWidth = 1;
  6775. platform = HThorCluster;
  6776. }
  6777. if (agent)
  6778. {
  6779. assertex(!roxie);
  6780. agentQueue.set(getClusterEclAgentQueueName(queue.clear(), name));
  6781. agentName.set(agent->queryProp("@name"));
  6782. }
  6783. else if (roxie)
  6784. agentQueue.set(getClusterRoxieQueueName(queue.clear(), name));
  6785. // MORE - does this need to be conditional?
  6786. serverQueue.set(getClusterEclCCServerQueueName(queue.clear(), name));
  6787. }
  6788. IStringVal & getName(IStringVal & str) const
  6789. {
  6790. str.set(name.get());
  6791. return str;
  6792. }
  6793. const char *getAlias() const
  6794. {
  6795. return alias;
  6796. }
  6797. IStringVal & getScope(IStringVal & str) const
  6798. {
  6799. str.set(prefix.get());
  6800. return str;
  6801. }
  6802. IStringVal & getAgentQueue(IStringVal & str) const
  6803. {
  6804. str.set(agentQueue);
  6805. return str;
  6806. }
  6807. IStringVal & getAgentName(IStringVal & str) const
  6808. {
  6809. str.set(agentName);
  6810. return str;
  6811. }
  6812. virtual IStringVal & getServerQueue(IStringVal & str) const
  6813. {
  6814. str.set(serverQueue);
  6815. return str;
  6816. }
  6817. IStringVal & getThorQueue(IStringVal & str) const
  6818. {
  6819. str.set(thorQueue);
  6820. return str;
  6821. }
  6822. unsigned getSize() const
  6823. {
  6824. return clusterWidth;
  6825. }
  6826. unsigned getNumberOfSlaveLogs() const
  6827. {
  6828. return numberOfSlaveLogs;
  6829. }
  6830. virtual ClusterType getPlatform() const
  6831. {
  6832. return platform;
  6833. }
  6834. IStringVal & getRoxieProcess(IStringVal & str) const
  6835. {
  6836. str.set(roxieProcess.get());
  6837. return str;
  6838. }
  6839. const StringArray & getThorProcesses() const
  6840. {
  6841. return thorProcesses;
  6842. }
  6843. const StringArray & getPrimaryThorProcesses() const
  6844. {
  6845. return primaryThorProcesses;
  6846. }
  6847. const SocketEndpointArray & getRoxieServers() const
  6848. {
  6849. return roxieServers;
  6850. }
  6851. unsigned getRoxieRedundancy() const
  6852. {
  6853. return roxieRedundancy;
  6854. }
  6855. unsigned getChannelsPerNode() const
  6856. {
  6857. return channelsPerNode;
  6858. }
  6859. int getRoxieReplicateOffset() const
  6860. {
  6861. return roxieReplicateOffset;
  6862. }
  6863. const char *getLdapUser() const
  6864. {
  6865. return ldapUser.get();
  6866. }
  6867. virtual const char *getLdapPassword() const
  6868. {
  6869. return ldapPassword.str();
  6870. }
  6871. };
  6872. IStringVal &getProcessQueueNames(IStringVal &ret, const char *process, const char *type, const char *suffix)
  6873. {
  6874. if (process)
  6875. {
  6876. Owned<IEnvironmentFactory> factory = getEnvironmentFactory(true);
  6877. Owned<IConstEnvironment> env = factory->openEnvironment();
  6878. Owned<IPropertyTree> root = &env->getPTree();
  6879. StringBuffer queueNames;
  6880. StringBuffer xpath;
  6881. xpath.appendf("%s[@process=\"%s\"]", type, process);
  6882. Owned<IPropertyTreeIterator> targets = root->getElements("Software/Topology/Cluster");
  6883. ForEach(*targets)
  6884. {
  6885. IPropertyTree &target = targets->query();
  6886. if (target.hasProp(xpath))
  6887. {
  6888. if (queueNames.length())
  6889. queueNames.append(',');
  6890. queueNames.append(target.queryProp("@name")).append(suffix);
  6891. }
  6892. }
  6893. ret.set(queueNames);
  6894. }
  6895. return ret;
  6896. }
  6897. #define ROXIE_QUEUE_EXT ".roxie"
  6898. #define THOR_QUEUE_EXT ".thor"
  6899. #define ECLCCSERVER_QUEUE_EXT ".eclserver"
  6900. #define ECLSERVER_QUEUE_EXT ECLCCSERVER_QUEUE_EXT
  6901. #define ECLSCHEDULER_QUEUE_EXT ".eclscheduler"
  6902. #define ECLAGENT_QUEUE_EXT ".agent"
  6903. extern WORKUNIT_API void getDFUServerQueueNames(StringArray &ret, const char *process)
  6904. {
  6905. Owned<IEnvironmentFactory> factory = getEnvironmentFactory(true);
  6906. Owned<IConstEnvironment> env = factory->openEnvironment();
  6907. StringBuffer xpath ("Software/DfuServerProcess");
  6908. if (!isEmptyString(process))
  6909. xpath.appendf("[@name=\"%s\"]", process);
  6910. Owned<IPropertyTree> root = &env->getPTree();
  6911. Owned<IPropertyTreeIterator> targets = root->getElements(xpath.str());
  6912. ForEach(*targets)
  6913. {
  6914. IPropertyTree &target = targets->query();
  6915. if (target.hasProp("@queue"))
  6916. ret.appendListUniq(target.queryProp("@queue"), ",");
  6917. }
  6918. return;
  6919. }
  6920. extern WORKUNIT_API IStringVal &getEclCCServerQueueNames(IStringVal &ret, const char *process)
  6921. {
  6922. return getProcessQueueNames(ret, process, "EclCCServerProcess", ECLCCSERVER_QUEUE_EXT);
  6923. }
  6924. extern WORKUNIT_API IStringVal &getEclServerQueueNames(IStringVal &ret, const char *process)
  6925. {
  6926. return getProcessQueueNames(ret, process, "EclServerProcess", ECLSERVER_QUEUE_EXT); // shares queue name with EclCCServer
  6927. }
  6928. extern WORKUNIT_API IStringVal &getEclSchedulerQueueNames(IStringVal &ret, const char *process)
  6929. {
  6930. return getProcessQueueNames(ret, process, "EclSchedulerProcess", ECLSCHEDULER_QUEUE_EXT); // Shares deployment/config with EclCCServer
  6931. }
  6932. extern WORKUNIT_API IStringVal &getAgentQueueNames(IStringVal &ret, const char *process)
  6933. {
  6934. return getProcessQueueNames(ret, process, "EclAgentProcess", ECLAGENT_QUEUE_EXT);
  6935. }
  6936. extern WORKUNIT_API IStringVal &getRoxieQueueNames(IStringVal &ret, const char *process)
  6937. {
  6938. return getProcessQueueNames(ret, process, "RoxieCluster", ROXIE_QUEUE_EXT);
  6939. }
  6940. extern WORKUNIT_API IStringVal &getThorQueueNames(IStringVal &ret, const char *process)
  6941. {
  6942. return getProcessQueueNames(ret, process, "ThorCluster", THOR_QUEUE_EXT);
  6943. }
  6944. extern WORKUNIT_API StringBuffer &getClusterThorQueueName(StringBuffer &ret, const char *cluster)
  6945. {
  6946. return ret.append(cluster).append(THOR_QUEUE_EXT);
  6947. }
  6948. extern WORKUNIT_API StringBuffer &getClusterThorGroupName(StringBuffer &ret, const char *cluster)
  6949. {
  6950. Owned<IEnvironmentFactory> factory = getEnvironmentFactory(true);
  6951. Owned<IConstEnvironment> env = factory->openEnvironment();
  6952. Owned<IPropertyTree> root = &env->getPTree();
  6953. StringBuffer path;
  6954. path.append("Software/ThorCluster[@name=\"").append(cluster).append("\"]");
  6955. IPropertyTree * child = root->queryPropTree(path);
  6956. if (child)
  6957. getClusterGroupName(*child, ret);
  6958. return ret;
  6959. }
  6960. extern WORKUNIT_API StringBuffer &getClusterGroupName(StringBuffer &ret, const char *cluster)
  6961. {
  6962. Owned<IEnvironmentFactory> factory = getEnvironmentFactory(true);
  6963. Owned<IConstEnvironment> env = factory->openEnvironment();
  6964. Owned<IPropertyTree> root = &env->getPTree();
  6965. StringBuffer path;
  6966. path.set("Software/ThorCluster[@name=\"").append(cluster).append("\"]");
  6967. IPropertyTree * child = root->queryPropTree(path);
  6968. if (child)
  6969. {
  6970. return getClusterGroupName(*child, ret);
  6971. }
  6972. path.set("Software/RoxieCluster[@name=\"").append(cluster).append("\"]");
  6973. child = root->queryPropTree(path);
  6974. if (child)
  6975. {
  6976. return getClusterGroupName(*child, ret);
  6977. }
  6978. path.set("Software/EclAgentProcess[@name=\"").append(cluster).append("\"]");
  6979. child = root->queryPropTree(path);
  6980. if (child)
  6981. {
  6982. return ret.setf("hthor__%s", cluster);
  6983. }
  6984. return ret;
  6985. }
  6986. extern WORKUNIT_API ClusterType getClusterTypeByClusterName(const char *cluster)
  6987. {
  6988. Owned<IEnvironmentFactory> factory = getEnvironmentFactory(true);
  6989. Owned<IConstEnvironment> env = factory->openEnvironment();
  6990. Owned<IPropertyTree> root = &env->getPTree();
  6991. StringBuffer path;
  6992. path.set("Software/ThorCluster[@name=\"").append(cluster).append("\"]");
  6993. if (root->hasProp(path))
  6994. return ThorLCRCluster;
  6995. path.set("Software/RoxieCluster[@name=\"").append(cluster).append("\"]");
  6996. if (root->hasProp(path))
  6997. return RoxieCluster;
  6998. path.set("Software/EclAgentProcess[@name=\"").append(cluster).append("\"]");
  6999. if (root->hasProp(path))
  7000. return HThorCluster;
  7001. return NoCluster;
  7002. }
  7003. extern WORKUNIT_API StringBuffer &getClusterRoxieQueueName(StringBuffer &ret, const char *cluster)
  7004. {
  7005. return ret.append(cluster).append(ROXIE_QUEUE_EXT);
  7006. }
  7007. extern WORKUNIT_API StringBuffer &getClusterEclCCServerQueueName(StringBuffer &ret, const char *cluster)
  7008. {
  7009. return ret.append(cluster).append(ECLCCSERVER_QUEUE_EXT);
  7010. }
  7011. extern WORKUNIT_API StringBuffer &getClusterEclServerQueueName(StringBuffer &ret, const char *cluster)
  7012. {
  7013. return ret.append(cluster).append(ECLSERVER_QUEUE_EXT);
  7014. }
  7015. extern WORKUNIT_API StringBuffer &getClusterEclAgentQueueName(StringBuffer &ret, const char *cluster)
  7016. {
  7017. return ret.append(cluster).append(ECLAGENT_QUEUE_EXT);
  7018. }
  7019. extern WORKUNIT_API IStringIterator *getTargetClusters(const char *processType, const char *processName)
  7020. {
  7021. Owned<CStringArrayIterator> ret = new CStringArrayIterator;
  7022. Owned<IEnvironmentFactory> factory = getEnvironmentFactory(true);
  7023. Owned<IConstEnvironment> env = factory->openEnvironment();
  7024. Owned<IPropertyTree> root = &env->getPTree();
  7025. StringBuffer xpath;
  7026. xpath.appendf("%s", processType ? processType : "*");
  7027. if (processName && *processName)
  7028. xpath.appendf("[@process=\"%s\"]", processName);
  7029. Owned<IPropertyTreeIterator> targets = root->getElements("Software/Topology/Cluster");
  7030. ForEach(*targets)
  7031. {
  7032. IPropertyTree &target = targets->query();
  7033. if (target.hasProp(xpath))
  7034. {
  7035. ret->append(target.queryProp("@name"));
  7036. }
  7037. }
  7038. return ret.getClear();
  7039. }
  7040. extern WORKUNIT_API bool isProcessCluster(const char *process)
  7041. {
  7042. if (!process || !*process)
  7043. return false;
  7044. Owned<IEnvironmentFactory> factory = getEnvironmentFactory(true);
  7045. Owned<IConstEnvironment> env = factory->openEnvironment();
  7046. Owned<IPropertyTree> root = &env->getPTree();
  7047. VStringBuffer xpath("Software/*Cluster[@name=\"%s\"]", process);
  7048. return root->hasProp(xpath.str());
  7049. }
  7050. extern WORKUNIT_API bool isProcessCluster(const char *remoteDali, const char *process)
  7051. {
  7052. if (!remoteDali || !*remoteDali)
  7053. return isProcessCluster(process);
  7054. if (!process || !*process)
  7055. return false;
  7056. Owned<INode> remote = createINode(remoteDali, 7070);
  7057. if (!remote)
  7058. return false;
  7059. //Cannot use getEnvironmentFactory() since it is using a remotedali
  7060. VStringBuffer xpath("Environment/Software/*Cluster[@name=\"%s\"]/@name", process);
  7061. try
  7062. {
  7063. Owned<IPropertyTreeIterator> clusters = querySDS().getElementsRaw(xpath, remote, 1000*60*1);
  7064. return clusters->first();
  7065. }
  7066. catch (IException *E)
  7067. {
  7068. StringBuffer msg;
  7069. E->errorMessage(msg);
  7070. DBGLOG("Exception validating cluster %s/%s: %s", remoteDali, xpath.str(), msg.str());
  7071. E->Release();
  7072. }
  7073. return true;
  7074. }
  7075. IConstWUClusterInfo* getTargetClusterInfo(IPropertyTree *environment, IPropertyTree *cluster)
  7076. {
  7077. const char *clustname = cluster->queryProp("@name");
  7078. // MORE - at the moment configenf specifies eclagent and thor queues by (in effect) placing an 'example' thor or eclagent in the topology
  7079. // that uses the queue that will be used.
  7080. // We should and I hope will change that, at which point the code below gets simpler
  7081. StringBuffer prefix(cluster->queryProp("@prefix"));
  7082. prefix.toLowerCase();
  7083. StringBuffer xpath;
  7084. StringBuffer querySetName;
  7085. IPropertyTree *agent = NULL;
  7086. const char *agentName = cluster->queryProp("EclAgentProcess/@process");
  7087. if (agentName)
  7088. {
  7089. xpath.clear().appendf("Software/EclAgentProcess[@name=\"%s\"]", agentName);
  7090. agent = environment->queryPropTree(xpath.str());
  7091. }
  7092. Owned<IPropertyTreeIterator> ti = cluster->getElements("ThorCluster");
  7093. IArrayOf<IPropertyTree> thors;
  7094. ForEach(*ti)
  7095. {
  7096. const char *thorName = ti->query().queryProp("@process");
  7097. if (thorName)
  7098. {
  7099. xpath.clear().appendf("Software/ThorCluster[@name=\"%s\"]", thorName);
  7100. thors.append(*environment->getPropTree(xpath.str()));
  7101. }
  7102. }
  7103. const char *roxieName = cluster->queryProp("RoxieCluster/@process");
  7104. return new CEnvironmentClusterInfo(clustname, prefix, cluster->queryProp("@alias"), agent, thors, queryRoxieProcessTree(environment, roxieName));
  7105. }
  7106. IPropertyTree* getTopologyCluster(Owned<IPropertyTree> &envRoot, const char *clustname)
  7107. {
  7108. if (!clustname || !*clustname)
  7109. return NULL;
  7110. Owned<IEnvironmentFactory> factory = getEnvironmentFactory(true);
  7111. Owned<IConstEnvironment> env = factory->openEnvironment();
  7112. envRoot.setown(&env->getPTree());
  7113. StringBuffer xpath;
  7114. xpath.appendf("Software/Topology/Cluster[@name=\"%s\"]", clustname);
  7115. return envRoot->getPropTree(xpath.str());
  7116. }
  7117. bool validateTargetClusterName(const char *clustname)
  7118. {
  7119. Owned<IPropertyTree> envRoot;
  7120. Owned<IPropertyTree> cluster = getTopologyCluster(envRoot, clustname);
  7121. return (cluster.get()!=NULL);
  7122. }
  7123. IConstWUClusterInfo* getTargetClusterInfo(const char *clustname)
  7124. {
  7125. Owned<IPropertyTree> envRoot;
  7126. Owned<IPropertyTree> cluster = getTopologyCluster(envRoot, clustname);
  7127. if (!cluster)
  7128. return NULL;
  7129. return getTargetClusterInfo(envRoot, cluster);
  7130. }
  7131. unsigned getEnvironmentClusterInfo(CConstWUClusterInfoArray &clusters)
  7132. {
  7133. Owned<IEnvironmentFactory> factory = getEnvironmentFactory(true);
  7134. Owned<IConstEnvironment> env = factory->openEnvironment();
  7135. Owned<IPropertyTree> root = &env->getPTree();
  7136. return getEnvironmentClusterInfo(root, clusters);
  7137. }
  7138. unsigned getEnvironmentClusterInfo(IPropertyTree* environmentRoot, CConstWUClusterInfoArray &clusters)
  7139. {
  7140. if (!environmentRoot)
  7141. return 0;
  7142. Owned<IPropertyTreeIterator> clusterIter = environmentRoot->getElements("Software/Topology/Cluster");
  7143. ForEach(*clusterIter)
  7144. {
  7145. IPropertyTree &node = clusterIter->query();
  7146. Owned<IConstWUClusterInfo> cluster = getTargetClusterInfo(environmentRoot, &node);
  7147. clusters.append(*cluster.getClear());
  7148. }
  7149. return clusters.ordinality();
  7150. }
  7151. const char *getTargetClusterComponentName(const char *clustname, const char *processType, StringBuffer &name)
  7152. {
  7153. if (!clustname)
  7154. return NULL;
  7155. Owned<IEnvironmentFactory> factory = getEnvironmentFactory(true);
  7156. Owned<IConstEnvironment> env = factory->openEnvironment();
  7157. Owned<IPropertyTree> root = &env->getPTree();
  7158. StringBuffer xpath;
  7159. xpath.appendf("Software/Topology/Cluster[@name=\"%s\"]", clustname);
  7160. Owned<IPropertyTree> cluster = root->getPropTree(xpath.str());
  7161. if (!cluster)
  7162. return NULL;
  7163. StringBuffer xpath1;
  7164. xpath1.appendf("%s/@process", processType);
  7165. name.append(cluster->queryProp(xpath1.str()));
  7166. return name.str();
  7167. }
  7168. unsigned getEnvironmentThorClusterNames(StringArray &thorNames, StringArray &groupNames, StringArray &targetNames, StringArray &queueNames)
  7169. {
  7170. Owned<IEnvironmentFactory> factory = getEnvironmentFactory(true);
  7171. Owned<IConstEnvironment> env = factory->openEnvironment();
  7172. Owned<IPropertyTree> root = &env->getPTree();
  7173. Owned<IPropertyTreeIterator> allTargets = root->getElements("Software/Topology/Cluster");
  7174. ForEach(*allTargets)
  7175. {
  7176. IPropertyTree &target = allTargets->query();
  7177. const char *targetName = target.queryProp("@name");
  7178. if (targetName && *targetName)
  7179. {
  7180. Owned<IPropertyTreeIterator> thorClusters = target.getElements("ThorCluster");
  7181. ForEach(*thorClusters)
  7182. {
  7183. const char *thorName = thorClusters->query().queryProp("@process");
  7184. VStringBuffer query("Software/ThorCluster[@name=\"%s\"]",thorName);
  7185. IPropertyTree *thorCluster = root->queryPropTree(query.str());
  7186. if (thorCluster)
  7187. {
  7188. const char *groupName = thorCluster->queryProp("@nodeGroup");
  7189. if (!groupName||!*groupName)
  7190. groupName = thorName;
  7191. thorNames.append(thorName);
  7192. groupNames.append(groupName);
  7193. targetNames.append(targetName);
  7194. StringBuffer queueName(targetName);
  7195. queueNames.append(queueName.append(THOR_QUEUE_EXT));
  7196. }
  7197. }
  7198. }
  7199. }
  7200. return thorNames.ordinality();
  7201. }
  7202. unsigned getEnvironmentHThorClusterNames(StringArray &eclAgentNames, StringArray &groupNames, StringArray &targetNames)
  7203. {
  7204. Owned<IEnvironmentFactory> factory = getEnvironmentFactory(true);
  7205. Owned<IConstEnvironment> env = factory->openEnvironment();
  7206. Owned<IPropertyTree> root = &env->getPTree();
  7207. Owned<IPropertyTreeIterator> allEclAgents = root->getElements("Software/EclAgentProcess");
  7208. ForEach(*allEclAgents)
  7209. {
  7210. IPropertyTree &eclAgent = allEclAgents->query();
  7211. const char *eclAgentName = eclAgent.queryProp("@name");
  7212. if (eclAgentName && *eclAgentName)
  7213. {
  7214. Owned<IPropertyTreeIterator> allTargets = root->getElements("Software/Topology/Cluster");
  7215. ForEach(*allTargets)
  7216. {
  7217. IPropertyTree &target = allTargets->query();
  7218. const char *targetName = target.queryProp("@name");
  7219. if (targetName && *targetName)
  7220. {
  7221. StringBuffer xpath;
  7222. xpath.appendf("EclAgentProcess[@process=\"%s\"]", eclAgentName);
  7223. if (target.hasProp(xpath) && !target.hasProp("ThorCluster"))
  7224. {
  7225. StringBuffer groupName("hthor__");
  7226. groupName.append(eclAgentName);
  7227. groupNames.append(groupName);
  7228. eclAgentNames.append(eclAgentName);
  7229. targetNames.append(targetName);
  7230. }
  7231. }
  7232. }
  7233. }
  7234. }
  7235. return eclAgentNames.ordinality();
  7236. }
  7237. IStringVal& CLocalWorkUnit::getScope(IStringVal &str) const
  7238. {
  7239. CriticalBlock block(crit);
  7240. if (p->hasProp("Debug/ForceScope"))
  7241. {
  7242. StringBuffer prefix(p->queryProp("Debug/ForceScope"));
  7243. str.set(prefix.toLowerCase().str());
  7244. }
  7245. else
  7246. {
  7247. Owned <IConstWUClusterInfo> ci = getTargetClusterInfo(p->queryProp("@clusterName"));
  7248. if (ci)
  7249. ci->getScope(str);
  7250. else
  7251. str.clear();
  7252. }
  7253. return str;
  7254. }
  7255. //Queries
  7256. void CLocalWorkUnit::setCodeVersion(unsigned codeVersion, const char * buildVersion, const char * eclVersion)
  7257. {
  7258. CriticalBlock block(crit);
  7259. p->setPropInt("@codeVersion", codeVersion);
  7260. p->setProp("@buildVersion", buildVersion);
  7261. p->setProp("@eclVersion", eclVersion);
  7262. }
  7263. unsigned CLocalWorkUnit::getCodeVersion() const
  7264. {
  7265. CriticalBlock block(crit);
  7266. return p->getPropInt("@codeVersion");
  7267. }
  7268. unsigned CLocalWorkUnit::getWuidVersion() const
  7269. {
  7270. CriticalBlock block(crit);
  7271. return p->getPropInt("@wuidVersion");
  7272. }
  7273. void CLocalWorkUnit::getBuildVersion(IStringVal & buildVersion, IStringVal & eclVersion) const
  7274. {
  7275. CriticalBlock block(crit);
  7276. buildVersion.set(p->queryProp("@buildVersion"));
  7277. eclVersion.set(p->queryProp("@eclVersion"));
  7278. }
  7279. void CLocalWorkUnit::setCloneable(bool value)
  7280. {
  7281. CriticalBlock block(crit);
  7282. p->setPropInt("@cloneable", value);
  7283. }
  7284. void CLocalWorkUnit::setIsClone(bool value)
  7285. {
  7286. CriticalBlock block(crit);
  7287. p->setPropInt("@isClone", value);
  7288. }
  7289. bool CLocalWorkUnit::getCloneable() const
  7290. {
  7291. CriticalBlock block(crit);
  7292. return p->getPropBool("@cloneable", false);
  7293. }
  7294. IUserDescriptor *CLocalWorkUnit::queryUserDescriptor() const
  7295. {
  7296. CriticalBlock block(crit);
  7297. if (!userDesc)
  7298. {
  7299. userDesc.setown(createUserDescriptor());
  7300. SCMStringBuffer token;
  7301. getWorkunitDistributedAccessToken(token);
  7302. userDesc->set(queryUser(), token.str());//use token as password
  7303. }
  7304. return userDesc;
  7305. }
  7306. bool CLocalWorkUnit::isProtected() const
  7307. {
  7308. CriticalBlock block(crit);
  7309. return p->getPropBool("@protected", false);
  7310. }
  7311. bool CLocalWorkUnit::isPausing() const
  7312. {
  7313. CriticalBlock block(crit);
  7314. if (WUActionPause == getAction())
  7315. {
  7316. switch (getState())
  7317. {
  7318. case WUStateRunning:
  7319. case WUStateAborting:
  7320. return true;
  7321. }
  7322. }
  7323. return false;
  7324. }
  7325. void CLocalWorkUnit::protect(bool protectMode)
  7326. {
  7327. CriticalBlock block(crit);
  7328. p->setPropBool("@protected", protectMode);
  7329. }
  7330. void CLocalWorkUnit::setResultLimit(unsigned value)
  7331. {
  7332. CriticalBlock block(crit);
  7333. p->setPropInt("resultLimit", value);
  7334. }
  7335. unsigned CLocalWorkUnit::getResultLimit() const
  7336. {
  7337. CriticalBlock block(crit);
  7338. return p->getPropInt("resultLimit");
  7339. }
  7340. IStringVal & CLocalWorkUnit::getSnapshot(IStringVal & str) const
  7341. {
  7342. CriticalBlock block(crit);
  7343. str.set(p->queryProp("SNAPSHOT"));
  7344. return str;
  7345. }
  7346. void CLocalWorkUnit::setSnapshot(const char * val)
  7347. {
  7348. CriticalBlock block(crit);
  7349. p->setProp("SNAPSHOT", val);
  7350. }
  7351. const static EnumMapping warningSeverityMap[] =
  7352. {
  7353. { SeverityInformation, "info" },
  7354. { SeverityWarning, "warning" },
  7355. { SeverityError, "error" },
  7356. { SeverityAlert, "alert" },
  7357. { SeverityIgnore, "ignore" },
  7358. { SeverityFatal, "fatal" },
  7359. { SeverityUnknown, NULL }
  7360. };
  7361. ErrorSeverity CLocalWorkUnit::getWarningSeverity(unsigned code, ErrorSeverity defaultSeverity) const
  7362. {
  7363. StringBuffer xpath;
  7364. xpath.append("OnWarnings/OnWarning[@code='").append(code).append("']");
  7365. CriticalBlock block(crit);
  7366. IPropertyTree * mapping = p->queryPropTree(xpath);
  7367. if (mapping)
  7368. return (ErrorSeverity) getEnum(mapping, "@severity", warningSeverityMap);
  7369. return defaultSeverity;
  7370. }
  7371. void CLocalWorkUnit::setWarningSeverity(unsigned code, ErrorSeverity severity)
  7372. {
  7373. StringBuffer xpath;
  7374. xpath.append("OnWarnings/OnWarning[@code='").append(code).append("']");
  7375. CriticalBlock block(crit);
  7376. IPropertyTree * mapping = p->queryPropTree(xpath);
  7377. if (!mapping)
  7378. {
  7379. IPropertyTree * onWarnings = ensurePTree(p, "OnWarnings");
  7380. mapping = onWarnings->addPropTree("OnWarning");
  7381. mapping->setPropInt("@code", code);
  7382. }
  7383. setEnum(mapping, "@severity", severity, warningSeverityMap);
  7384. }
  7385. static int comparePropTrees(IInterface * const *ll, IInterface * const *rr)
  7386. {
  7387. IPropertyTree *l = (IPropertyTree *) *ll;
  7388. IPropertyTree *r = (IPropertyTree *) *rr;
  7389. return stricmp(l->queryName(), r->queryName());
  7390. };
  7391. unsigned CLocalWorkUnit::calculateHash(unsigned crc)
  7392. {
  7393. // Any other values in the WU that could affect generated code should be crc'ed here
  7394. IPropertyTree *tree = p->queryBranch("Debug");
  7395. if (tree)
  7396. {
  7397. Owned<IPropertyTreeIterator> sub = tree->getElements("*");
  7398. ICopyArrayOf<IPropertyTree> subs;
  7399. for(sub->first(); sub->isValid(); sub->next())
  7400. subs.append(sub->query());
  7401. subs.sort(comparePropTrees);
  7402. ForEachItemIn(idx, subs)
  7403. {
  7404. const char *name = subs.item(idx).queryName();
  7405. const char *val = subs.item(idx).queryProp(NULL);
  7406. crc = crc32(name, (size32_t)strlen(name), crc);
  7407. if (val)
  7408. crc = crc32(val, (size32_t)strlen(val), crc);
  7409. }
  7410. }
  7411. Owned<IConstWUPluginIterator> plugins = &getPlugins();
  7412. for (plugins->first();plugins->isValid();plugins->next())
  7413. {
  7414. IConstWUPlugin &thisplugin = plugins->query();
  7415. SCMStringBuffer version;
  7416. thisplugin.getPluginVersion(version);
  7417. crc = crc32(version.str(), version.length(), crc);
  7418. }
  7419. return crc;
  7420. }
  7421. static void updateProp(IPropertyTree * to, const IPropertyTree * from, const char * xpath)
  7422. {
  7423. if (!to->hasProp(xpath) && from->hasProp(xpath))
  7424. to->setProp(xpath, from->queryProp(xpath));
  7425. }
  7426. static void setProp(IPropertyTree * to, const IPropertyTree * from, const char * xpath)
  7427. {
  7428. if (from->hasProp(xpath))
  7429. to->setProp(xpath, from->queryProp(xpath));
  7430. }
  7431. static void copyTree(IPropertyTree * to, const IPropertyTree * from, const char * xpath)
  7432. {
  7433. IPropertyTree * match = from->getBranch(xpath);
  7434. if (match)
  7435. to->setPropTree(xpath, match);
  7436. }
  7437. IPropertyTree *CLocalWorkUnit::queryPTree() const
  7438. {
  7439. return p;
  7440. }
  7441. void CLocalWorkUnit::copyWorkUnit(IConstWorkUnit *cached, bool copyStats, bool all)
  7442. {
  7443. CLocalWorkUnit *from = QUERYINTERFACE(cached, CLocalWorkUnit);
  7444. if (!from)
  7445. {
  7446. CLockedWorkUnit *fl = QUERYINTERFACE(cached, CLockedWorkUnit);
  7447. if (!fl)
  7448. throw MakeStringException(WUERR_InternalUnknownImplementation, "Cached workunit not created using workunit dll");
  7449. from = fl->c;
  7450. }
  7451. // Need to copy the query, the results, and the graphs from the cached query.
  7452. // The cache is made before the query is executed so there is no need to clear them.
  7453. if (!cached->getCloneable())
  7454. throw MakeStringException(WUERR_CannotCloneWorkunit, "Source work unit not marked as clonable");
  7455. const IPropertyTree * fromP = from->p;
  7456. IPropertyTree *pt;
  7457. CriticalBlock block(crit);
  7458. clearCached(false);
  7459. query.clear();
  7460. updateProp(p, fromP, "@jobName");
  7461. copyTree(p, fromP, "Query");
  7462. pt = fromP->getBranch("Application/LibraryModule");
  7463. if (pt)
  7464. {
  7465. ensurePTree(p, "Application");
  7466. p->setPropTree("Application/LibraryModule", pt);
  7467. }
  7468. pt = fromP->queryBranch("Debug");
  7469. if (pt)
  7470. {
  7471. IPropertyTree *curDebug = p->queryPropTree("Debug");
  7472. if (curDebug)
  7473. {
  7474. Owned<IPropertyTreeIterator> elems = pt->getElements("*");
  7475. ForEach(*elems)
  7476. {
  7477. IPropertyTree *elem = &elems->query();
  7478. if (!curDebug->hasProp(elem->queryName()))
  7479. curDebug->setPropTree(elem->queryName(),LINK(elem));
  7480. }
  7481. }
  7482. else
  7483. p->setPropTree("Debug", LINK(pt));
  7484. }
  7485. copyTree(p, fromP, "OnWarnings");
  7486. copyTree(p, fromP, "Plugins");
  7487. copyTree(p, fromP, "Libraries");
  7488. copyTree(p, fromP, "Results");
  7489. copyTree(p, fromP, "Graphs");
  7490. copyTree(p, fromP, "Workflow");
  7491. copyTree(p, fromP, "WebServicesInfo");
  7492. if (copyStats)
  7493. {
  7494. // Merge timing info from both branches
  7495. pt = fromP->getBranch("Statistics");
  7496. if (pt)
  7497. {
  7498. IPropertyTree *tgtStatistics = ensurePTree(p, "Statistics");
  7499. mergePTree(tgtStatistics, pt);
  7500. pt->Release();
  7501. }
  7502. }
  7503. updateProp(p, fromP, "@clusterName");
  7504. updateProp(p, fromP, "allowedclusters");
  7505. updateProp(p, fromP, "@submitID");
  7506. updateProp(p, fromP, "SNAPSHOT");
  7507. setProp(p, fromP, "@eventScheduledCount");
  7508. //MORE: This is very adhoc. All options that should be cloned should really be in a common branch
  7509. if (all)
  7510. {
  7511. setProp(p, fromP, "PriorityFlag");
  7512. setProp(p, fromP, "@priorityClass");
  7513. setProp(p, fromP, "@protected");
  7514. setProp(p, fromP, "@clusterName");
  7515. updateProp(p, fromP, "@scope");
  7516. }
  7517. //Variables may have been set up as parameters to the query - so need to preserve any values that were supplied.
  7518. pt = fromP->getBranch("Variables");
  7519. if (pt)
  7520. {
  7521. IPropertyTree *ptTgtVariables = ensurePTree(p, "Variables");
  7522. Owned<IPropertyTreeIterator> ptiVariable = pt->getElements("Variable");
  7523. for (ptiVariable->first(); ptiVariable->isValid(); ptiVariable->next())
  7524. {
  7525. IPropertyTree *ptSrcVariable = &ptiVariable->query();
  7526. const char *name = ptSrcVariable->queryProp("@name");
  7527. assertex(name);
  7528. StringBuffer xpath;
  7529. xpath.append("Variable[@name='").append(name).append("']");
  7530. IPropertyTree *ptTgtVariable = ptTgtVariables->queryPropTree(xpath.str());
  7531. IPropertyTree *merged = createPTreeFromIPT(ptSrcVariable); // clone entire source info...
  7532. merged->removeProp("Value"); // except value and status
  7533. merged->setProp("@status", "undefined");
  7534. if (!merged->getPropBool("@isScalar"))
  7535. merged->removeProp("totalRowCount");
  7536. merged->removeProp("rowCount");
  7537. // If there are any other fields that get set ONLY by eclagent, strip them out here...
  7538. if (ptTgtVariable)
  7539. {
  7540. // copy status and Value from what is already set in target
  7541. merged->setProp("@status", ptTgtVariable->queryProp("@status"));
  7542. MemoryBuffer value;
  7543. if (ptTgtVariable->getPropBin("Value", value))
  7544. merged->setPropBin("Value", value.length(), value.toByteArray());
  7545. ptTgtVariable->removeProp(xpath.str());
  7546. // If there are any other fields in a variable that get set by ws_ecl before submitting, copy them across here...
  7547. }
  7548. ptTgtVariables->addPropTree("Variable", merged);
  7549. }
  7550. pt->Release();
  7551. }
  7552. p->setProp("@codeVersion", fromP->queryProp("@codeVersion"));
  7553. p->setProp("@buildVersion", fromP->queryProp("@buildVersion"));
  7554. p->setProp("@eclVersion", fromP->queryProp("@eclVersion"));
  7555. p->setProp("@totalThorTime", fromP->queryProp("@totalThorTime"));
  7556. p->setProp("@hash", fromP->queryProp("@hash"));
  7557. p->setPropBool("@cloneable", true);
  7558. p->setPropBool("@isClone", true);
  7559. resetWorkflow(); // the source Workflow section may have had some parts already executed...
  7560. Owned<IPropertyTreeIterator> results = p->getElements("Results/Result");
  7561. ForEach(*results)
  7562. {
  7563. CLocalWUResult result(LINK(&results->query()));
  7564. result.setResultStatus(ResultStatusUndefined);
  7565. }
  7566. copyTree(p, fromP, "usedsources"); // field usage
  7567. }
  7568. bool CLocalWorkUnit::hasDebugValue(const char *propname) const
  7569. {
  7570. StringBuffer lower;
  7571. lower.append(propname).toLowerCase();
  7572. CriticalBlock block(crit);
  7573. StringBuffer prop("Debug/");
  7574. return p->hasProp(prop.append(lower));
  7575. }
  7576. IStringVal& CLocalWorkUnit::getDebugValue(const char *propname, IStringVal &str) const
  7577. {
  7578. StringBuffer lower;
  7579. lower.append(propname).toLowerCase();
  7580. CriticalBlock block(crit);
  7581. StringBuffer prop("Debug/");
  7582. str.set(p->queryProp(prop.append(lower).str()));
  7583. return str;
  7584. }
  7585. IStringIterator& CLocalWorkUnit::getDebugValues() const
  7586. {
  7587. return getDebugValues(NULL);
  7588. }
  7589. IStringIterator& CLocalWorkUnit::getDebugValues(const char *prop) const
  7590. {
  7591. CriticalBlock block(crit);
  7592. StringBuffer path("Debug/");
  7593. if (prop)
  7594. {
  7595. StringBuffer lower;
  7596. lower.append(prop).toLowerCase();
  7597. path.append(lower);
  7598. }
  7599. else
  7600. path.append("*");
  7601. return *new CStringPTreeTagIterator(p->getElements(path.str()));
  7602. }
  7603. int CLocalWorkUnit::getDebugValueInt(const char *propname, int defVal) const
  7604. {
  7605. StringBuffer lower;
  7606. lower.append(propname).toLowerCase();
  7607. CriticalBlock block(crit);
  7608. StringBuffer prop("Debug/");
  7609. prop.append(lower);
  7610. return p->getPropInt(prop.str(), defVal);
  7611. }
  7612. __int64 CLocalWorkUnit::getDebugValueInt64(const char *propname, __int64 defVal) const
  7613. {
  7614. StringBuffer lower;
  7615. lower.append(propname).toLowerCase();
  7616. CriticalBlock block(crit);
  7617. StringBuffer prop("Debug/");
  7618. prop.append(lower);
  7619. return p->getPropInt64(prop.str(), defVal);
  7620. }
  7621. bool CLocalWorkUnit::getDebugValueBool(const char * propname, bool defVal) const
  7622. {
  7623. StringBuffer lower;
  7624. lower.append(propname).toLowerCase();
  7625. CriticalBlock block(crit);
  7626. StringBuffer prop("Debug/");
  7627. prop.append(lower);
  7628. return p->getPropBool(prop.str(), defVal);
  7629. }
  7630. IStringIterator *CLocalWorkUnit::getLogs(const char *type, const char *instance) const
  7631. {
  7632. VStringBuffer xpath("Process/%s/", type);
  7633. if (instance)
  7634. xpath.append(instance);
  7635. else
  7636. xpath.append("*");
  7637. CriticalBlock block(crit);
  7638. if (p->getPropInt("@wuidVersion") < 1) // legacy wuid
  7639. {
  7640. // NB: instance unused
  7641. if (streq("EclAgent", type))
  7642. return new CStringPTreeIterator(p->getElements("Debug/eclagentlog"));
  7643. else if (streq("Thor", type))
  7644. return new CStringPTreeIterator(p->getElements("Debug/thorlog*"));
  7645. VStringBuffer xpath("Debug/%s", type);
  7646. return new CStringPTreeIterator(p->getElements(xpath.str()));
  7647. }
  7648. else
  7649. return new CStringPTreeAttrIterator(p->getElements(xpath.str()), "@log");
  7650. }
  7651. IPropertyTreeIterator* CLocalWorkUnit::getProcesses(const char *type, const char *instance) const
  7652. {
  7653. VStringBuffer xpath("Process/%s/", type);
  7654. if (instance)
  7655. xpath.append(instance);
  7656. else
  7657. xpath.append("*");
  7658. CriticalBlock block(crit);
  7659. return p->getElements(xpath.str());
  7660. }
  7661. IStringIterator *CLocalWorkUnit::getProcesses(const char *type) const
  7662. {
  7663. VStringBuffer xpath("Process/%s/*", type);
  7664. CriticalBlock block(crit);
  7665. return new CStringPTreeTagIterator(p->getElements(xpath.str()));
  7666. }
  7667. void CLocalWorkUnit::addProcess(const char *type, const char *instance, unsigned pid, const char *log)
  7668. {
  7669. VStringBuffer processType("Process/%s", type);
  7670. VStringBuffer xpath("%s/%s", processType.str(), instance);
  7671. if (log)
  7672. xpath.appendf("[@log=\"%s\"]", log);
  7673. CriticalBlock block(crit);
  7674. if (!p->hasProp(xpath))
  7675. {
  7676. IPropertyTree *node = ensurePTree(p, processType.str());
  7677. node = node->addPropTree(instance);
  7678. node->setProp("@log", log);
  7679. node->setPropInt("@pid", pid);
  7680. }
  7681. }
  7682. void CLocalWorkUnit::setDebugValue(const char *propname, const char *value, bool overwrite)
  7683. {
  7684. StringBuffer lower;
  7685. lower.append(propname).toLowerCase();
  7686. CriticalBlock block(crit);
  7687. StringBuffer prop("Debug/");
  7688. prop.append(lower);
  7689. if (overwrite || !p->hasProp(prop.str()))
  7690. {
  7691. // MORE - not sure this line should be needed....
  7692. p->setProp("Debug", "");
  7693. p->setProp(prop.str(), value);
  7694. }
  7695. }
  7696. void CLocalWorkUnit::setDebugValueInt(const char *propname, int value, bool overwrite)
  7697. {
  7698. StringBuffer lower;
  7699. lower.append(propname).toLowerCase();
  7700. CriticalBlock block(crit);
  7701. StringBuffer prop("Debug/");
  7702. prop.append(lower);
  7703. if (overwrite || !p->hasProp(prop.str()))
  7704. {
  7705. // MORE - not sure this line should be needed....
  7706. p->setProp("Debug", "");
  7707. p->setPropInt(prop.str(), value);
  7708. }
  7709. }
  7710. void CLocalWorkUnit::setTracingValue(const char *propname, const char *value)
  7711. {
  7712. CriticalBlock block(crit);
  7713. // MORE - not sure this line should be needed....
  7714. p->setProp("Tracing", "");
  7715. StringBuffer prop("Tracing/");
  7716. p->setProp(prop.append(propname).str(), value);
  7717. }
  7718. void CLocalWorkUnit::setTracingValueInt(const char *propname, int value)
  7719. {
  7720. CriticalBlock block(crit);
  7721. StringBuffer prop("Tracing/");
  7722. p->setPropInt(prop.append(propname).str(), value);
  7723. }
  7724. void CLocalWorkUnit::setTracingValueInt64(const char *propname, __int64 value)
  7725. {
  7726. CriticalBlock block(crit);
  7727. VStringBuffer prop("Tracing/%s", propname);
  7728. p->setPropInt64(prop.str(), value);
  7729. }
  7730. IConstWUQuery* CLocalWorkUnit::getQuery() const
  7731. {
  7732. // For this to be legally called, we must have the read-able interface. So we are already locked for (at least) read.
  7733. CriticalBlock block(crit);
  7734. if (!query)
  7735. {
  7736. IPropertyTree *s = p->getPropTree("Query");
  7737. if (s)
  7738. query.setown(new CLocalWUQuery(s)); // NB takes ownership of 's'
  7739. }
  7740. return query.getLink();
  7741. }
  7742. IWUQuery* CLocalWorkUnit::updateQuery()
  7743. {
  7744. // For this to be legally called, we must have the write-able interface. So we are already locked for write.
  7745. CriticalBlock block(crit);
  7746. if (!query)
  7747. {
  7748. IPropertyTree *s = p->queryPropTree("Query");
  7749. if (!s)
  7750. s = p->addPropTree("Query", createPTreeFromXMLString("<Query fetchEntire='1'/>")); // Is this really desirable (the fetchEntire) ?
  7751. s->Link();
  7752. query.setown(new CLocalWUQuery(s));
  7753. }
  7754. return query.getLink();
  7755. }
  7756. void CLocalWorkUnit::loadPlugins() const
  7757. {
  7758. CriticalBlock block(crit);
  7759. if (!pluginsCached)
  7760. {
  7761. assertex(plugins.length() == 0);
  7762. Owned<IPropertyTreeIterator> r = p->getElements("Plugins/Plugin");
  7763. for (r->first(); r->isValid(); r->next())
  7764. {
  7765. IPropertyTree *rp = &r->query();
  7766. rp->Link();
  7767. plugins.append(*new CLocalWUPlugin(rp));
  7768. }
  7769. pluginsCached = true;
  7770. }
  7771. }
  7772. IConstWUPluginIterator& CLocalWorkUnit::getPlugins() const
  7773. {
  7774. CriticalBlock block(crit);
  7775. loadPlugins();
  7776. return *new CArrayIteratorOf<IConstWUPlugin,IConstWUPluginIterator> (plugins, 0, (IConstWorkUnit *) this);
  7777. }
  7778. void CLocalWorkUnit::loadLibraries() const
  7779. {
  7780. CriticalBlock block(crit);
  7781. if (!librariesCached)
  7782. {
  7783. assertex(libraries.length() == 0);
  7784. Owned<IPropertyTreeIterator> r = p->getElements("Libraries/Library");
  7785. ForEach(*r)
  7786. {
  7787. IPropertyTree *rp = &r->query();
  7788. rp->Link();
  7789. libraries.append(*new CLocalWULibrary(rp));
  7790. }
  7791. librariesCached = true;
  7792. }
  7793. }
  7794. IConstWULibraryIterator& CLocalWorkUnit::getLibraries() const
  7795. {
  7796. CriticalBlock block(crit);
  7797. loadLibraries();
  7798. return *new CArrayIteratorOf<IConstWULibrary,IConstWULibraryIterator> (libraries, 0, (IConstWorkUnit *) this);
  7799. }
  7800. IConstWULibrary * CLocalWorkUnit::getLibraryByName(const char * search) const
  7801. {
  7802. CriticalBlock block(crit);
  7803. loadLibraries();
  7804. ForEachItemIn(idx, libraries)
  7805. {
  7806. SCMStringBuffer name;
  7807. IConstWULibrary &cur = libraries.item(idx);
  7808. cur.getName(name);
  7809. if (stricmp(name.str(), search)==0)
  7810. return &OLINK(cur);
  7811. }
  7812. return NULL;
  7813. }
  7814. StringBuffer &formatGraphTimerLabel(StringBuffer &str, const char *graphName, unsigned subGraphNum, unsigned __int64 subId)
  7815. {
  7816. str.append("Graph ").append(graphName);
  7817. if (subGraphNum) str.append(" - ").append(subGraphNum).append(" (").append(subId).append(")");
  7818. else if (subId) str.append(" - id(").append(subId).append(")");
  7819. return str;
  7820. }
  7821. StringBuffer &formatGraphTimerScope(StringBuffer &str, unsigned wfid, const char *graphName, unsigned subGraphNum, unsigned __int64 subId)
  7822. {
  7823. if (wfid)
  7824. str.append(WorkflowScopePrefix).append(wfid).append(":");
  7825. str.append(graphName);
  7826. if (subId) str.append(":sg").append(subId);
  7827. return str;
  7828. }
  7829. bool parseGraphTimerLabel(const char *label, StringAttr &graphName, unsigned & graphNum, unsigned &subGraphNum, unsigned &subId)
  7830. {
  7831. // expects format: "Graph <graphname>[ - <subgraphnum> (<subgraphid>)]"
  7832. unsigned len = (size32_t)strlen(label);
  7833. if (len < 6 || (0 != memcmp(label, "Graph ", 6)))
  7834. return false;
  7835. graphNum = 0;
  7836. subGraphNum = 0;
  7837. subId = 0;
  7838. const char *finger = label+6;
  7839. const char *finger2 = strchr(finger, '-');
  7840. if (NULL == finger2) // just graphName
  7841. graphName.set(finger);
  7842. else
  7843. {
  7844. graphName.set(finger, (size32_t)((finger2-1)-finger));
  7845. finger = finger2+2; // skip '-' and space
  7846. finger2 = strchr(finger, ' ');
  7847. if (finger2)
  7848. {
  7849. subGraphNum = atoi_l(finger, (size32_t)(finger2-finger));
  7850. finger = finger2+2; // skip space and '('
  7851. finger2 = strchr(finger, ')');
  7852. if (finger2)
  7853. subId = atoi_l(finger, (size32_t)(finger2-finger));
  7854. }
  7855. else if (((len-(finger-label))>3) && 0 == memcmp(finger, "id(", 3)) // subgraph id only, new format.
  7856. {
  7857. finger += 3;
  7858. finger2 = strchr(finger, ')');
  7859. if (finger2)
  7860. subId = atoi_l(finger, (size32_t)(finger2-finger));
  7861. }
  7862. }
  7863. if (graphName && !memicmp(graphName, "graph", 5))
  7864. graphNum = atoi(graphName + 5);
  7865. return true;
  7866. }
  7867. bool parseGraphScope(const char *scope, StringAttr &graphName, unsigned & graphNum, unsigned &subGraphId)
  7868. {
  7869. if (!MATCHES_CONST_PREFIX(scope, GraphScopePrefix))
  7870. return false;
  7871. graphNum = atoi(scope + strlen(GraphScopePrefix));
  7872. subGraphId = 0;
  7873. const char * colon = strchr(scope, ':');
  7874. if (!colon)
  7875. {
  7876. graphName.set(scope);
  7877. return true;
  7878. }
  7879. const char * subgraph = colon+1;
  7880. graphName.set(scope, (size32_t)(colon - scope));
  7881. if (MATCHES_CONST_PREFIX(subgraph, SubGraphScopePrefix))
  7882. subGraphId = atoi(subgraph+strlen(SubGraphScopePrefix));
  7883. return true;
  7884. }
  7885. void CLocalWorkUnit::setStatistic(StatisticCreatorType creatorType, const char * creator, StatisticScopeType scopeType, const char * scope, StatisticKind kind, const char * optDescription, unsigned __int64 value, unsigned __int64 count, unsigned __int64 maxValue, StatsMergeAction mergeAction)
  7886. {
  7887. if (!scope) scope = GLOBAL_SCOPE;
  7888. const char * kindName = queryStatisticName(kind);
  7889. StatisticMeasure measure = queryMeasure(kind);
  7890. //creator. scope and name must all be present, and must not contain semi colons.
  7891. assertex(creator && scope);
  7892. CriticalBlock block(crit);
  7893. IPropertyTree * stats = p->queryPropTree("Statistics");
  7894. if (!stats)
  7895. stats = p->addPropTree("Statistics");
  7896. IPropertyTree * statTree = NULL;
  7897. if (mergeAction != StatsMergeAppend)
  7898. {
  7899. StringBuffer xpath;
  7900. xpath.append("Statistic[@creator='").append(creator).append("'][@scope='").append(scope).append("'][@kind='").append(kindName).append("']");
  7901. statTree = stats->queryPropTree(xpath.str());
  7902. }
  7903. if (!statTree)
  7904. {
  7905. statTree = stats->addPropTree("Statistic");
  7906. statTree->setProp("@creator", creator);
  7907. statTree->setProp("@scope", scope);
  7908. statTree->setProp("@kind", kindName);
  7909. //These items are primarily here to facilitate filtering.
  7910. statTree->setProp("@unit", queryMeasureName(measure));
  7911. statTree->setProp("@c", queryCreatorTypeName(creatorType));
  7912. statTree->setProp("@s", queryScopeTypeName(scopeType));
  7913. statTree->setPropInt64("@ts", getTimeStampNowValue());
  7914. if (optDescription)
  7915. statTree->setProp("@desc", optDescription);
  7916. if (statistics.cached)
  7917. statistics.append(LINK(statTree));
  7918. mergeAction = StatsMergeAppend;
  7919. }
  7920. if (mergeAction != StatsMergeAppend) // RKC->GH Is this right??
  7921. {
  7922. unsigned __int64 oldValue = statTree->getPropInt64("@value", 0);
  7923. unsigned __int64 oldCount = statTree->getPropInt64("@count", 0);
  7924. unsigned __int64 oldMax = statTree->getPropInt64("@max", 0);
  7925. if (oldMax < oldValue)
  7926. oldMax = oldValue;
  7927. statTree->setPropInt64("@value", mergeStatisticValue(oldValue, value, mergeAction));
  7928. statTree->setPropInt64("@count", count + oldCount);
  7929. if (maxValue > oldMax)
  7930. statTree->setPropInt64("@max", maxValue);
  7931. }
  7932. else
  7933. {
  7934. statTree->setPropInt64("@value", value);
  7935. statTree->setPropInt64("@count", count);
  7936. if (maxValue)
  7937. statTree->setPropInt64("@max", maxValue);
  7938. else
  7939. statTree->removeProp("@max");
  7940. }
  7941. //Whenever a graph time is updated recalculate the total time spent in thor, and save it
  7942. if ((scopeType == SSTgraph) && (kind == StTimeElapsed))
  7943. {
  7944. _loadStatistics();
  7945. stat_type totalTime = 0;
  7946. ForEachItemIn(i, statistics)
  7947. {
  7948. IConstWUStatistic & cur = statistics.item(i);
  7949. if ((cur.getScopeType() == SSTgraph) && (cur.getKind() == StTimeElapsed))
  7950. totalTime += cur.getValue();
  7951. }
  7952. StringBuffer t;
  7953. formatTimeCollatable(t, totalTime, false);
  7954. p->setProp("@totalThorTime", t);
  7955. }
  7956. }
  7957. void CLocalWorkUnit::_loadStatistics() const
  7958. {
  7959. statistics.load(p,"Statistics/*");
  7960. }
  7961. bool CLocalWorkUnit::getStatistic(stat_type & value, const char * scope, StatisticKind kind) const
  7962. {
  7963. //MORE: Optimize this....
  7964. WuScopeFilter filter;
  7965. filter.addScope(scope).setIncludeNesting(0).addRequiredStat(kind).addOutputStatistic(kind).finishedFilter();
  7966. Owned<IConstWUScopeIterator> stats = &getScopeIterator(filter);
  7967. if (stats->first())
  7968. return stats->getStat(kind, value);
  7969. return false;
  7970. }
  7971. IConstWUScopeIterator & CLocalWorkUnit::getScopeIterator(const WuScopeFilter & filter) const
  7972. {
  7973. assertex(filter.isOptimized());
  7974. WuScopeSourceFlags sources = filter.sourceFlags;
  7975. Owned<CompoundStatisticsScopeIterator> compoundIter = new CompoundStatisticsScopeIterator(filter);
  7976. if (sources & SSFsearchGlobalStats)
  7977. {
  7978. {
  7979. CriticalBlock block(crit);
  7980. statistics.loadBranch(p,"Statistics");
  7981. }
  7982. Owned<IConstWUScopeIterator> localStats(new WorkUnitStatisticsScopeIterator(statistics, filter.queryIterFilter()));
  7983. compoundIter->addIter(localStats);
  7984. }
  7985. if (sources & SSFsearchGraphStats)
  7986. {
  7987. const char * wuid = p->queryName();
  7988. Owned<IConstWUScopeIterator> scopeIter(new CConstGraphProgressScopeIterator(wuid, filter.queryIterFilter(), filter.minVersion));
  7989. compoundIter->addIter(scopeIter);
  7990. }
  7991. if (sources & SSFsearchGraph)
  7992. {
  7993. Owned<IConstWUScopeIterator> graphIter(new GraphScopeIterator(this, filter.queryIterFilter()));
  7994. compoundIter->addIter(graphIter);
  7995. }
  7996. if (sources & SSFsearchWorkflow)
  7997. {
  7998. Owned<IConstWorkflowItemIterator> iter = getWorkflowItems();
  7999. if (iter)
  8000. {
  8001. Owned<IConstWUScopeIterator> workflowIter(new WorkflowStatisticsScopeIterator(iter));
  8002. compoundIter->addIter(workflowIter);
  8003. }
  8004. }
  8005. if (sources & SSFsearchExceptions)
  8006. {
  8007. Owned<IConstWUScopeIterator> notesIter(new NotesIterator(this,filter.queryIterFilter()));
  8008. compoundIter->addIter(notesIter);
  8009. }
  8010. return *compoundIter.getClear();
  8011. }
  8012. IWUPlugin* CLocalWorkUnit::updatePluginByName(const char *qname)
  8013. {
  8014. CriticalBlock block(crit);
  8015. IConstWUPlugin *existing = getPluginByName(qname);
  8016. if (existing)
  8017. return (IWUPlugin *) existing;
  8018. if (!plugins.length())
  8019. p->addPropTree("Plugins");
  8020. IPropertyTree *pl = p->queryPropTree("Plugins");
  8021. IPropertyTree *s = pl->addPropTree("Plugin");
  8022. s->Link();
  8023. IWUPlugin* q = new CLocalWUPlugin(s);
  8024. q->Link();
  8025. plugins.append(*q);
  8026. q->setPluginName(qname);
  8027. return q;
  8028. }
  8029. IConstWUPlugin* CLocalWorkUnit::getPluginByName(const char *qname) const
  8030. {
  8031. CriticalBlock block(crit);
  8032. loadPlugins();
  8033. ForEachItemIn(idx, plugins)
  8034. {
  8035. SCMStringBuffer name;
  8036. IConstWUPlugin &cur = plugins.item(idx);
  8037. cur.getPluginName(name);
  8038. if (stricmp(name.str(), qname)==0)
  8039. {
  8040. cur.Link();
  8041. return &cur;
  8042. }
  8043. }
  8044. return NULL;
  8045. }
  8046. IWULibrary* CLocalWorkUnit::updateLibraryByName(const char *qname)
  8047. {
  8048. CriticalBlock block(crit);
  8049. IConstWULibrary *existing = getLibraryByName(qname);
  8050. if (existing)
  8051. return (IWULibrary *) existing;
  8052. if (!libraries.length())
  8053. p->addPropTree("Libraries");
  8054. IPropertyTree *pl = p->queryPropTree("Libraries");
  8055. IPropertyTree *s = pl->addPropTree("Library");
  8056. s->Link();
  8057. IWULibrary* q = new CLocalWULibrary(s);
  8058. q->Link();
  8059. libraries.append(*q);
  8060. q->setName(qname);
  8061. return q;
  8062. }
  8063. void CLocalWorkUnit::_loadExceptions() const
  8064. {
  8065. assertex(exceptions.length() == 0);
  8066. Owned<IPropertyTreeIterator> r = p->getElements("Exceptions/Exception");
  8067. for (r->first(); r->isValid(); r->next())
  8068. {
  8069. IPropertyTree *rp = &r->query();
  8070. rp->Link();
  8071. exceptions.append(*new CLocalWUException(rp));
  8072. }
  8073. }
  8074. void CLocalWorkUnit::loadExceptions() const
  8075. {
  8076. CriticalBlock block(crit);
  8077. if (!exceptionsCached)
  8078. {
  8079. _loadExceptions();
  8080. exceptionsCached = true;
  8081. }
  8082. }
  8083. IConstWUExceptionIterator& CLocalWorkUnit::getExceptions() const
  8084. {
  8085. CriticalBlock block(crit);
  8086. loadExceptions();
  8087. return *new CArrayIteratorOf<IConstWUException,IConstWUExceptionIterator> (exceptions, 0, (IConstWorkUnit *) this);
  8088. }
  8089. unsigned CLocalWorkUnit::getExceptionCount() const
  8090. {
  8091. CriticalBlock block(crit);
  8092. loadExceptions();
  8093. return exceptions.length();
  8094. }
  8095. void CLocalWorkUnit::clearExceptions(const char *source)
  8096. {
  8097. CriticalBlock block(crit);
  8098. // For this to be legally called, we must have the write-able interface. So we are already locked for write.
  8099. loadExceptions();
  8100. ForEachItemInRev(idx, exceptions)
  8101. {
  8102. IWUException &e = exceptions.item(idx);
  8103. SCMStringBuffer s;
  8104. e.getExceptionSource(s);
  8105. if (source)
  8106. {
  8107. if (!strieq(s.s, source))
  8108. continue;
  8109. }
  8110. else
  8111. {
  8112. if (strieq(s.s, "eclcc") || strieq(s.s, "eclccserver") || strieq(s.s, "eclserver") )
  8113. break;
  8114. }
  8115. VStringBuffer xpath("Exceptions/Exception[@sequence='%d']", e.getSequence());
  8116. p->removeProp(xpath);
  8117. exceptions.remove(idx);
  8118. }
  8119. if (exceptions.length() == 0)
  8120. p->removeProp("Exceptions");
  8121. }
  8122. IWUException* CLocalWorkUnit::createException()
  8123. {
  8124. CriticalBlock block(crit);
  8125. // For this to be legally called, we must have the write-able interface. So we are already locked for write.
  8126. loadExceptions();
  8127. if (!exceptions.length())
  8128. p->addPropTree("Exceptions");
  8129. IPropertyTree *r = p->queryPropTree("Exceptions");
  8130. IPropertyTree *s = r->addPropTree("Exception");
  8131. s->setPropInt("@sequence", exceptions.ordinality());
  8132. IWUException* q = new CLocalWUException(LINK(s));
  8133. exceptions.append(*LINK(q));
  8134. Owned<IJlibDateTime> now = createDateTimeNow();
  8135. SCMStringBuffer temp;
  8136. now->getString(temp);
  8137. q->setTimeStamp(temp.str());
  8138. return q;
  8139. }
  8140. IConstWUWebServicesInfo* CLocalWorkUnit::getWebServicesInfo() const
  8141. {
  8142. // For this to be legally called, we must have the read-able interface. So we are already locked for (at least) read.
  8143. CriticalBlock block(crit);
  8144. if (!webServicesInfoCached)
  8145. {
  8146. assertex(!webServicesInfo);
  8147. IPropertyTree *s = p->getPropTree("WebServicesInfo");
  8148. if (s)
  8149. webServicesInfo.setown(new CLocalWUWebServicesInfo(s)); // NB takes ownership of 's'
  8150. webServicesInfoCached = true;
  8151. }
  8152. return webServicesInfo.getLink();
  8153. }
  8154. IWUWebServicesInfo* CLocalWorkUnit::updateWebServicesInfo(bool create)
  8155. {
  8156. // For this to be legally called, we must have the write-able interface. So we are already locked for write.
  8157. CriticalBlock block(crit);
  8158. if (!webServicesInfoCached)
  8159. {
  8160. IPropertyTree *s = p->queryPropTree("WebServicesInfo");
  8161. if (!s)
  8162. {
  8163. if (create)
  8164. s = p->addPropTree("WebServicesInfo");
  8165. else
  8166. return NULL;
  8167. }
  8168. s->Link();
  8169. webServicesInfo.setown(new CLocalWUWebServicesInfo(s));
  8170. webServicesInfoCached = true;
  8171. }
  8172. return webServicesInfo.getLink();
  8173. }
  8174. static int compareResults(IInterface * const *ll, IInterface * const *rr)
  8175. {
  8176. CLocalWUResult *l = (CLocalWUResult *) *ll;
  8177. CLocalWUResult *r = (CLocalWUResult *) *rr;
  8178. return l->getResultSequence() - r->getResultSequence();
  8179. }
  8180. void CLocalWorkUnit::_loadResults() const
  8181. {
  8182. Owned<IPropertyTreeIterator> r = p->getElements("Results/Result");
  8183. for (r->first(); r->isValid(); r->next())
  8184. {
  8185. IPropertyTree *rp = &r->query();
  8186. rp->Link();
  8187. results.append(*new CLocalWUResult(rp));
  8188. }
  8189. }
  8190. void CLocalWorkUnit::loadResults() const
  8191. {
  8192. if (!resultsCached)
  8193. {
  8194. assertex(results.length() == 0);
  8195. _loadResults();
  8196. results.sort(compareResults);
  8197. resultsCached = true;
  8198. }
  8199. }
  8200. void CLocalWorkUnit::_loadVariables() const
  8201. {
  8202. Owned<IPropertyTreeIterator> r = p->getElements("Variables/Variable");
  8203. for (r->first(); r->isValid(); r->next())
  8204. {
  8205. IPropertyTree *rp = &r->query();
  8206. rp->Link();
  8207. variables.append(*new CLocalWUResult(rp));
  8208. }
  8209. }
  8210. void CLocalWorkUnit::loadVariables() const
  8211. {
  8212. if (!variablesCached)
  8213. {
  8214. assertex(variables.length() == 0);
  8215. _loadVariables();
  8216. variablesCached = true;
  8217. }
  8218. }
  8219. void CLocalWorkUnit::_loadTemporaries() const
  8220. {
  8221. Owned<IPropertyTreeIterator> r = p->getElements("Temporaries/Variable");
  8222. for (r->first(); r->isValid(); r->next())
  8223. {
  8224. IPropertyTree *rp = &r->query();
  8225. rp->Link();
  8226. temporaries.append(*new CLocalWUResult(rp));
  8227. }
  8228. }
  8229. void CLocalWorkUnit::loadTemporaries() const
  8230. {
  8231. if (!temporariesCached)
  8232. {
  8233. assertex(temporaries.length() == 0);
  8234. _loadTemporaries();
  8235. temporariesCached = true;
  8236. }
  8237. }
  8238. void CLocalWorkUnit::deleteTemporaries()
  8239. {
  8240. CriticalBlock block(crit);
  8241. if (temporariesCached)
  8242. {
  8243. temporaries.kill();
  8244. temporariesCached = false;
  8245. }
  8246. p->removeProp("Temporaries");
  8247. }
  8248. IWUResult* CLocalWorkUnit::createResult()
  8249. {
  8250. CriticalBlock block(crit);
  8251. // For this to be legally called, we must have the write-able interface. So we are already locked for write.
  8252. loadResults();
  8253. if (!results.length())
  8254. p->addPropTree("Results");
  8255. IPropertyTree *r = p->queryPropTree("Results");
  8256. IPropertyTree *s = r->addPropTree("Result");
  8257. s->Link();
  8258. IWUResult* q = new CLocalWUResult(s);
  8259. q->Link();
  8260. results.append(*q);
  8261. return q;
  8262. }
  8263. IWUResult* CLocalWorkUnit::updateResultByName(const char *qname)
  8264. {
  8265. CriticalBlock block(crit);
  8266. IConstWUResult *existing = getResultByName(qname);
  8267. if (existing)
  8268. return (IWUResult *) existing;
  8269. IWUResult* q = createResult();
  8270. q->setResultName(qname);
  8271. return q;
  8272. }
  8273. IWUResult* CLocalWorkUnit::updateResultBySequence(unsigned seq)
  8274. {
  8275. CriticalBlock block(crit);
  8276. IConstWUResult *existing = getResultBySequence(seq);
  8277. if (existing)
  8278. return (IWUResult *) existing;
  8279. IWUResult* q = createResult();
  8280. q->setResultSequence(seq);
  8281. return q;
  8282. }
  8283. IConstWUResultIterator& CLocalWorkUnit::getResults() const
  8284. {
  8285. CriticalBlock block(crit);
  8286. loadResults();
  8287. return *new CArrayIteratorOf<IConstWUResult,IConstWUResultIterator> (results, 0, (IConstWorkUnit *) this);
  8288. }
  8289. IConstWUResult* CLocalWorkUnit::getResultByName(const char *qname) const
  8290. {
  8291. CriticalBlock block(crit);
  8292. loadResults();
  8293. ForEachItemIn(idx, results)
  8294. {
  8295. SCMStringBuffer name;
  8296. IConstWUResult &cur = results.item(idx);
  8297. cur.getResultName(name);
  8298. if (stricmp(name.str(), qname)==0)
  8299. {
  8300. cur.Link();
  8301. return &cur;
  8302. }
  8303. }
  8304. return NULL;
  8305. }
  8306. IConstWUResult* CLocalWorkUnit::getQueryResultByName(const char *qname) const
  8307. {
  8308. CriticalBlock block(crit);
  8309. loadResults();
  8310. ForEachItemIn(idx, results)
  8311. {
  8312. IConstWUResult &cur = results.item(idx);
  8313. if (!isSpecialResultSequence(cur.getResultSequence()))
  8314. {
  8315. SCMStringBuffer name;
  8316. cur.getResultName(name);
  8317. if (stricmp(name.str(), qname)==0)
  8318. {
  8319. cur.Link();
  8320. return &cur;
  8321. }
  8322. }
  8323. }
  8324. return NULL;
  8325. }
  8326. IConstWUResult* CLocalWorkUnit::getResultBySequence(unsigned seq) const
  8327. {
  8328. CriticalBlock block(crit);
  8329. loadResults();
  8330. ForEachItemIn(idx, results)
  8331. {
  8332. IConstWUResult &cur = results.item(idx);
  8333. if (cur.getResultSequence() == seq)
  8334. {
  8335. cur.Link();
  8336. return &cur;
  8337. }
  8338. }
  8339. return NULL;
  8340. }
  8341. IConstWUResultIterator& CLocalWorkUnit::getVariables() const
  8342. {
  8343. CriticalBlock block(crit);
  8344. loadVariables();
  8345. return *new CArrayIteratorOf<IConstWUResult,IConstWUResultIterator> (variables, 0, (IConstWorkUnit *) this);
  8346. }
  8347. IConstWUResult* CLocalWorkUnit::getGlobalByName(const char *qname) const
  8348. {
  8349. CriticalBlock block(crit);
  8350. if (strcmp(p->queryName(), GLOBAL_WORKUNIT)==0)
  8351. return getVariableByName(qname);
  8352. Owned <IWorkUnit> global = factory->getGlobalWorkUnit(secMgr, secUser);
  8353. return global->getVariableByName(qname);
  8354. }
  8355. IWUResult* CLocalWorkUnit::updateGlobalByName(const char *qname)
  8356. {
  8357. CriticalBlock block(crit);
  8358. if (strcmp(p->queryName(), GLOBAL_WORKUNIT)==0)
  8359. return updateVariableByName(qname);
  8360. Owned <IWorkUnit> global = factory->getGlobalWorkUnit(secMgr, secUser);
  8361. return global->updateVariableByName(qname);
  8362. }
  8363. IConstWUResult* CLocalWorkUnit::getVariableByName(const char *qname) const
  8364. {
  8365. CriticalBlock block(crit);
  8366. loadVariables();
  8367. ForEachItemIn(idx, variables)
  8368. {
  8369. SCMStringBuffer name;
  8370. IConstWUResult &cur = variables.item(idx);
  8371. cur.getResultName(name);
  8372. if (stricmp(name.str(), qname)==0)
  8373. {
  8374. cur.Link();
  8375. return &cur;
  8376. }
  8377. }
  8378. return NULL;
  8379. }
  8380. IConstWUResult* CLocalWorkUnit::getTemporaryByName(const char *qname) const
  8381. {
  8382. CriticalBlock block(crit);
  8383. loadTemporaries();
  8384. ForEachItemIn(idx, temporaries)
  8385. {
  8386. SCMStringBuffer name;
  8387. IConstWUResult &cur = temporaries.item(idx);
  8388. cur.getResultName(name);
  8389. if (stricmp(name.str(), qname)==0)
  8390. {
  8391. cur.Link();
  8392. return &cur;
  8393. }
  8394. }
  8395. return NULL;
  8396. }
  8397. IConstWUResultIterator& CLocalWorkUnit::getTemporaries() const
  8398. {
  8399. CriticalBlock block(crit);
  8400. loadTemporaries();
  8401. return *new CArrayIteratorOf<IConstWUResult,IConstWUResultIterator> (temporaries, 0, (IConstWorkUnit *) this);
  8402. }
  8403. IWUResult* CLocalWorkUnit::updateTemporaryByName(const char *qname)
  8404. {
  8405. CriticalBlock block(crit);
  8406. IConstWUResult *existing = getTemporaryByName(qname);
  8407. if (existing)
  8408. return (IWUResult *) existing;
  8409. IPropertyTree *vars = (temporaries.length()) ? p->queryPropTree("Temporaries") : p->addPropTree("Temporaries");
  8410. IPropertyTree *s = vars->addPropTree("Variable");
  8411. s->Link();
  8412. IWUResult* q = new CLocalWUResult(s);
  8413. q->Link();
  8414. temporaries.append(*q);
  8415. q->setResultName(qname);
  8416. q->setResultSequence(ResultSequenceInternal);
  8417. return q;
  8418. }
  8419. IWUResult* CLocalWorkUnit::updateVariableByName(const char *qname)
  8420. {
  8421. CriticalBlock block(crit);
  8422. IConstWUResult *existing = getVariableByName(qname);
  8423. if (existing)
  8424. return (IWUResult *) existing;
  8425. if (!variables.length())
  8426. p->addPropTree("Variables");
  8427. IPropertyTree *vars = p->queryPropTree("Variables");
  8428. IPropertyTree *s = vars->addPropTree("Variable");
  8429. s->Link();
  8430. IWUResult* q = new CLocalWUResult(s);
  8431. q->Link();
  8432. variables.append(*q);
  8433. q->setResultName(qname);
  8434. q->setResultSequence(ResultSequenceStored);
  8435. return q;
  8436. }
  8437. void CLocalWorkUnit::deleteTempFiles(const char *graph, bool deleteOwned, bool deleteJobOwned)
  8438. {
  8439. CriticalBlock block(crit);
  8440. IPropertyTree *files = p->queryPropTree("Files");
  8441. if (!files) return;
  8442. Owned<IPropertyTreeIterator> iter = files->getElements("File");
  8443. ICopyArrayOf<IPropertyTree> toRemove;
  8444. ForEach (*iter)
  8445. {
  8446. IPropertyTree &file = iter->query();
  8447. WUFileKind fileKind = (WUFileKind) file.getPropInt("@kind", WUFileStandard);
  8448. if(file.getPropBool("@temporary")) fileKind = WUFileTemporary; // @temporary, legacy check
  8449. bool needDelete;
  8450. switch(fileKind)
  8451. {
  8452. case WUFileTemporary:
  8453. if(graph==NULL)
  8454. needDelete = true;
  8455. else
  8456. {
  8457. const char *graphOwner = file.queryProp("@graph");
  8458. needDelete = ((graphOwner==NULL) || (strcmp(graph, graphOwner)==0));
  8459. }
  8460. break;
  8461. case WUFileJobOwned:
  8462. needDelete = ((graph==NULL) && deleteJobOwned);
  8463. break;
  8464. case WUFileOwned:
  8465. needDelete = ((graph==NULL) && deleteOwned);
  8466. break;
  8467. default:
  8468. needDelete = false;
  8469. }
  8470. if(needDelete)
  8471. {
  8472. const char *name = file.queryProp("@name");
  8473. LOG(MCdebugProgress, unknownJob, "Removing workunit file %s from DFS", name);
  8474. queryDistributedFileDirectory().removeEntry(name, queryUserDescriptor());
  8475. toRemove.append(file);
  8476. }
  8477. }
  8478. ForEachItemIn(r, toRemove) files->removeTree(&toRemove.item(r));
  8479. }
  8480. static void _noteFileRead(IDistributedFile *file, IPropertyTree *filesRead)
  8481. {
  8482. IDistributedSuperFile *super = file->querySuperFile();
  8483. StringBuffer fname;
  8484. file->getLogicalName(fname);
  8485. if (fname.length())
  8486. {
  8487. StringBuffer path("File[@name=\"");
  8488. path.append(fname).append("\"]");
  8489. IPropertyTree *fileTree = filesRead->queryPropTree(path.str());
  8490. if (fileTree)
  8491. fileTree->setPropInt("@useCount", fileTree->getPropInt("@useCount")+1);
  8492. else
  8493. {
  8494. StringBuffer cluster;
  8495. file->getClusterName(0,cluster);
  8496. fileTree = createPTree();
  8497. fileTree->setProp("@name", fname.str());
  8498. fileTree->setProp("@cluster", cluster.str());
  8499. fileTree->setPropInt("@useCount", 1);
  8500. fileTree = filesRead->addPropTree("File", fileTree);
  8501. }
  8502. if (super)
  8503. {
  8504. fileTree->setPropBool("@super", true);
  8505. Owned<IDistributedFileIterator> iter = super->getSubFileIterator(false);
  8506. ForEach (*iter)
  8507. {
  8508. IDistributedFile &file = iter->query();
  8509. StringBuffer fname;
  8510. file.getLogicalName(fname);
  8511. fileTree->addPropTree("Subfile")->setProp("@name", fname.str());
  8512. _noteFileRead(&file, filesRead);
  8513. }
  8514. }
  8515. }
  8516. }
  8517. void CLocalWorkUnit::_loadFilesRead() const
  8518. {
  8519. // Nothing to do
  8520. }
  8521. void CLocalWorkUnit::noteFileRead(IDistributedFile *file)
  8522. {
  8523. if (file)
  8524. {
  8525. CriticalBlock block(crit);
  8526. IPropertyTree *files = p->queryPropTree("FilesRead");
  8527. if (!files)
  8528. files = p->addPropTree("FilesRead");
  8529. _noteFileRead(file, files);
  8530. }
  8531. }
  8532. void CLocalWorkUnit::noteFieldUsage(IPropertyTree * fieldUsage)
  8533. {
  8534. if (fieldUsage)
  8535. {
  8536. CriticalBlock block(crit);
  8537. p->addPropTree("usedsources", fieldUsage);
  8538. }
  8539. }
  8540. void CLocalWorkUnit::_loadFilesWritten() const
  8541. {
  8542. // Nothing to do
  8543. }
  8544. static void addFile(IPropertyTree *files, const char *fileName, const char *cluster, unsigned usageCount, WUFileKind fileKind, const char *graphOwner)
  8545. {
  8546. StringBuffer path("File[@name=\"");
  8547. path.append(fileName).append("\"]");
  8548. if (cluster)
  8549. path.append("[@cluster=\"").append(cluster).append("\"]");
  8550. IPropertyTree *file = files->queryPropTree(path.str());
  8551. if (file) files->removeTree(file);
  8552. file = createPTree();
  8553. file->setProp("@name", fileName);
  8554. if (cluster)
  8555. file->setProp("@cluster", cluster);
  8556. if (graphOwner)
  8557. file->setProp("@graph", graphOwner);
  8558. file->setPropInt("@kind", (unsigned)fileKind);
  8559. if (WUFileTemporary == fileKind)
  8560. file->setPropInt("@usageCount", usageCount);
  8561. files->addPropTree("File", file);
  8562. }
  8563. void CLocalWorkUnit::addFile(const char *fileName, StringArray *clusters, unsigned usageCount, WUFileKind fileKind, const char *graphOwner)
  8564. {
  8565. CriticalBlock block(crit);
  8566. IPropertyTree *files = p->queryPropTree("Files");
  8567. if (!files)
  8568. files = p->addPropTree("Files");
  8569. if (!clusters)
  8570. ::addFile(files, fileName, NULL, usageCount, fileKind, graphOwner);
  8571. else
  8572. {
  8573. ForEachItemIn(c, *clusters)
  8574. ::addFile(files, fileName, clusters->item(c), usageCount, fileKind, graphOwner);
  8575. }
  8576. }
  8577. void CLocalWorkUnit::releaseFile(const char *fileName)
  8578. {
  8579. StringBuffer path("File[@name=\"");
  8580. path.append(fileName).append("\"]");
  8581. CriticalBlock block(crit);
  8582. IPropertyTree *files = p->queryPropTree("Files");
  8583. if (!files) return;
  8584. Owned<IPropertyTreeIterator> fiter = files->getElements(path.str());
  8585. ForEach (*fiter)
  8586. {
  8587. IPropertyTree *file = &fiter->query();
  8588. unsigned usageCount = file->getPropInt("@usageCount");
  8589. if (usageCount > 1)
  8590. file->setPropInt("@usageCount", usageCount-1);
  8591. else
  8592. {
  8593. StringAttr name(file->queryProp("@name"));
  8594. files->removeTree(file);
  8595. if (!name.isEmpty()&&(1 == usageCount))
  8596. {
  8597. if (queryDistributedFileDirectory().removeEntry(fileName, queryUserDescriptor()))
  8598. LOG(MCdebugProgress, unknownJob, "Removed (released) file %s from DFS", name.get());
  8599. }
  8600. }
  8601. }
  8602. }
  8603. void CLocalWorkUnit::clearGraphProgress() const
  8604. {
  8605. }
  8606. void CLocalWorkUnit::resetBeforeGeneration()
  8607. {
  8608. CriticalBlock block(crit);
  8609. //Remove all associated files
  8610. Owned<IWUQuery> q = updateQuery();
  8611. q->removeAssociatedFiles();
  8612. //Remove any pre-existing workflow information
  8613. workflowIterator.clear();
  8614. p->removeProp("Workflow");
  8615. }
  8616. unsigned CLocalWorkUnit::queryFileUsage(const char *fileName) const
  8617. {
  8618. StringBuffer path("Files/File[@name=\"");
  8619. path.append(fileName).append("\"]/@usageCount");
  8620. CriticalBlock block(crit);
  8621. return p->getPropInt(path.str());
  8622. }
  8623. IConstWUFileUsageIterator * CLocalWorkUnit::getFieldUsage() const
  8624. {
  8625. CriticalBlock block(crit);
  8626. IPropertyTree* fieldUsageTree = p->queryPropTree("usedsources");
  8627. if (!fieldUsageTree)
  8628. return NULL;
  8629. IPropertyTreeIterator* iter = fieldUsageTree->getElements("*");
  8630. return new CConstWUFileUsageIterator(iter);
  8631. }
  8632. bool isFilenameResolved(StringBuffer& filename)
  8633. {
  8634. size32_t length = filename.length();
  8635. // With current implementation, if filename is surrounded by single quotes, it means that the filename was resolved at compile time.
  8636. if (filename.length() >= 2 && filename.charAt(0) == '\'' && filename.charAt(length-1) == '\'')
  8637. return true;
  8638. else
  8639. return false;
  8640. }
  8641. bool CLocalWorkUnit::getFieldUsageArray(StringArray & filenames, StringArray & columnnames, const char * clusterName) const
  8642. {
  8643. bool scopeLoaded = false;
  8644. SCMStringBuffer defaultScope;
  8645. Owned<IConstWUFileUsageIterator> files = getFieldUsage();
  8646. if (!files)
  8647. return false; // this query was not compiled with recordFieldUsage option.
  8648. ForEach(*files)
  8649. {
  8650. Owned<IConstWUFileUsage> file = files->get();
  8651. StringBuffer filename(file->queryName());
  8652. size32_t length = filename.length();
  8653. if (length == 0)
  8654. throw MakeStringException(WUERR_InvalidFieldUsage, "Invalid FieldUsage found in WU. Cannot enforce view security.");
  8655. StringBuffer normalizedFilename;
  8656. // Two cases to handle:
  8657. // 1. Filename was known at compile time, and is surrounded in single quotes (i.e. 'filename').
  8658. // 2. Filename could not be resolved at compile time (i.e. filename is an input to a query),
  8659. // and is a raw expression WITHOUT surrounding single quotes (i.e. STORED('input_filename')).
  8660. if (isFilenameResolved(filename))
  8661. {
  8662. // filename cannot be empty (i.e. empty single quotes '')
  8663. if (length == 2)
  8664. throw MakeStringException(WUERR_InvalidFieldUsage, "Invalid FieldUsage found in WU. Cannot enforce view security.");
  8665. // Remove surrounding single quotes
  8666. StringAttr cleanFilename(filename.str()+1, length-2);
  8667. // When a filename doesn't start with a tilde (~), it means scope is omitted and is relying on a default scope.
  8668. // We need to load a default scope from config and prefix the filename with it.
  8669. if (cleanFilename.str()[0] != '~')
  8670. {
  8671. // loading a default scope from config is expensive, and should be only done once and be reused later.
  8672. if (!scopeLoaded)
  8673. {
  8674. Owned<IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(clusterName);
  8675. if (!clusterInfo)
  8676. throw MakeStringException(WUERR_InvalidCluster, "Unknown cluster %s", clusterName);
  8677. clusterInfo->getScope(defaultScope);
  8678. scopeLoaded = true;
  8679. }
  8680. normalizedFilename.append(defaultScope.str());
  8681. normalizedFilename.append(cleanFilename.str());
  8682. }
  8683. else
  8684. {
  8685. normalizedFilename.append(cleanFilename);
  8686. }
  8687. }
  8688. else
  8689. {
  8690. // When filename is an unresolved expression, simply treat the expression as a "non-existent" filename.
  8691. // It will have an effect of this query accessing a non-existent filename, and will be denied access unconditionally.
  8692. normalizedFilename.append(filename.str());
  8693. }
  8694. Owned<IConstWUFieldUsageIterator> fields = file->getFields();
  8695. ForEach(*fields)
  8696. {
  8697. Owned<IConstWUFieldUsage> field = fields->get();
  8698. filenames.append(normalizedFilename.str());
  8699. columnnames.append(field->queryName());
  8700. }
  8701. }
  8702. return true;
  8703. }
  8704. IPropertyTree *CLocalWorkUnit::getDiskUsageStats()
  8705. {
  8706. return p->getPropTree("DiskUsageStats");
  8707. }
  8708. void CLocalWorkUnit::addDiskUsageStats(__int64 _avgNodeUsage, unsigned _minNode, __int64 _minNodeUsage, unsigned _maxNode, __int64 _maxNodeUsage, __int64 _graphId)
  8709. {
  8710. IPropertyTree *stats = p->queryPropTree("DiskUsageStats");
  8711. offset_t maxNodeUsage;
  8712. if (stats)
  8713. maxNodeUsage = stats->getPropInt64("@maxNodeUsage");
  8714. else
  8715. {
  8716. stats = p->addPropTree("DiskUsageStats");
  8717. maxNodeUsage = 0;
  8718. }
  8719. if ((offset_t)_maxNodeUsage > maxNodeUsage)
  8720. {
  8721. // record all details at time of max node usage.
  8722. stats->setPropInt("@minNode", _minNode);
  8723. stats->setPropInt("@maxNode", _maxNode);
  8724. stats->setPropInt64("@minNodeUsage", _minNodeUsage);
  8725. stats->setPropInt64("@maxNodeUsage", _maxNodeUsage);
  8726. stats->setPropInt64("@graphId", _graphId);
  8727. if (_avgNodeUsage)
  8728. {
  8729. unsigned _skewHi = (unsigned)((100 * (_maxNodeUsage-_avgNodeUsage))/_avgNodeUsage);
  8730. unsigned _skewLo = (unsigned)((100 * (_avgNodeUsage-_minNodeUsage))/_avgNodeUsage);
  8731. stats->setPropInt("@skewHi", _skewHi);
  8732. stats->setPropInt("@skewLo", _skewLo);
  8733. }
  8734. }
  8735. }
  8736. IPropertyTreeIterator & CLocalWorkUnit::getFileIterator() const
  8737. {
  8738. CriticalBlock block(crit);
  8739. _loadFilesWritten();
  8740. return * p->getElements("Files/File");
  8741. }
  8742. IPropertyTreeIterator & CLocalWorkUnit::getFilesReadIterator() const
  8743. {
  8744. CriticalBlock block(crit);
  8745. _loadFilesRead();
  8746. return * p->getElements("FilesRead/File");
  8747. }
  8748. //=================================================================================================
  8749. bool CLocalWorkUnit::switchThorQueue(const char *cluster, IQueueSwitcher *qs)
  8750. {
  8751. CriticalBlock block(crit);
  8752. if (qs->isAuto()&&!getAllowAutoQueueSwitch())
  8753. return false;
  8754. Owned<IConstWUClusterInfo> newci = getTargetClusterInfo(cluster);
  8755. if (!newci)
  8756. return false;
  8757. StringBuffer currentcluster;
  8758. if (!p->getProp("@clusterName",currentcluster))
  8759. return false;
  8760. Owned<IConstWUClusterInfo> curci = getTargetClusterInfo(currentcluster.str());
  8761. if (!curci)
  8762. return false;
  8763. SCMStringBuffer curqname;
  8764. curci->getThorQueue(curqname);
  8765. const char *wuid = p->queryName();
  8766. void *qi = qs->getQ(curqname.str(),wuid);
  8767. if (!qi)
  8768. return false;
  8769. setClusterName(cluster);
  8770. SCMStringBuffer newqname;
  8771. newci->getThorQueue(newqname);
  8772. qs->putQ(newqname.str(),wuid,qi);
  8773. return true;
  8774. }
  8775. //=================================================================================================
  8776. IPropertyTree *CLocalWorkUnit::getUnpackedTree(bool includeProgress) const
  8777. {
  8778. Owned<IPropertyTree> ret = createPTreeFromIPT(p);
  8779. Owned<IConstWUGraphIterator> graphIter = &getGraphs(GraphTypeAny);
  8780. ForEach(*graphIter)
  8781. {
  8782. IConstWUGraph &graph = graphIter->query();
  8783. Owned<IPropertyTree> graphTree = graph.getXGMMLTree(includeProgress);
  8784. SCMStringBuffer gName;
  8785. graph.getName(gName);
  8786. StringBuffer xpath("Graphs/Graph[@name=\"");
  8787. xpath.append(gName.s).append("\"]/xgmml");
  8788. IPropertyTree *xgmml = ret->queryPropTree(xpath.str());
  8789. if (xgmml) // don't know of any reason it shouldn't exist
  8790. {
  8791. xgmml->removeProp("graphBin");
  8792. xgmml->setPropTree("graph", graphTree.getClear());
  8793. }
  8794. }
  8795. return ret.getClear();
  8796. }
  8797. void CLocalWorkUnit::_loadGraphs(bool heavy) const
  8798. {
  8799. Owned<IPropertyTreeIterator> iter = p->getElements("Graphs/Graph");
  8800. ForEach(*iter)
  8801. {
  8802. IPropertyTree &graph = iter->query();
  8803. graphs.append(*new CLocalWUGraph(*this, LINK(&graph)));
  8804. }
  8805. }
  8806. void CLocalWorkUnit::loadGraphs(bool heavy) const
  8807. {
  8808. if (graphsCached < (heavy ? 2 : 1))
  8809. {
  8810. graphs.kill();
  8811. _loadGraphs(heavy);
  8812. graphsCached = (heavy ? 2 : 1);
  8813. }
  8814. }
  8815. EnumMapping graphTypes[] = {
  8816. { GraphTypeAny, "unknown" },
  8817. { GraphTypeProgress, "progress" },
  8818. { GraphTypeEcl, "ECL" },
  8819. { GraphTypeActivities, "activities" },
  8820. { GraphTypeSubProgress, "subgraph" },
  8821. { GraphTypeSize, NULL },
  8822. };
  8823. WUGraphType getGraphTypeFromString(const char* type)
  8824. {
  8825. return (WUGraphType) getEnum(type, graphTypes);
  8826. }
  8827. CLocalWUGraph::CLocalWUGraph(const CLocalWorkUnit &_owner, IPropertyTree *props) : p(props), owner(_owner)
  8828. {
  8829. wuidVersion = owner.getWuidVersion();
  8830. }
  8831. IStringVal& CLocalWUGraph::getName(IStringVal &str) const
  8832. {
  8833. str.set(p->queryProp("@name"));
  8834. return str;
  8835. }
  8836. IStringVal& CLocalWUGraph::getLabel(IStringVal &str) const
  8837. {
  8838. if (wuidVersion >= 2)
  8839. {
  8840. str.set(p->queryProp("@label"));
  8841. return str;
  8842. }
  8843. else
  8844. {
  8845. Owned<IPropertyTree> xgmml = getXGMMLTree(false);
  8846. str.set(xgmml->queryProp("@label"));
  8847. return str;
  8848. }
  8849. }
  8850. WUGraphState CLocalWUGraph::getState() const
  8851. {
  8852. return owner.queryGraphState(p->queryProp("@name"));
  8853. }
  8854. IStringVal& CLocalWUGraph::getXGMML(IStringVal &str, bool mergeProgress) const
  8855. {
  8856. Owned<IPropertyTree> xgmml = getXGMMLTree(mergeProgress);
  8857. if (xgmml)
  8858. {
  8859. StringBuffer x;
  8860. toXML(xgmml, x);
  8861. str.set(x.str());
  8862. }
  8863. return str;
  8864. }
  8865. unsigned CLocalWorkUnit::getGraphCount() const
  8866. {
  8867. CriticalBlock block(crit);
  8868. if (p->hasProp("Graphs"))
  8869. {
  8870. return p->queryPropTree("Graphs")->numChildren();
  8871. }
  8872. return 0;
  8873. }
  8874. unsigned CLocalWorkUnit::getSourceFileCount() const
  8875. {
  8876. CriticalBlock block(crit);
  8877. _loadFilesRead();
  8878. if (p->hasProp("FilesRead"))
  8879. {
  8880. return p->queryPropTree("FilesRead")->numChildren();
  8881. }
  8882. return 0;
  8883. }
  8884. unsigned CLocalWorkUnit::getResultCount() const
  8885. {
  8886. CriticalBlock block(crit);
  8887. if (p->hasProp("Results"))
  8888. {
  8889. return p->queryPropTree("Results")->numChildren();
  8890. }
  8891. return 0;
  8892. }
  8893. unsigned CLocalWorkUnit::getVariableCount() const
  8894. {
  8895. CriticalBlock block(crit);
  8896. if (p->hasProp("Variables"))
  8897. {
  8898. return p->queryPropTree("Variables")->numChildren();
  8899. }
  8900. return 0;
  8901. }
  8902. unsigned CLocalWorkUnit::getApplicationValueCount() const
  8903. {
  8904. CriticalBlock block(crit);
  8905. if (p->hasProp("Application"))
  8906. {
  8907. return p->queryPropTree("Application")->numChildren();
  8908. }
  8909. return 0;
  8910. }
  8911. StringBuffer &appendPTreeOpenTag(StringBuffer &s, IPropertyTree *tree, const char *name, unsigned indent, bool hidePasswords)
  8912. {
  8913. appendXMLOpenTag(s, name, NULL, false);
  8914. Owned<IAttributeIterator> attrs = tree->getAttributes(true);
  8915. if (attrs->first())
  8916. {
  8917. unsigned attributeindent = indent + (size32_t) strlen(name);
  8918. unsigned count = attrs->count();
  8919. bool doindent = false;
  8920. ForEach(*attrs)
  8921. {
  8922. if (hidePasswords && streq(attrs->queryName(), "@token"))
  8923. continue;
  8924. #ifndef _DEBUG
  8925. if (hidePasswords && streq(attrs->queryName(), "@distributedAccessToken"))
  8926. continue;
  8927. #endif
  8928. if (doindent)
  8929. s.append('\n').appendN(attributeindent, ' ');
  8930. else if (count > 3)
  8931. doindent = true;
  8932. appendXMLAttr(s, attrs->queryName()+1, attrs->queryValue());
  8933. }
  8934. }
  8935. s.append('>');
  8936. return s;
  8937. }
  8938. IStringVal &CLocalWorkUnit::getXmlParams(IStringVal &str, bool hidePasswords) const
  8939. {
  8940. CriticalBlock block(crit);
  8941. IPropertyTree *paramTree = p->queryPropTree("Parameters");
  8942. if (!paramTree)
  8943. return str;
  8944. StringBuffer xml;
  8945. if (!hidePasswords)
  8946. toXML(paramTree, xml);
  8947. else
  8948. {
  8949. appendPTreeOpenTag(xml.append(' '), paramTree, "Parameters", 0, false).append('\n');
  8950. Owned<IPropertyTreeIterator> elems = paramTree->getElements("*");
  8951. ForEach(*elems)
  8952. {
  8953. const char *paramname = elems->query().queryName();
  8954. VStringBuffer xpath("Variables/Variable[@name='%s']/Format/@password", paramname);
  8955. if (p->getPropBool(xpath))
  8956. appendXMLTag(xml.append(" "), paramname, "***").append('\n');
  8957. else
  8958. toXML(&elems->query(), xml, 2);
  8959. }
  8960. appendXMLCloseTag(xml.append(' '), "Parameters").append('\n');
  8961. }
  8962. str.set(xml);
  8963. return str;
  8964. }
  8965. const IPropertyTree *CLocalWorkUnit::getXmlParams() const
  8966. {
  8967. CriticalBlock block(crit);
  8968. return p->getPropTree("Parameters");
  8969. }
  8970. void CLocalWorkUnit::setXmlParams(const char *params)
  8971. {
  8972. CriticalBlock block(crit);
  8973. p->setPropTree("Parameters", createPTreeFromXMLString(params));
  8974. }
  8975. void CLocalWorkUnit::setXmlParams(IPropertyTree *tree)
  8976. {
  8977. CriticalBlock block(crit);
  8978. p->setPropTree("Parameters", tree);
  8979. }
  8980. unsigned __int64 CLocalWorkUnit::getHash() const
  8981. {
  8982. CriticalBlock block(crit);
  8983. return p->getPropInt64("@hash");
  8984. }
  8985. void CLocalWorkUnit::setHash(unsigned __int64 hash)
  8986. {
  8987. CriticalBlock block(crit);
  8988. p->setPropInt64("@hash", hash);
  8989. }
  8990. // getGraphs / getGraphsMeta
  8991. // These are basically the same except for the amount of preloading they do, and the type of the iterator they return...
  8992. // If a type other than any is requested, a postfilter is needed.
  8993. template <class T, class U> class CFilteredGraphIteratorOf : public CInterfaceOf<T>
  8994. {
  8995. WUGraphType type;
  8996. Owned<T> base;
  8997. bool match()
  8998. {
  8999. return base->query().getType()==type;
  9000. }
  9001. public:
  9002. CFilteredGraphIteratorOf<T,U>(T *_base, WUGraphType _type)
  9003. : base(_base), type(_type)
  9004. {
  9005. }
  9006. bool first()
  9007. {
  9008. if (!base->first())
  9009. return false;
  9010. if (match())
  9011. return true;
  9012. return next();
  9013. }
  9014. bool next()
  9015. {
  9016. while (base->next())
  9017. if (match())
  9018. return true;
  9019. return false;
  9020. }
  9021. virtual bool isValid()
  9022. {
  9023. return base->isValid();
  9024. }
  9025. U & query()
  9026. {
  9027. return base->query();
  9028. }
  9029. };
  9030. IConstWUGraphMetaIterator& CLocalWorkUnit::getGraphsMeta(WUGraphType type) const
  9031. {
  9032. /* NB: this method should be 'cheap', loadGraphs() creates IConstWUGraph interfaces to the graphs
  9033. * it does not actually pull the graph data. We only use IConstWUGraphMeta here, which never probes the xgmml.
  9034. */
  9035. CriticalBlock block(crit);
  9036. loadGraphs(false);
  9037. IConstWUGraphMetaIterator *giter = new CArrayIteratorOf<IConstWUGraph,IConstWUGraphMetaIterator> (graphs, 0, (IConstWorkUnit *) this);
  9038. if (type!=GraphTypeAny)
  9039. giter = new CFilteredGraphIteratorOf<IConstWUGraphMetaIterator, IConstWUGraphMeta>(giter,type);
  9040. return *giter;
  9041. }
  9042. IConstWUGraphIterator& CLocalWorkUnit::getGraphs(WUGraphType type) const
  9043. {
  9044. CriticalBlock block(crit);
  9045. loadGraphs(true);
  9046. IConstWUGraphIterator *giter = new CArrayIteratorOf<IConstWUGraph,IConstWUGraphIterator> (graphs, 0, (IConstWorkUnit *) this);
  9047. if (type!=GraphTypeAny)
  9048. giter = new CFilteredGraphIteratorOf<IConstWUGraphIterator, IConstWUGraph>(giter, type);
  9049. return *giter;
  9050. }
  9051. IConstWUGraph* CLocalWorkUnit::getGraph(const char *qname) const
  9052. {
  9053. CriticalBlock block(crit);
  9054. VStringBuffer xpath("Graphs/Graph[@name='%s']", qname);
  9055. // NOTE - this would go wrong if we had other graphs of same name but different type. Ignore for now.
  9056. IPTree *graph = p->queryPropTree(xpath);
  9057. if (graph)
  9058. return new CLocalWUGraph(*this, LINK(graph));
  9059. return NULL;
  9060. }
  9061. void CLocalWorkUnit::createGraph(const char * name, const char *label, WUGraphType type, IPropertyTree *xgmml, unsigned wfid)
  9062. {
  9063. CriticalBlock block(crit);
  9064. if (!graphs.length())
  9065. p->addPropTree("Graphs");
  9066. IPropertyTree *r = p->queryPropTree("Graphs");
  9067. IPropertyTree *s = r->addPropTree("Graph");
  9068. CLocalWUGraph *q = new CLocalWUGraph(*this, LINK(s));
  9069. q->setName(name);
  9070. q->setLabel(label);
  9071. q->setType(type);
  9072. q->setWfid(wfid);
  9073. q->setXGMMLTree(xgmml);
  9074. graphs.append(*q);
  9075. }
  9076. IConstWUGraphProgress *CLocalWorkUnit::getGraphProgress(const char *name) const
  9077. {
  9078. /* Owned<IRemoteConnection> conn = getProgressConnection();
  9079. if (conn)
  9080. {
  9081. IPTree *progress = conn->queryRoot()->queryPropTree(graphName);
  9082. if (progress)
  9083. return new CConstGraphProgress(p->queryName(), graphName, progress);
  9084. }
  9085. */
  9086. return NULL;
  9087. }
  9088. WUGraphState CLocalWorkUnit::queryGraphState(const char *graphName) const
  9089. {
  9090. throwUnexpected(); // Should only be used for persisted workunits
  9091. }
  9092. WUGraphState CLocalWorkUnit::queryNodeState(const char *graphName, WUGraphIDType nodeId) const
  9093. {
  9094. throwUnexpected(); // Should only be used for persisted workunits
  9095. }
  9096. void CLocalWorkUnit::setGraphState(const char *graphName, unsigned wfid, WUGraphState state) const
  9097. {
  9098. throwUnexpected(); // Should only be used for persisted workunits
  9099. }
  9100. void CLocalWorkUnit::setNodeState(const char *graphName, WUGraphIDType nodeId, WUGraphState state) const
  9101. {
  9102. throwUnexpected(); // Should only be used for persisted workunits
  9103. }
  9104. IWUGraphStats *CLocalWorkUnit::updateStats(const char *graphName, StatisticCreatorType creatorType, const char * creator, unsigned _wfid, unsigned subgraph) const
  9105. {
  9106. return new CWuGraphStats(LINK(p), creatorType, creator, _wfid, graphName, subgraph);
  9107. }
  9108. void CLocalWUGraph::setName(const char *str)
  9109. {
  9110. p->setProp("@name", str);
  9111. }
  9112. void CLocalWUGraph::setLabel(const char *str)
  9113. {
  9114. p->setProp("@label", str);
  9115. }
  9116. void CLocalWUGraph::setWfid(unsigned wfid)
  9117. {
  9118. p->setPropInt("@wfid", wfid);
  9119. }
  9120. void CLocalWUGraph::setXGMML(const char *str)
  9121. {
  9122. setXGMMLTree(createPTreeFromXMLString(str,ipt_lowmem));
  9123. }
  9124. void CLocalWUGraph::setXGMMLTree(IPropertyTree *_graph)
  9125. {
  9126. assertex(strcmp(_graph->queryName(), "graph")==0);
  9127. IPropertyTree *xgmml = p->setPropTree("xgmml");
  9128. MemoryBuffer mb;
  9129. _graph->serialize(mb);
  9130. // Note - we could compress further but that would introduce compatibility concerns, so don't bother
  9131. // Cassandra workunit code actually lzw compresses the parent anyway
  9132. xgmml->setPropBin("graphBin", mb.length(), mb.toByteArray());
  9133. graph.setown(_graph);
  9134. }
  9135. static void expandAttributes(IPropertyTree & targetNode, IPropertyTree & progressNode)
  9136. {
  9137. Owned<IAttributeIterator> aIter = progressNode.getAttributes();
  9138. ForEach (*aIter)
  9139. {
  9140. const char *aName = aIter->queryName()+1;
  9141. if (0 != stricmp("id", aName)) // "id" reserved.
  9142. {
  9143. IPropertyTree *att = targetNode.addPropTree("att");
  9144. att->setProp("@name", aName);
  9145. att->setProp("@value", aIter->queryValue());
  9146. }
  9147. }
  9148. }
  9149. void CLocalWUGraph::mergeProgress(IPropertyTree &rootNode, IPropertyTree &progressTree, const unsigned &progressV) const
  9150. {
  9151. IPropertyTree *graphNode = rootNode.queryPropTree("att/graph");
  9152. if (!graphNode) return;
  9153. unsigned nodeId = rootNode.getPropInt("@id");
  9154. StringBuffer progressNodePath("node[@id=\"");
  9155. progressNodePath.append(nodeId).append("\"]");
  9156. IPropertyTree *progressNode = progressTree.queryPropTree(progressNodePath.str());
  9157. if (progressNode)
  9158. {
  9159. expandAttributes(*graphNode, *progressNode);
  9160. Owned<IPropertyTreeIterator> edges = progressNode->getElements("edge");
  9161. ForEach (*edges)
  9162. {
  9163. IPropertyTree &edge = edges->query();
  9164. StringBuffer edgePath("edge[@id=\"");
  9165. edgePath.append(edge.queryProp("@id")).append("\"]");
  9166. IPropertyTree *graphEdge = graphNode->queryPropTree(edgePath.str());
  9167. if (graphEdge)
  9168. {
  9169. if (progressV < 1)
  9170. mergePTree(graphEdge, &edge);
  9171. else
  9172. { // must translate to XGMML format
  9173. expandAttributes(*graphEdge, edge);
  9174. // This is really only here, so that our progress format can use non-attribute values, which have different efficiency qualifies (e.g. can be external by dali)
  9175. Owned<IPropertyTreeIterator> iter = edge.getElements("*");
  9176. ForEach (*iter)
  9177. {
  9178. IPropertyTree &t = iter->query();
  9179. IPropertyTree *att = graphEdge->addPropTree("att");
  9180. att->setProp("@name", t.queryName());
  9181. att->setProp("@value", t.queryProp(NULL));
  9182. }
  9183. }
  9184. }
  9185. }
  9186. Owned<IPropertyTreeIterator> nodes = progressNode->getElements("node");
  9187. ForEach (*nodes)
  9188. {
  9189. IPropertyTree &node = nodes->query();
  9190. StringBuffer nodePath("node[@id=\"");
  9191. nodePath.append(node.queryProp("@id")).append("\"]");
  9192. IPropertyTree *_node = graphNode->queryPropTree(nodePath.str());
  9193. if (_node)
  9194. {
  9195. if (progressV < 1)
  9196. mergePTree(_node, &node);
  9197. else
  9198. { // must translate to XGMML format
  9199. expandAttributes(*_node, node);
  9200. }
  9201. }
  9202. }
  9203. }
  9204. Owned<IPropertyTreeIterator> iter = graphNode->getElements("node");
  9205. ForEach (*iter)
  9206. mergeProgress(iter->query(), progressTree, progressV);
  9207. }
  9208. IPropertyTree * CLocalWUGraph::getXGMMLTreeRaw() const
  9209. {
  9210. return p->getPropTree("xgmml");
  9211. }
  9212. IPropertyTree * CLocalWUGraph::getXGMMLTree(bool doMergeProgress) const
  9213. {
  9214. if (!graph)
  9215. {
  9216. // NB: although graphBin introduced in wuidVersion==2,
  9217. // daliadmin can retrospectively compress existing graphs, so need to check for all versions
  9218. MemoryBuffer mb;
  9219. if (p->getPropBin("xgmml/graphBin", mb))
  9220. graph.setown(createPTree(mb, ipt_lowmem));
  9221. else
  9222. graph.setown(p->getBranch("xgmml/graph"));
  9223. if (!graph)
  9224. return NULL;
  9225. }
  9226. if (!doMergeProgress)
  9227. return graph.getLink();
  9228. else
  9229. {
  9230. Owned<IPropertyTree> copy = createPTreeFromIPT(graph, ipt_lowmem);
  9231. Owned<IConstWUGraphProgress> progress = owner.getGraphProgress(p->queryProp("@name"));
  9232. if (progress)
  9233. {
  9234. //MORE: Eventually this should directly access the new stats structure
  9235. unsigned progressV = progress->queryFormatVersion();
  9236. Owned<IPropertyTree> progressTree = progress->getProgressTree();
  9237. Owned<IPropertyTreeIterator> nodeIterator = copy->getElements("node");
  9238. ForEach (*nodeIterator)
  9239. mergeProgress(nodeIterator->query(), *progressTree, progressV);
  9240. }
  9241. return copy.getClear();
  9242. }
  9243. }
  9244. WUGraphType CLocalWUGraph::getType() const
  9245. {
  9246. return (WUGraphType) getEnum(p, "@type", graphTypes);
  9247. }
  9248. IStringVal & CLocalWUGraph::getTypeName(IStringVal &str) const
  9249. {
  9250. str.set(p->queryProp("@type"));
  9251. if (!str.length())
  9252. str.set("unknown");
  9253. return str;
  9254. }
  9255. unsigned CLocalWUGraph::getWfid() const
  9256. {
  9257. return p->getPropInt("@wfid", 0);
  9258. }
  9259. void CLocalWUGraph::setType(WUGraphType _type)
  9260. {
  9261. setEnum(p, "@type", _type, graphTypes);
  9262. }
  9263. //=================================================================================================
  9264. EnumMapping queryFileTypes[] = {
  9265. { FileTypeCpp, "cpp" },
  9266. { FileTypeDll, "dll" },
  9267. { FileTypeResText, "res" },
  9268. { FileTypeHintXml, "hint" },
  9269. { FileTypeXml, "xml" },
  9270. { FileTypeLog, "log" },
  9271. { FileTypeSize, NULL },
  9272. };
  9273. CLocalWUAssociated::CLocalWUAssociated(IPropertyTree *props) : p(props)
  9274. {
  9275. }
  9276. WUFileType CLocalWUAssociated::getType() const
  9277. {
  9278. return (WUFileType)getEnum(p, "@type", queryFileTypes);
  9279. }
  9280. IStringVal & CLocalWUAssociated::getDescription(IStringVal & str) const
  9281. {
  9282. str.set(p->queryProp("@desc"));
  9283. return str;
  9284. }
  9285. IStringVal & CLocalWUAssociated::getIp(IStringVal & str) const
  9286. {
  9287. str.set(p->queryProp("@ip"));
  9288. return str;
  9289. }
  9290. IStringVal & CLocalWUAssociated::getName(IStringVal & str) const
  9291. {
  9292. str.set(p->queryProp("@filename"));
  9293. return str;
  9294. }
  9295. IStringVal & CLocalWUAssociated::getNameTail(IStringVal & str) const
  9296. {
  9297. str.set(pathTail(p->queryProp("@filename")));
  9298. return str;
  9299. }
  9300. unsigned CLocalWUAssociated::getCrc() const
  9301. {
  9302. return p->getPropInt("@crc", 0);
  9303. }
  9304. unsigned CLocalWUAssociated::getMinActivityId() const
  9305. {
  9306. return p->getPropInt("@minActivity", 0);
  9307. }
  9308. unsigned CLocalWUAssociated::getMaxActivityId() const
  9309. {
  9310. return p->getPropInt("@maxActivity", 0);
  9311. }
  9312. //=================================================================================================
  9313. CLocalWUQuery::CLocalWUQuery(IPropertyTree *props) : p(props)
  9314. {
  9315. associatedCached = false;
  9316. }
  9317. EnumMapping queryTypes[] = {
  9318. { QueryTypeUnknown, "unknown" },
  9319. { QueryTypeEcl, "ECL" },
  9320. { QueryTypeSql, "SQL" },
  9321. { QueryTypeXml, "XML" },
  9322. { QueryTypeAttribute, "Attribute" },
  9323. { QueryTypeSize, NULL },
  9324. };
  9325. WUQueryType CLocalWUQuery::getQueryType() const
  9326. {
  9327. return (WUQueryType) getEnum(p, "@type", queryTypes);
  9328. }
  9329. void CLocalWUQuery::setQueryType(WUQueryType qt)
  9330. {
  9331. setEnum(p, "@type", qt, queryTypes);
  9332. }
  9333. IStringVal& CLocalWUQuery::getQueryText(IStringVal &str) const
  9334. {
  9335. const char *text = p->queryProp("Text");
  9336. if (!text)
  9337. text = p->queryProp("ShortText");
  9338. str.set(text);
  9339. return str;
  9340. }
  9341. IStringVal& CLocalWUQuery::getQueryShortText(IStringVal &str) const
  9342. {
  9343. const char * text = p->queryProp("ShortText");
  9344. if (text)
  9345. str.set(text);
  9346. else
  9347. {
  9348. text = p->queryProp("Text");
  9349. if (isArchiveQuery(text))
  9350. {
  9351. Owned<IPropertyTree> xml = createPTreeFromXMLString(text, ipt_caseInsensitive|ipt_lowmem);
  9352. const char * path = xml->queryProp("Query/@attributePath");
  9353. if (path)
  9354. {
  9355. IPropertyTree * resolved = resolveDefinitionInArchive(xml, path);
  9356. if (resolved)
  9357. str.set(resolved->queryProp(NULL));
  9358. }
  9359. else
  9360. str.set(xml->queryProp("Query"));
  9361. }
  9362. else
  9363. str.set(text);
  9364. }
  9365. return str;
  9366. }
  9367. bool CLocalWUQuery::isArchive() const
  9368. {
  9369. if (p->hasProp("@isArchive"))
  9370. return p->getPropBool("@isArchive");
  9371. const char *text = p->queryProp("Text");
  9372. return isArchiveQuery(text);
  9373. }
  9374. IStringVal& CLocalWUQuery::getQueryName(IStringVal &str) const
  9375. {
  9376. str.set(p->queryProp("@name"));
  9377. return str;
  9378. }
  9379. IStringVal & CLocalWUQuery::getQueryMainDefinition(IStringVal & str) const
  9380. {
  9381. str.set(p->queryProp("@main"));
  9382. return str;
  9383. }
  9384. IStringVal& CLocalWUQuery::getQueryDllName(IStringVal &str) const
  9385. {
  9386. Owned<IConstWUAssociatedFile> entry = getAssociatedFile(FileTypeDll, 0);
  9387. if (entry)
  9388. entry->getNameTail(str);
  9389. return str;
  9390. }
  9391. IStringVal& CLocalWUQuery::getQueryCppName(IStringVal &str) const
  9392. {
  9393. Owned<IConstWUAssociatedFile> entry = getAssociatedFile(FileTypeCpp, 0);
  9394. if (entry)
  9395. entry->getName(str);
  9396. return str;
  9397. }
  9398. IStringVal& CLocalWUQuery::getQueryResTxtName(IStringVal &str) const
  9399. {
  9400. Owned<IConstWUAssociatedFile> entry = getAssociatedFile(FileTypeResText, 0);
  9401. if (entry)
  9402. entry->getName(str);
  9403. return str;
  9404. }
  9405. unsigned CLocalWUQuery::getQueryDllCrc() const
  9406. {
  9407. Owned<IConstWUAssociatedFile> entry = getAssociatedFile(FileTypeDll, 0);
  9408. if (entry)
  9409. return entry->getCrc();
  9410. return 0;
  9411. }
  9412. void CLocalWUQuery::setQueryText(const char *text)
  9413. {
  9414. bool isArchive = isArchiveQuery(text);
  9415. if (isArchive)
  9416. {
  9417. p->setProp("Text", text);
  9418. Owned<IPropertyTree> xml = createPTreeFromXMLString(text, ipt_caseInsensitive|ipt_lowmem);
  9419. const char * path = xml->queryProp("Query/@attributePath");
  9420. if (path)
  9421. {
  9422. IPropertyTree * resolved = resolveDefinitionInArchive(xml, path);
  9423. if (resolved)
  9424. p->setProp("ShortText", resolved->queryProp(NULL));
  9425. }
  9426. else
  9427. p->setProp("ShortText", xml->queryProp("Query"));
  9428. }
  9429. else
  9430. {
  9431. p->setProp("Text", text); // At some point in the future we may be able to remove this,
  9432. // but as long as there may be new workunits compiled by old systems, we can't
  9433. p->setProp("ShortText", text);
  9434. }
  9435. p->setPropBool("@isArchive", isArchive);
  9436. if (isArchive)
  9437. p->setPropBool("@hasArchive", true); //preserved if setQueryText is called multiple times. Should setting this be more explicit?
  9438. }
  9439. void CLocalWUQuery::setQueryName(const char *qname)
  9440. {
  9441. p->setProp("@name", qname);
  9442. }
  9443. void CLocalWUQuery::setQueryMainDefinition(const char * str)
  9444. {
  9445. p->setProp("@main", str);
  9446. }
  9447. void CLocalWUQuery::addAssociatedFile(WUFileType type, const char * name, const char * ip, const char * desc, unsigned crc, unsigned minActivity, unsigned maxActivity)
  9448. {
  9449. CriticalBlock block(crit);
  9450. loadAssociated();
  9451. StringBuffer xpath;
  9452. xpath.append("Associated/File[@filename=\"").append(name).append("\"]");
  9453. if (p->hasProp(xpath))
  9454. return;
  9455. if (!associated.length())
  9456. p->addPropTree("Associated");
  9457. IPropertyTree *pl = p->queryPropTree("Associated");
  9458. IPropertyTree *s = pl->addPropTree("File");
  9459. setEnum(s, "@type", type, queryFileTypes);
  9460. s->setProp("@filename", name);
  9461. s->setProp("@ip", ip);
  9462. s->setProp("@desc", desc);
  9463. if (crc)
  9464. s->setPropInt("@crc", crc);
  9465. if (minActivity)
  9466. s->setPropInt("@minActivity", minActivity);
  9467. if (maxActivity)
  9468. s->setPropInt("@maxActivity", maxActivity);
  9469. IConstWUAssociatedFile * q = new CLocalWUAssociated(LINK(s));
  9470. associated.append(*q);
  9471. }
  9472. void CLocalWUQuery::removeAssociatedFile(WUFileType type, const char * name, const char * desc)
  9473. {
  9474. CriticalBlock block(crit);
  9475. associatedCached = false;
  9476. associated.kill();
  9477. StringBuffer xpath;
  9478. xpath.append("Associated/File");
  9479. if (type)
  9480. xpath.append("[@type=\"").append(getEnumText(type, queryFileTypes)).append("\"]");
  9481. if (name)
  9482. xpath.append("[@filename=\"").append(name).append("\"]");
  9483. if (desc)
  9484. xpath.append("[@desc=\"").append(desc).append("\"]");
  9485. p->removeProp(xpath.str());
  9486. }
  9487. void CLocalWUQuery::removeAssociatedFiles()
  9488. {
  9489. associatedCached = false;
  9490. associated.kill();
  9491. p->removeProp("Associated");
  9492. }
  9493. IConstWUAssociatedFile * CLocalWUQuery::getAssociatedFile(WUFileType type, unsigned index) const
  9494. {
  9495. CriticalBlock block(crit);
  9496. loadAssociated();
  9497. ForEachItemIn(idx, associated)
  9498. {
  9499. CLocalWUAssociated &cur = static_cast<CLocalWUAssociated &>(associated.item(idx));
  9500. if (cur.getType() == type)
  9501. {
  9502. if (index-- == 0)
  9503. return &OLINK(cur);
  9504. }
  9505. }
  9506. return NULL;
  9507. }
  9508. void CLocalWUQuery::addSpecialCaseAssociated(WUFileType type, const char * propname, unsigned crc) const
  9509. {
  9510. const char * name = p->queryProp(propname);
  9511. if (name)
  9512. {
  9513. IPropertyTree *s = createPTree("File");
  9514. setEnum(s, "@type", type, queryFileTypes);
  9515. s->setProp("@filename", name);
  9516. if (crc)
  9517. s->setPropInt("@crc", crc);
  9518. associated.append(*new CLocalWUAssociated(s));
  9519. }
  9520. }
  9521. void CLocalWUQuery::loadAssociated() const
  9522. {
  9523. CriticalBlock block(crit);
  9524. if (!associatedCached)
  9525. {
  9526. assertex(associated.length() == 0);
  9527. addSpecialCaseAssociated(FileTypeDll, "DllName", p->getPropInt("DllCrc", 0));
  9528. addSpecialCaseAssociated(FileTypeCpp, "CppName", 0);
  9529. addSpecialCaseAssociated(FileTypeResText, "ResTxtName", 0);
  9530. Owned<IPropertyTreeIterator> r = p->getElements("Associated/File");
  9531. for (r->first(); r->isValid(); r->next())
  9532. {
  9533. IPropertyTree *rp = &r->query();
  9534. rp->Link();
  9535. associated.append(*new CLocalWUAssociated(rp));
  9536. }
  9537. associatedCached = true;
  9538. }
  9539. }
  9540. IConstWUAssociatedFileIterator& CLocalWUQuery::getAssociatedFiles() const
  9541. {
  9542. CriticalBlock block(crit);
  9543. loadAssociated();
  9544. return *new CArrayIteratorOf<IConstWUAssociatedFile,IConstWUAssociatedFileIterator> (associated, 0, (IConstWUQuery *) this);
  9545. }
  9546. //========================================================================================
  9547. CLocalWUWebServicesInfo::CLocalWUWebServicesInfo(IPropertyTree *props) : p(props)
  9548. {
  9549. }
  9550. IStringVal& CLocalWUWebServicesInfo::getModuleName(IStringVal &str) const
  9551. {
  9552. str.set(p->queryProp("@module"));
  9553. return str;
  9554. }
  9555. IStringVal& CLocalWUWebServicesInfo::getAttributeName(IStringVal &str) const
  9556. {
  9557. str.set(p->queryProp("@attribute"));
  9558. return str;
  9559. }
  9560. IStringVal& CLocalWUWebServicesInfo::getDefaultName(IStringVal &str) const
  9561. {
  9562. str.set(p->queryProp("@defaultName"));
  9563. return str;
  9564. }
  9565. unsigned CLocalWUWebServicesInfo::getWebServicesCRC() const
  9566. {
  9567. return (unsigned) p->getPropInt("@crc");
  9568. }
  9569. IStringVal& CLocalWUWebServicesInfo::getInfo(const char *name, IStringVal &str) const
  9570. {
  9571. if (!name)
  9572. {
  9573. StringBuffer ws_info;
  9574. ws_info.appendf("<%s ", p->queryName());
  9575. Owned<IAttributeIterator> attrs = p->getAttributes();
  9576. for(attrs->first(); attrs->isValid(); attrs->next())
  9577. {
  9578. const char *name = attrs->queryName()+1;
  9579. const char *value = attrs->queryValue();
  9580. ws_info.appendf("%s='%s' ", name, value);
  9581. }
  9582. ws_info.append("> \n");
  9583. Owned<IPropertyTreeIterator> info = p->getElements("*");
  9584. ForEach(*info)
  9585. {
  9586. IPropertyTree &item = info->query();
  9587. const char *name = item.queryName();
  9588. if (name)
  9589. {
  9590. MemoryBuffer mb;
  9591. bool isbin = p->isBinary(name);
  9592. if (isbin)
  9593. {
  9594. p->getPropBin(name,mb);
  9595. if (mb.length())
  9596. {
  9597. unsigned len = 0;
  9598. mb.read(len);
  9599. StringBuffer encodedString;
  9600. StringBuffer val(len, (const char *) mb.readDirect(len));
  9601. encodeXML(val, encodedString);
  9602. ws_info.appendf("<%s>%s</%s>", name, encodedString.str(), name);
  9603. }
  9604. }
  9605. else
  9606. {
  9607. StringBuffer tmp;
  9608. toXML(&item, tmp);
  9609. ws_info.append(tmp.str());
  9610. }
  9611. }
  9612. }
  9613. ws_info.appendf("</%s>", p->queryName());
  9614. str.setLen(ws_info.str(), ws_info.length());
  9615. }
  9616. else
  9617. {
  9618. MemoryBuffer mb;
  9619. p->getPropBin(name,mb);
  9620. if (mb.length())
  9621. {
  9622. unsigned len;
  9623. mb.read(len);
  9624. str.setLen((const char *) mb.readDirect(len), len);
  9625. }
  9626. }
  9627. return str;
  9628. }
  9629. IStringVal& CLocalWUWebServicesInfo::getText(const char *name, IStringVal &str) const
  9630. {
  9631. str.set(p->queryProp(name));
  9632. return str;
  9633. }
  9634. void CLocalWUWebServicesInfo::setModuleName(const char *mname)
  9635. {
  9636. p->setProp("@module", mname);
  9637. }
  9638. void CLocalWUWebServicesInfo::setAttributeName(const char *aname)
  9639. {
  9640. p->setProp("@attribute", aname);
  9641. }
  9642. void CLocalWUWebServicesInfo::setDefaultName(const char *dname)
  9643. {
  9644. p->setProp("@defaultName", dname);
  9645. }
  9646. void CLocalWUWebServicesInfo::setWebServicesCRC(unsigned crc)
  9647. {
  9648. p->setPropInt("@crc", crc);
  9649. }
  9650. void CLocalWUWebServicesInfo::setInfo(const char *name, const char *info)
  9651. {
  9652. MemoryBuffer m;
  9653. unsigned len = (size32_t)strlen(info);
  9654. serializeLPString(len, info, m);
  9655. p->setPropBin(name, m.length(), m.toByteArray());
  9656. }
  9657. void CLocalWUWebServicesInfo::setText(const char *name, const char *info)
  9658. {
  9659. p->setProp(name, info);
  9660. }
  9661. //========================================================================================
  9662. CLocalWUResult::CLocalWUResult(IPropertyTree *props) : p(props)
  9663. {
  9664. }
  9665. EnumMapping resultStatuses[] = {
  9666. { ResultStatusUndefined, "undefined" },
  9667. { ResultStatusCalculated, "calculated" },
  9668. { ResultStatusSupplied, "supplied" },
  9669. { ResultStatusFailed, "failed" },
  9670. { ResultStatusPartial, "partial" },
  9671. { ResultStatusSize, NULL }
  9672. };
  9673. WUResultStatus CLocalWUResult::getResultStatus() const
  9674. {
  9675. return (WUResultStatus ) getEnum(p, "@status", resultStatuses);
  9676. }
  9677. IStringVal& CLocalWUResult::getResultName(IStringVal &str) const
  9678. {
  9679. str.set(p->queryProp("@name"));
  9680. return str;
  9681. }
  9682. int CLocalWUResult::getResultSequence() const
  9683. {
  9684. return p->getPropInt("@sequence", -1);
  9685. }
  9686. bool CLocalWUResult::isResultScalar() const
  9687. {
  9688. return p->getPropInt("@isScalar", 1) != 0;
  9689. }
  9690. bool findSize(int size, IntArray &sizes)
  9691. {
  9692. ForEachItemIn(idx, sizes)
  9693. {
  9694. if (sizes.item(idx)==size)
  9695. return true;
  9696. }
  9697. return false;
  9698. }
  9699. void CLocalWUResult::getSchema(IArrayOf<ITypeInfo> &types, StringAttrArray &names, IStringVal * eclText) const
  9700. {
  9701. MemoryBuffer schema;
  9702. p->getPropBin("SchemaRaw", schema);
  9703. if (schema.length())
  9704. {
  9705. for (;;)
  9706. {
  9707. StringAttr name;
  9708. schema.read(name);
  9709. if (*schema.readDirect(0)==type_void)
  9710. break;
  9711. names.append(*new StringAttrItem(name));
  9712. types.append(*deserializeType(schema)); // MORE - nested records!
  9713. }
  9714. schema.skip(1);
  9715. if (schema.length() != schema.getPos())
  9716. {
  9717. unsigned eclLen;
  9718. schema.read(eclLen);
  9719. const char * schemaData = (const char *)schema.readDirect(eclLen);
  9720. if (eclText)
  9721. {
  9722. eclText->setLen(schemaData, eclLen);
  9723. if ((eclLen == 0) && names.ordinality())
  9724. {
  9725. const char * firstName = names.item(0).text;
  9726. StringBuffer temp;
  9727. temp.append("RECORD ");
  9728. types.item(0).getECLType(temp);
  9729. temp.append(" value{NAMED('").append(firstName).append("')}").append("; END;");
  9730. eclText->set(temp.str());
  9731. }
  9732. }
  9733. }
  9734. }
  9735. }
  9736. void readRow(StringBuffer &out, MemoryBuffer &in, TypeInfoArray &types, StringAttrArray &names)
  9737. {
  9738. ForEachItemIn(idx, types)
  9739. {
  9740. StringAttrItem &name = names.item(idx);
  9741. ITypeInfo &type = types.item(idx);
  9742. unsigned size = type.getSize();
  9743. switch(type.getTypeCode())
  9744. {
  9745. case type_data:
  9746. if (size==UNKNOWN_LENGTH)
  9747. {
  9748. if (in.remaining() < sizeof(int))
  9749. throw MakeStringException(WUERR_CorruptResult, "corrupt workunit information");
  9750. in.read(size);
  9751. }
  9752. outputXmlData(size, in.readDirect(size), name.text, out);
  9753. break;
  9754. case type_string:
  9755. if (size==UNKNOWN_LENGTH)
  9756. {
  9757. if (in.remaining() < sizeof(int))
  9758. throw MakeStringException(WUERR_CorruptResult, "corrupt workunit information");
  9759. in.read(size);
  9760. }
  9761. outputXmlString(size, (const char *) in.readDirect(size), name.text, out);
  9762. break;
  9763. case type_varstring:
  9764. {
  9765. if (size == UNKNOWN_LENGTH)
  9766. size = (size32_t)strlen((const char *) in.readDirect(0))+1;
  9767. const char * text = (const char *) in.readDirect(size);
  9768. outputXmlString((size32_t)strlen(text), text, name.text, out);
  9769. break;
  9770. }
  9771. case type_unicode:
  9772. {
  9773. unsigned len = type.getStringLen();
  9774. if (size==UNKNOWN_LENGTH)
  9775. in.read(len);
  9776. outputXmlUnicode(len, (UChar const *) in.readDirect(len*2), name.text, out);
  9777. }
  9778. break;
  9779. case type_utf8:
  9780. {
  9781. unsigned len = type.getStringLen();
  9782. if (size==UNKNOWN_LENGTH)
  9783. {
  9784. in.read(len);
  9785. size = rtlUtf8Size(len, in.readDirect(0));
  9786. }
  9787. outputXmlUtf8(len, (const char *) in.readDirect(size), name.text, out);
  9788. }
  9789. break;
  9790. case type_qstring:
  9791. {
  9792. unsigned len = type.getStringLen();
  9793. if (size==UNKNOWN_LENGTH)
  9794. in.read(len);
  9795. unsigned outlen;
  9796. char *outstr;
  9797. rtlQStrToStrX(outlen, outstr, len, (const char *) in.readDirect(rtlQStrSize(len)));
  9798. outputXmlString(outlen, outstr, name.text, out);
  9799. free(outstr);
  9800. break;
  9801. }
  9802. case type_int:
  9803. case type_swapint:
  9804. if (type.isSigned())
  9805. {
  9806. const unsigned char *raw = (const unsigned char *) in.readDirect(size);
  9807. unsigned __int64 cval8 = 0;
  9808. //MORE: I think this is wrong - swapped doesn't mean little/big/
  9809. if (type.isSwappedEndian())
  9810. {
  9811. unsigned idx = 0;
  9812. if (raw[idx] & 0x80)
  9813. cval8 = (__int64)-1;
  9814. while (size--)
  9815. cval8 = (cval8 << 8) | raw[idx++];
  9816. }
  9817. else
  9818. {
  9819. if (raw[size-1] & 0x80)
  9820. cval8 = (__int64)-1;
  9821. while (size--)
  9822. cval8 = (cval8 << 8) | raw[size];
  9823. }
  9824. outputXmlInt((__int64) cval8, name.text, out);
  9825. }
  9826. else
  9827. {
  9828. const unsigned char *raw = (const unsigned char *) in.readDirect(size);
  9829. unsigned __int64 cval8 = 0;
  9830. if (type.isSwappedEndian())
  9831. {
  9832. unsigned idx = 0;
  9833. while (size--)
  9834. cval8 = (cval8 << 8) | raw[idx++];
  9835. }
  9836. else
  9837. {
  9838. while (size--)
  9839. cval8 = (cval8 << 8) | raw[size];
  9840. }
  9841. outputXmlUInt(cval8, name.text, out);
  9842. }
  9843. break;
  9844. case type_boolean:
  9845. bool cvalb;
  9846. in.read(cvalb);
  9847. outputXmlBool(cvalb, name.text, out);
  9848. break;
  9849. case type_decimal:
  9850. if (type.isSigned())
  9851. outputXmlDecimal(in.readDirect(size), size, type.getPrecision(), name.text, out);
  9852. else
  9853. outputXmlUDecimal(in.readDirect(size), size, type.getPrecision(), name.text, out);
  9854. break;
  9855. case type_real:
  9856. double cvald;
  9857. switch(size)
  9858. {
  9859. case 4:
  9860. float cvalf;
  9861. in.read(cvalf);
  9862. cvald = cvalf;
  9863. break;
  9864. case 8:
  9865. in.read(cvald);
  9866. break;
  9867. default:
  9868. throwUnexpected();
  9869. }
  9870. outputXmlReal(cvald, name.text, out);
  9871. break;
  9872. default:
  9873. assertex(!"unexpected type in raw record");
  9874. break;
  9875. }
  9876. }
  9877. }
  9878. IStringVal& CLocalWUResult::getResultXml(IStringVal &str, bool hidePassword) const
  9879. {
  9880. TypeInfoArray types;
  9881. StringAttrArray names;
  9882. getSchema(types, names);
  9883. StringBuffer xml;
  9884. const char * name = p->queryProp("@name");
  9885. if (name)
  9886. xml.appendf("<Dataset name=\'%s\'>\n", name);
  9887. else
  9888. xml.append("<Dataset>\n");
  9889. if (hidePassword && p->getPropBool("Format/@password"))
  9890. {
  9891. xml.append(" <Row>");
  9892. appendXMLTag(xml, name, "****");
  9893. xml.append("</Row>\n");
  9894. }
  9895. else if (p->hasProp("Value"))
  9896. {
  9897. MemoryBuffer raw;
  9898. p->getPropBin("Value", raw);
  9899. unsigned __int64 numrows = getResultRowCount();
  9900. while (numrows--)
  9901. {
  9902. xml.append(" <Row>");
  9903. readRow(xml, raw, types, names);
  9904. xml.append("</Row>\n");
  9905. }
  9906. }
  9907. else if (p->hasProp("xmlValue"))
  9908. {
  9909. xml.append(" <Row>");
  9910. appendXMLTag(xml, name, p->queryProp("xmlValue"));
  9911. xml.append("</Row>\n");
  9912. }
  9913. xml.append("</Dataset>\n");
  9914. str.set(xml.str());
  9915. return str;
  9916. }
  9917. IProperties *CLocalWUResult::queryResultXmlns()
  9918. {
  9919. CriticalBlock block(crit);
  9920. if (xmlns)
  9921. return xmlns;
  9922. xmlns.setown(createProperties());
  9923. Owned<IAttributeIterator> it = p->getAttributes();
  9924. unsigned prefixLen = strlen("@xmlns");
  9925. ForEach(*it)
  9926. {
  9927. const char *name = it->queryName();
  9928. if (!strncmp("@xmlns", name, prefixLen))
  9929. {
  9930. if (name[prefixLen]==':') //normal case
  9931. xmlns->setProp(name+prefixLen+1, it->queryValue());
  9932. else if (!name[prefixLen]) //special case, unprefixed namespace
  9933. xmlns->setProp("xmlns", it->queryValue());
  9934. }
  9935. }
  9936. return xmlns;
  9937. }
  9938. unsigned CLocalWUResult::getResultFetchSize() const
  9939. {
  9940. return p->getPropInt("fetchSize", 100);
  9941. }
  9942. __int64 CLocalWUResult::getResultTotalRowCount() const
  9943. {
  9944. return p->getPropInt64("totalRowCount", -1);
  9945. }
  9946. __int64 CLocalWUResult::getResultRowCount() const
  9947. {
  9948. return p->getPropInt64("rowCount", 0);
  9949. }
  9950. void CLocalWUResult::getResultDataset(IStringVal & ecl, IStringVal & defs) const
  9951. {
  9952. ecl.set(p->queryProp("datasetEcl"));
  9953. defs.set(p->queryProp("datasetEclDefs"));
  9954. }
  9955. IStringVal& CLocalWUResult::getResultLogicalName(IStringVal & val) const
  9956. {
  9957. val.set(p->queryProp("logicalName"));
  9958. return val;
  9959. }
  9960. IStringVal& CLocalWUResult::getResultKeyField(IStringVal & ecl) const
  9961. {
  9962. ecl.set(p->queryProp("keyField"));
  9963. return ecl;
  9964. }
  9965. unsigned CLocalWUResult::getResultRequestedRows() const
  9966. {
  9967. return p->getPropInt("requestedRows", 1);
  9968. }
  9969. IStringVal& CLocalWUResult::getResultEclSchema(IStringVal & str) const
  9970. {
  9971. TypeInfoArray types;
  9972. StringAttrArray names;
  9973. getSchema(types, names, &str);
  9974. return str;
  9975. }
  9976. IStringVal& CLocalWUResult::getResultRecordSizeEntry(IStringVal & str) const
  9977. {
  9978. str.set(p->queryProp("@recordSizeEntry"));
  9979. return str;
  9980. }
  9981. IStringVal& CLocalWUResult::getResultTransformerEntry(IStringVal & str) const
  9982. {
  9983. str.set(p->queryProp("@transformerEntry"));
  9984. return str;
  9985. }
  9986. __int64 CLocalWUResult::getResultRowLimit() const
  9987. {
  9988. return p->getPropInt64("@rowLimit");
  9989. }
  9990. IStringVal& CLocalWUResult::getResultFilename(IStringVal & str) const
  9991. {
  9992. str.set(p->queryProp("@tempFilename"));
  9993. return str;
  9994. }
  9995. IStringVal& CLocalWUResult::getResultFieldOpt(const char *name, IStringVal &str) const
  9996. {
  9997. str.clear();
  9998. if (!name || !*name)
  9999. return str;
  10000. IPropertyTree *format = p->queryPropTree("Format");
  10001. if (!format)
  10002. return str;
  10003. VStringBuffer xpath("@%s", name);
  10004. str.set(format->queryProp(xpath));
  10005. return str;
  10006. }
  10007. void CLocalWUResult::getResultWriteLocation(IStringVal & _graph, unsigned & _activityId) const
  10008. {
  10009. _graph.set(p->queryProp("@graph"));
  10010. _activityId = p->getPropInt("@activity", 0);
  10011. }
  10012. void CLocalWUResult::setResultStatus(WUResultStatus status)
  10013. {
  10014. setEnum(p, "@status", status, resultStatuses);
  10015. if (status==ResultStatusUndefined)
  10016. {
  10017. p->removeProp("Value");
  10018. p->removeProp("totalRowCount");
  10019. p->removeProp("rowCount");
  10020. p->removeProp("@format");
  10021. p->removeProp("@tempFileNmae");
  10022. p->removeProp("logicalName");
  10023. }
  10024. }
  10025. void CLocalWUResult::setResultName(const char *s)
  10026. {
  10027. p->setProp("@name", s);
  10028. }
  10029. void CLocalWUResult::setResultSequence(unsigned seq)
  10030. {
  10031. p->setPropInt("@sequence", seq);
  10032. }
  10033. void CLocalWUResult::setResultSchemaRaw(unsigned size, const void *schema)
  10034. {
  10035. p->setPropBin("SchemaRaw", size, schema);
  10036. }
  10037. void CLocalWUResult::setResultXmlns(const char *prefix, const char *uri)
  10038. {
  10039. StringBuffer xpath("@xmlns");
  10040. if (prefix && *prefix)
  10041. xpath.append(':').append(prefix);
  10042. p->setProp(xpath, uri);
  10043. }
  10044. void CLocalWUResult::setResultFieldOpt(const char *name, const char *value)
  10045. {
  10046. if (!name || !*name)
  10047. return;
  10048. IPropertyTree *format = ensurePTree(p, "Format");
  10049. VStringBuffer xpath("@%s", name);
  10050. format->setProp(xpath, value);
  10051. }
  10052. void CLocalWUResult::setResultWriteLocation(const char * _graph, unsigned _activityId)
  10053. {
  10054. p->setProp("@graph", _graph);
  10055. p->setPropInt("@activity", _activityId);
  10056. }
  10057. void CLocalWUResult::setResultScalar(bool isScalar)
  10058. {
  10059. p->setPropInt("@isScalar", (int) isScalar);
  10060. if (isScalar)
  10061. setResultTotalRowCount(1);
  10062. }
  10063. void CLocalWUResult::setResultRaw(unsigned len, const void *data, WUResultFormat format)
  10064. {
  10065. p->setPropBin("Value", len, data);
  10066. setResultStatus(ResultStatusSupplied);
  10067. setResultFormat(format);
  10068. }
  10069. void CLocalWUResult::setResultFormat(WUResultFormat format)
  10070. {
  10071. switch (format)
  10072. {
  10073. case ResultFormatXml:
  10074. p->setProp("@format","xml");
  10075. break;
  10076. case ResultFormatXmlSet:
  10077. p->setProp("@format","xmlSet");
  10078. break;
  10079. case ResultFormatCsv:
  10080. p->setProp("@format","csv");
  10081. break;
  10082. default:
  10083. p->removeProp("@format");
  10084. break;
  10085. }
  10086. }
  10087. void CLocalWUResult::setResultXML(const char *val)
  10088. {
  10089. p->setProp("xmlValue", val);
  10090. }
  10091. void CLocalWUResult::addResultRaw(unsigned len, const void *data, WUResultFormat format)
  10092. {
  10093. p->appendPropBin("Value", len, data);
  10094. setResultStatus(ResultStatusPartial);
  10095. const char *existingFormat = p->queryProp("@format");
  10096. const char *formatStr = NULL;
  10097. switch (format)
  10098. {
  10099. case ResultFormatXml:
  10100. formatStr = "xml";
  10101. break;
  10102. case ResultFormatXmlSet:
  10103. formatStr = "xmlSet";
  10104. break;
  10105. case ResultFormatCsv:
  10106. formatStr = "csv";
  10107. break;
  10108. default:
  10109. existingFormat = nullptr;
  10110. p->removeProp("@format");
  10111. break;
  10112. }
  10113. if (format)
  10114. {
  10115. if (existingFormat)
  10116. {
  10117. if (0 != stricmp(formatStr, existingFormat))
  10118. throw MakeStringException(WUERR_ResultFormatMismatch, "addResult format %s, does not match existing format %s", formatStr, existingFormat);
  10119. }
  10120. else
  10121. p->setProp("@format", formatStr);
  10122. }
  10123. }
  10124. void CLocalWUResult::setResultFetchSize(unsigned rows)
  10125. {
  10126. p->setPropInt("fetchSize", rows);
  10127. }
  10128. void CLocalWUResult::setResultTotalRowCount(__int64 rows)
  10129. {
  10130. p->setPropInt64("totalRowCount", rows);
  10131. }
  10132. void CLocalWUResult::setResultRowCount(__int64 rows)
  10133. {
  10134. p->setPropInt64("rowCount", rows);
  10135. }
  10136. void CLocalWUResult::setResultDataset(const char *ecl, const char *defs)
  10137. {
  10138. p->setProp("datasetEcl", ecl);
  10139. p->setProp("datasetEclDefs", defs);
  10140. }
  10141. void CLocalWUResult::setResultLogicalName(const char *logicalName)
  10142. {
  10143. p->setProp("logicalName", logicalName);
  10144. }
  10145. void CLocalWUResult::setResultKeyField(const char *ecl)
  10146. {
  10147. p->setProp("keyField", ecl);
  10148. }
  10149. void CLocalWUResult::setResultRequestedRows(unsigned rows)
  10150. {
  10151. p->setPropInt("requestedRows", rows);
  10152. }
  10153. void CLocalWUResult::setResultRecordSizeEntry(const char * entry)
  10154. {
  10155. p->setProp("@recordSizeEntry", entry);
  10156. }
  10157. void CLocalWUResult::setResultTransformerEntry(const char * entry)
  10158. {
  10159. p->setProp("@transformerEntry", entry);
  10160. }
  10161. void CLocalWUResult::setResultRowLimit(__int64 value)
  10162. {
  10163. p->setPropInt64("@rowLimit", value);
  10164. }
  10165. void CLocalWUResult::setResultFilename(const char * name)
  10166. {
  10167. p->setProp("@tempFilename", name);
  10168. }
  10169. // MORE - it's an undetected error if we call getResult... of a type that does not match schema
  10170. __int64 CLocalWUResult::getResultInt() const
  10171. {
  10172. __int64 result = 0;
  10173. MemoryBuffer s;
  10174. p->getPropBin("Value", s);
  10175. if (s.length())
  10176. s.read(result);
  10177. else
  10178. result = p->getPropInt64("xmlValue");
  10179. return result;
  10180. }
  10181. bool CLocalWUResult::getResultBool() const
  10182. {
  10183. bool result = false;
  10184. MemoryBuffer s;
  10185. p->getPropBin("Value", s);
  10186. if (s.length())
  10187. s.read(result);
  10188. else
  10189. result = p->getPropBool("xmlValue");
  10190. return result;
  10191. }
  10192. double CLocalWUResult::getResultReal() const
  10193. {
  10194. double result = 0;
  10195. MemoryBuffer s;
  10196. p->getPropBin("Value", s);
  10197. if (s.length())
  10198. s.read(result);
  10199. else
  10200. {
  10201. const char *xmlVal = p->queryProp("xmlValue");
  10202. if (xmlVal)
  10203. result = atof(xmlVal);
  10204. }
  10205. return result;
  10206. }
  10207. void CLocalWUResult::getResultDecimal(void * val, unsigned len, unsigned precision, bool isSigned) const
  10208. {
  10209. MemoryBuffer s;
  10210. p->getPropBin("Value", s);
  10211. if (s.length())
  10212. {
  10213. assertex(s.length() == len);
  10214. s.read(len, val);
  10215. }
  10216. else
  10217. {
  10218. const char *xmlVal = p->queryProp("xmlValue");
  10219. if (xmlVal)
  10220. {
  10221. Decimal d;
  10222. d.setString(strlen(xmlVal), xmlVal);
  10223. if (isSigned)
  10224. d.getDecimal(len, precision, val);
  10225. else
  10226. d.getUDecimal(len, precision, val);
  10227. }
  10228. else
  10229. memset(val, 0, len);
  10230. }
  10231. }
  10232. IStringVal& CLocalWUResult::getResultString(IStringVal & str, bool hidePassword) const
  10233. {
  10234. if (hidePassword && p->getPropBool("@password"))
  10235. {
  10236. str.set("****");
  10237. return str;
  10238. }
  10239. MemoryBuffer s;
  10240. p->getPropBin("Value", s);
  10241. if (s.length())
  10242. {
  10243. unsigned len;
  10244. s.read(len);
  10245. str.setLen((const char *) s.readDirect(len), len);
  10246. }
  10247. else
  10248. {
  10249. p->getPropBin("xmlValue", s);
  10250. if (p->isBinary("xmlValue"))
  10251. str.setLen(s.toByteArray(), s.length());
  10252. else
  10253. {
  10254. char *ascii = rtlUtf8ToVStr(rtlUtf8Length(s.length(), s.toByteArray()), s.toByteArray());
  10255. str.set(ascii);
  10256. rtlFree(ascii);
  10257. }
  10258. }
  10259. return str;
  10260. }
  10261. WUResultFormat CLocalWUResult::getResultFormat() const
  10262. {
  10263. const char * format = p->queryProp("@format");
  10264. if (!format)
  10265. return ResultFormatRaw;
  10266. else if (strcmp(format, "xml") == 0)
  10267. return ResultFormatXml;
  10268. else if (strcmp(format, "xmlSet") == 0)
  10269. return ResultFormatXmlSet;
  10270. else if (strcmp(format, "csv") == 0)
  10271. return ResultFormatCsv;
  10272. else
  10273. throw MakeStringException(WUERR_InvalidResultFormat, "Unrecognised result format %s", format);
  10274. }
  10275. IDataVal& CLocalWUResult::getResultRaw(IDataVal & data, IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const
  10276. {
  10277. MemoryBuffer s;
  10278. p->getPropBin("Value", s);
  10279. unsigned len = s.length();
  10280. if (len)
  10281. {
  10282. WUResultFormat format = getResultFormat();
  10283. if (format == ResultFormatXml || format == ResultFormatXmlSet)
  10284. {
  10285. if (!xmlTransformer)
  10286. throw MakeStringException(WUERR_MissingFormatTranslator, "No transformer supplied to translate XML format result");
  10287. xmlTransformer->transform(data, len, s.readDirect(len), format == ResultFormatXml);
  10288. }
  10289. else if (format == ResultFormatCsv)
  10290. {
  10291. if (!csvTransformer)
  10292. throw MakeStringException(WUERR_MissingFormatTranslator, "No transformer supplied to translate Csv format result");
  10293. csvTransformer->transform(data, len, s.readDirect(len), true);
  10294. }
  10295. else
  10296. data.setLen(s.readDirect(len), len);
  10297. }
  10298. else
  10299. data.clear();
  10300. return data;
  10301. }
  10302. unsigned CLocalWUResult::getResultHash() const
  10303. {
  10304. MemoryBuffer s;
  10305. p->getPropBin("Value", s);
  10306. unsigned len = s.length();
  10307. const byte * data = (const byte *)s.toByteArray();
  10308. return ~hashc(data, len, ~0);
  10309. }
  10310. IDataVal& CLocalWUResult::getResultUnicode(IDataVal & data) const
  10311. {
  10312. MemoryBuffer s;
  10313. p->getPropBin("Value", s);
  10314. if (s.length())
  10315. {
  10316. unsigned len;
  10317. s.read(len);
  10318. data.setLen(s.readDirect(len*2), len*2);
  10319. }
  10320. else
  10321. {
  10322. StringBuffer utf8;
  10323. if (p->getProp("xmlValue", utf8))
  10324. {
  10325. unsigned outlen;
  10326. UChar *out;
  10327. rtlUtf8ToUnicodeX(outlen, out, utf8.length(), utf8.str());
  10328. data.setLen(out, outlen*2);
  10329. rtlFree(out);
  10330. }
  10331. else
  10332. data.clear();
  10333. }
  10334. return data;
  10335. }
  10336. __int64 CLocalWUResult::getResultRawSize(IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const
  10337. {
  10338. WUResultFormat format = getResultFormat();
  10339. if (format == ResultFormatRaw)
  10340. {
  10341. //MORE: This should not load the whole property...
  10342. MemoryBuffer s;
  10343. p->getPropBin("Value", s);
  10344. return s.length();
  10345. }
  10346. else
  10347. {
  10348. MemoryBuffer temp;
  10349. MemoryBuffer2IDataVal adaptor(temp);
  10350. getResultRaw(adaptor, xmlTransformer, csvTransformer);
  10351. return temp.length();
  10352. }
  10353. }
  10354. IDataVal& CLocalWUResult::getResultRaw(IDataVal & data, __int64 from, __int64 length, IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const
  10355. {
  10356. WUResultFormat format = getResultFormat();
  10357. if (format != ResultFormatRaw)
  10358. {
  10359. MemoryBuffer temp;
  10360. MemoryBuffer2IDataVal adaptor(temp);
  10361. getResultRaw(adaptor, xmlTransformer, csvTransformer);
  10362. unsigned len = temp.length();
  10363. if (from > len) from = len;
  10364. if (from + length > len) length = len - from;
  10365. data.setLen(temp.readDirect(len) + from, (size32_t)length);
  10366. return data;
  10367. }
  10368. else
  10369. {
  10370. //MORE: This should not load the whole property, and should be different from the code above...
  10371. MemoryBuffer s;
  10372. p->getPropBin("Value", s);
  10373. unsigned len = s.length();
  10374. if (from > len) from = len;
  10375. if (from + length > len) length = len - from;
  10376. data.setLen(s.readDirect(len) + from, (size32_t)length);
  10377. return data;
  10378. }
  10379. }
  10380. bool CLocalWUResult::getResultIsAll() const
  10381. {
  10382. return p->getPropBool("@isAll", false);
  10383. }
  10384. // MORE - it's an undetected error if we call setResult... of a type that does not match schema
  10385. void CLocalWUResult::setResultInt(__int64 val)
  10386. {
  10387. // Note: we always serialize scalar integer results as int8, and schema must reflect this
  10388. MemoryBuffer m;
  10389. serializeInt8(val, m);
  10390. p->setPropBin("Value", m.length(), m.toByteArray());
  10391. setResultRowCount(1);
  10392. setResultTotalRowCount(1);
  10393. }
  10394. void CLocalWUResult::setResultUInt(unsigned __int64 val)
  10395. {
  10396. setResultInt((__int64) val);
  10397. }
  10398. void CLocalWUResult::setResultReal(double val)
  10399. {
  10400. // Note: we always serialize scalar real results as real8, and schema must reflect this
  10401. MemoryBuffer m;
  10402. serializeReal8(val, m);
  10403. p->setPropBin("Value", m.length(), m.toByteArray());
  10404. setResultRowCount(1);
  10405. setResultTotalRowCount(1);
  10406. }
  10407. void CLocalWUResult::setResultBool(bool val)
  10408. {
  10409. MemoryBuffer m;
  10410. serializeBool(val, m);
  10411. p->setPropBin("Value", m.length(), m.toByteArray());
  10412. setResultRowCount(1);
  10413. setResultTotalRowCount(1);
  10414. }
  10415. void CLocalWUResult::setResultString(const char *val, unsigned len)
  10416. {
  10417. // Note: we always serialize scalar strings with length prefix, and schema must reflect this
  10418. MemoryBuffer m;
  10419. serializeLPString(len, val, m);
  10420. p->setPropBin("Value", m.length(), m.toByteArray());
  10421. setResultRowCount(1);
  10422. setResultTotalRowCount(1);
  10423. }
  10424. void CLocalWUResult::setResultUnicode(const void *val, unsigned len)
  10425. {
  10426. // Note: we always serialize scalar strings with length prefix, and schema must reflect this
  10427. MemoryBuffer m;
  10428. m.append(len).append(len*2, val);
  10429. p->setPropBin("Value", m.length(), m.toByteArray());
  10430. setResultRowCount(1);
  10431. setResultTotalRowCount(1);
  10432. }
  10433. void CLocalWUResult::setResultData(const void *val, unsigned len)
  10434. {
  10435. // Note: we always serialize scalar data with length prefix, and schema must reflect this
  10436. MemoryBuffer m;
  10437. serializeLPString(len, (const char *)val, m);
  10438. p->setPropBin("Value", m.length(), m.toByteArray());
  10439. setResultRowCount(1);
  10440. setResultTotalRowCount(1);
  10441. }
  10442. void CLocalWUResult::setResultDecimal(const void *val, unsigned len)
  10443. {
  10444. // Note: serialized as data but with length known from schema
  10445. MemoryBuffer m;
  10446. serializeFixedData(len, val, m);
  10447. p->setPropBin("Value", m.length(), m.toByteArray());
  10448. setResultRowCount(1);
  10449. setResultTotalRowCount(1);
  10450. }
  10451. void CLocalWUResult::setResultRow(unsigned len, const void * data)
  10452. {
  10453. p->setPropBin("Value", len, data);
  10454. setResultRowCount(1);
  10455. setResultTotalRowCount(1);
  10456. setResultFormat(ResultFormatRaw);
  10457. }
  10458. void CLocalWUResult::setResultIsAll(bool value)
  10459. {
  10460. p->setPropBool("@isAll", value);
  10461. }
  10462. //==========================================================================================
  10463. CLocalWUPlugin::CLocalWUPlugin(IPropertyTree *props) : p(props)
  10464. {
  10465. }
  10466. IStringVal& CLocalWUPlugin::getPluginName(IStringVal &str) const
  10467. {
  10468. str.set(p->queryProp("@dllname"));
  10469. return str;
  10470. }
  10471. IStringVal& CLocalWUPlugin::getPluginVersion(IStringVal &str) const
  10472. {
  10473. str.set(p->queryProp("@version"));
  10474. return str;
  10475. }
  10476. void CLocalWUPlugin::setPluginName(const char *str)
  10477. {
  10478. p->setProp("@dllname", str);
  10479. }
  10480. void CLocalWUPlugin::setPluginVersion(const char *str)
  10481. {
  10482. p->setProp("@version", str);
  10483. }
  10484. //==========================================================================================
  10485. CLocalWULibrary::CLocalWULibrary(IPropertyTree *props) : p(props)
  10486. {
  10487. }
  10488. IStringVal& CLocalWULibrary::getName(IStringVal &str) const
  10489. {
  10490. str.set(p->queryProp("@name"));
  10491. return str;
  10492. }
  10493. void CLocalWULibrary::setName(const char *str)
  10494. {
  10495. p->setProp("@name", str);
  10496. }
  10497. //==========================================================================================
  10498. CLocalWUException::CLocalWUException(IPropertyTree *props) : p(props)
  10499. {
  10500. }
  10501. IStringVal& CLocalWUException::getExceptionSource(IStringVal &str) const
  10502. {
  10503. str.set(p->queryProp("@source"));
  10504. return str;
  10505. }
  10506. IStringVal& CLocalWUException::getExceptionMessage(IStringVal &str) const
  10507. {
  10508. str.set(p->queryProp(NULL));
  10509. return str;
  10510. }
  10511. unsigned CLocalWUException::getExceptionCode() const
  10512. {
  10513. return p->getPropInt("@code", 0);
  10514. }
  10515. ErrorSeverity CLocalWUException::getSeverity() const
  10516. {
  10517. return (ErrorSeverity)p->getPropInt("@severity", SeverityError);
  10518. }
  10519. IStringVal & CLocalWUException::getTimeStamp(IStringVal & dt) const
  10520. {
  10521. dt.set(p->queryProp("@time"));
  10522. return dt;
  10523. }
  10524. IStringVal & CLocalWUException::getExceptionFileName(IStringVal & str) const
  10525. {
  10526. str.set(p->queryProp("@filename"));
  10527. return str;
  10528. }
  10529. unsigned CLocalWUException::getExceptionLineNo() const
  10530. {
  10531. return p->getPropInt("@row", 0);
  10532. }
  10533. unsigned CLocalWUException::getExceptionColumn() const
  10534. {
  10535. return p->getPropInt("@col", 0);
  10536. }
  10537. unsigned CLocalWUException::getActivityId() const
  10538. {
  10539. const char * scope = queryScope();
  10540. if (scope)
  10541. {
  10542. const char * colon = strrchr(scope, ':');
  10543. if (colon && hasPrefix(colon+1, ActivityScopePrefix, true))
  10544. return atoi(colon+1+strlen(ActivityScopePrefix));
  10545. }
  10546. return p->getPropInt("@activity", 0);
  10547. }
  10548. unsigned CLocalWUException::getSequence() const
  10549. {
  10550. return p->getPropInt("@sequence", 0);
  10551. }
  10552. const char * CLocalWUException::queryScope() const
  10553. {
  10554. return p->queryProp("@scope");
  10555. }
  10556. unsigned CLocalWUException::getPriority() const
  10557. {
  10558. return p->getPropInt("@prio", 0);
  10559. }
  10560. void CLocalWUException::setExceptionSource(const char *str)
  10561. {
  10562. p->setProp("@source", str);
  10563. }
  10564. void CLocalWUException::setExceptionMessage(const char *str)
  10565. {
  10566. p->setProp(NULL, str);
  10567. }
  10568. void CLocalWUException::setExceptionCode(unsigned code)
  10569. {
  10570. p->setPropInt("@code", code);
  10571. }
  10572. void CLocalWUException::setSeverity(ErrorSeverity level)
  10573. {
  10574. p->setPropInt("@severity", level);
  10575. }
  10576. void CLocalWUException::setTimeStamp(const char *str)
  10577. {
  10578. p->setProp("@time", str);
  10579. }
  10580. void CLocalWUException::setExceptionFileName(const char *str)
  10581. {
  10582. p->setProp("@filename", str);
  10583. }
  10584. void CLocalWUException::setExceptionLineNo(unsigned r)
  10585. {
  10586. p->setPropInt("@row", r);
  10587. }
  10588. void CLocalWUException::setExceptionColumn(unsigned c)
  10589. {
  10590. p->setPropInt("@col", c);
  10591. }
  10592. void CLocalWUException::setActivityId(unsigned _id)
  10593. {
  10594. p->setPropInt("@activity", _id);
  10595. }
  10596. void CLocalWUException::setScope(const char * _scope)
  10597. {
  10598. p->setProp("@scope", _scope);
  10599. }
  10600. void CLocalWUException::setPriority(unsigned _priority)
  10601. {
  10602. p->setPropInt("@prio", _priority);
  10603. }
  10604. //==========================================================================================
  10605. CLocalWUAppValue::CLocalWUAppValue(IPropertyTree *props, unsigned child) : p(props)
  10606. {
  10607. StringAttrBuilder propPath(prop);
  10608. propPath.append("*[").append(child).append("]");
  10609. }
  10610. const char * CLocalWUAppValue::queryApplication() const
  10611. {
  10612. return p->queryName();
  10613. }
  10614. const char * CLocalWUAppValue::queryName() const
  10615. {
  10616. IPropertyTree* val=p->queryPropTree(prop.str());
  10617. if(val)
  10618. return val->queryName();
  10619. return ""; // Should not happen in normal usage
  10620. }
  10621. const char * CLocalWUAppValue::queryValue() const
  10622. {
  10623. return p->queryProp(prop.str());
  10624. }
  10625. //==========================================================================================
  10626. CLocalWUStatistic::CLocalWUStatistic(IPropertyTree *props) : p(props)
  10627. {
  10628. }
  10629. IStringVal & CLocalWUStatistic::getCreator(IStringVal & str) const
  10630. {
  10631. const char * creator = p->queryProp("@creator");
  10632. str.set(creator);
  10633. return str;
  10634. }
  10635. IStringVal & CLocalWUStatistic::getDescription(IStringVal & str, bool createDefault) const
  10636. {
  10637. const char * desc = p->queryProp("@desc");
  10638. if (desc)
  10639. {
  10640. str.set(desc); // legacy and in case it is overridden
  10641. }
  10642. else if (createDefault)
  10643. {
  10644. StatisticKind kind = getKind();
  10645. assertex(kind != StKindNone);
  10646. const char * scope = p->queryProp("@scope");
  10647. assertex(scope);
  10648. //Clean up the format of the scope when converting it to a description
  10649. StringBuffer descriptionText;
  10650. if (isGlobalScope(scope))
  10651. {
  10652. const char * creator = p->queryProp("@creator");
  10653. descriptionText.append(creator).append(":");
  10654. queryLongStatisticName(descriptionText, kind);
  10655. }
  10656. else
  10657. {
  10658. for (;;)
  10659. {
  10660. char c = *scope++;
  10661. if (!c)
  10662. break;
  10663. if (c == ':')
  10664. descriptionText.append(": ");
  10665. else
  10666. descriptionText.append(c);
  10667. }
  10668. if (kind != StTimeElapsed)
  10669. queryLongStatisticName(descriptionText.append(": "), kind);
  10670. }
  10671. str.set(descriptionText);
  10672. }
  10673. else
  10674. str.clear();
  10675. return str;
  10676. }
  10677. IStringVal & CLocalWUStatistic::getFormattedValue(IStringVal & str) const
  10678. {
  10679. StringBuffer formatted;
  10680. formatStatistic(formatted, getValue(), getMeasure());
  10681. str.set(formatted);
  10682. return str;
  10683. }
  10684. StatisticCreatorType CLocalWUStatistic::getCreatorType() const
  10685. {
  10686. return queryCreatorType(p->queryProp("@c"), SCTnone);
  10687. }
  10688. StatisticScopeType CLocalWUStatistic::getScopeType() const
  10689. {
  10690. return queryScopeType(p->queryProp("@s"), SSTnone);
  10691. }
  10692. StatisticKind CLocalWUStatistic::getKind() const
  10693. {
  10694. return queryStatisticKind(p->queryProp("@kind"), StKindNone);
  10695. }
  10696. const char * CLocalWUStatistic::queryScope() const
  10697. {
  10698. const char * scope = p->queryProp("@scope");
  10699. if (scope && streq(scope, LEGACY_GLOBAL_SCOPE))
  10700. scope = GLOBAL_SCOPE;
  10701. return scope;
  10702. }
  10703. StatisticMeasure CLocalWUStatistic::getMeasure() const
  10704. {
  10705. return queryMeasure(p->queryProp("@unit"), SMeasureNone);
  10706. }
  10707. unsigned __int64 CLocalWUStatistic::getValue() const
  10708. {
  10709. return p->getPropInt64("@value", 0);
  10710. }
  10711. unsigned __int64 CLocalWUStatistic::getCount() const
  10712. {
  10713. return p->getPropInt64("@count", 0);
  10714. }
  10715. unsigned __int64 CLocalWUStatistic::getMax() const
  10716. {
  10717. return p->getPropInt64("@max", 0);
  10718. }
  10719. unsigned __int64 CLocalWUStatistic::getTimestamp() const
  10720. {
  10721. return p->getPropInt64("@ts", 0);
  10722. }
  10723. //==========================================================================================
  10724. extern WORKUNIT_API ILocalWorkUnit * createLocalWorkUnit(const char *xml)
  10725. {
  10726. Owned<CLocalWorkUnit> cw = new CLocalWorkUnit((ISecManager *) NULL, NULL);
  10727. if (xml)
  10728. cw->loadPTree(createPTreeFromXMLString(xml, ipt_lowmem));
  10729. else
  10730. {
  10731. Owned<IPropertyTree> p = createPTree("W_LOCAL", ipt_lowmem);
  10732. p->setProp("@xmlns:xsi", "http://www.w3.org/1999/XMLSchema-instance");
  10733. cw->loadPTree(p.getClear());
  10734. }
  10735. ILocalWorkUnit* ret = QUERYINTERFACE(&cw->lockRemote(false), ILocalWorkUnit);
  10736. return ret;
  10737. }
  10738. void exportWorkUnitToXMLWithHiddenPasswords(IPropertyTree *p, IIOStream &out, unsigned extraXmlFlags)
  10739. {
  10740. const char *name = p->queryName();
  10741. if (!name)
  10742. name = "__unnamed__";
  10743. StringBuffer temp;
  10744. writeStringToStream(out, appendPTreeOpenTag(temp, p, name, 1, true));
  10745. Owned<IPropertyTreeIterator> elems = p->getElements("*", iptiter_sort);
  10746. ForEach(*elems)
  10747. {
  10748. IPropertyTree &elem = elems->query();
  10749. if (streq(elem.queryName(), "Parameters"))
  10750. {
  10751. writeStringToStream(out, appendPTreeOpenTag(temp.clear().append(' '), &elem, "Parameters", 2, false).append('\n'));
  10752. Owned<IPropertyTreeIterator> params = elem.getElements("*", iptiter_sort);
  10753. ForEach(*params)
  10754. {
  10755. IPropertyTree &param = params->query();
  10756. const char *paramname = param.queryName();
  10757. VStringBuffer xpath("Variables/Variable[@name='%s']/Format/@password", paramname);
  10758. if (p->getPropBool(xpath))
  10759. writeStringToStream(out, appendXMLTag(temp.clear().append(" "), paramname, "****").append('\n'));
  10760. else
  10761. {
  10762. toXML(&param, out, 2, XML_Format|XML_SortTags|extraXmlFlags);
  10763. }
  10764. }
  10765. writeStringToStream(out, appendXMLCloseTag(temp.clear().append(' '), "Parameters").append('\n'));
  10766. }
  10767. else if (streq(elem.queryName(), "Variables"))
  10768. {
  10769. writeStringToStream(out, appendPTreeOpenTag(temp.clear().append(' '), &elem, "Variables", 2, false).append('\n'));
  10770. Owned<IPropertyTreeIterator> vars = elem.getElements("*", iptiter_sort);
  10771. ForEach(*vars)
  10772. {
  10773. Owned<IPropertyTree> var = LINK(&vars->query());
  10774. if (var->getPropBool("Format/@password"))
  10775. {
  10776. var.setown(createPTreeFromIPT(var)); //copy and remove password values
  10777. var->removeProp("Value");
  10778. var->removeProp("xmlValue");
  10779. }
  10780. toXML(var, out, 2, XML_Format|XML_SortTags|extraXmlFlags);
  10781. }
  10782. writeStringToStream(out, appendXMLCloseTag(temp.clear().append(' '), "Variables").append('\n'));
  10783. }
  10784. else
  10785. toXML(&elem, out, 1, XML_Format|XML_SortTags|extraXmlFlags);
  10786. }
  10787. writeStringToStream(out, appendXMLCloseTag(temp.clear(), name));
  10788. }
  10789. StringBuffer &exportWorkUnitToXMLWithHiddenPasswords(IPropertyTree *p, StringBuffer &str)
  10790. {
  10791. class CAdapter : public CInterface, implements IIOStream
  10792. {
  10793. StringBuffer &out;
  10794. public:
  10795. IMPLEMENT_IINTERFACE;
  10796. CAdapter(StringBuffer &_out) : out(_out) { }
  10797. virtual void flush() { }
  10798. virtual size32_t read(size32_t len, void * data) { UNIMPLEMENTED; return 0; }
  10799. virtual size32_t write(size32_t len, const void * data) { out.append(len, (const char *)data); return len; }
  10800. } adapter(str);
  10801. exportWorkUnitToXMLWithHiddenPasswords(p->queryBranch(NULL), adapter, 0);
  10802. return str;
  10803. }
  10804. void exportWorkUnitToXMLFileWithHiddenPasswords(IPropertyTree *p, const char *filename, unsigned extraXmlFlags)
  10805. {
  10806. OwnedIFile ifile = createIFile(filename);
  10807. OwnedIFileIO ifileio = ifile->open(IFOcreate);
  10808. Owned<IIOStream> stream = createIOStream(ifileio);
  10809. exportWorkUnitToXMLWithHiddenPasswords(p->queryBranch(NULL), *stream, extraXmlFlags);
  10810. }
  10811. extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool unpack, bool includeProgress, bool hidePasswords)
  10812. {
  10813. // MORE - queryPTree isn't really safe without holding CLocalWorkUnit::crit - really need to move these functions into CLocalWorkunit
  10814. const IExtendedWUInterface *ewu = queryExtendedWU(wu);
  10815. if (ewu)
  10816. {
  10817. Linked<IPropertyTree> p;
  10818. if (unpack||includeProgress)
  10819. p.setown(ewu->getUnpackedTree(includeProgress));
  10820. else
  10821. p.set(ewu->queryPTree());
  10822. if (hidePasswords)
  10823. return exportWorkUnitToXMLWithHiddenPasswords(p, str);
  10824. else
  10825. return toXML(p, str, 0, XML_Format|XML_SortTags);
  10826. }
  10827. else
  10828. return str.append("Unrecognized workunit format");
  10829. }
  10830. extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool unpack, bool includeProgress, bool hidePasswords, bool regressionTest)
  10831. {
  10832. const IExtendedWUInterface *ewu = queryExtendedWU(wu);
  10833. if (ewu)
  10834. {
  10835. Linked<IPropertyTree> p;
  10836. if (unpack||includeProgress)
  10837. p.setown(ewu->getUnpackedTree(includeProgress));
  10838. else if (regressionTest)
  10839. p.setown(createPTreeFromIPT(ewu->queryPTree()));
  10840. else
  10841. p.set(ewu->queryPTree());
  10842. if (hidePasswords)
  10843. return exportWorkUnitToXMLFileWithHiddenPasswords(p, filename, extraXmlFlags);
  10844. if (regressionTest)
  10845. {
  10846. //This removes any items from the xml that will vary from run to run, so they can be binary compared from run to run
  10847. //The following attributes change with the build. Simpler to remove rather than needing to updated each build
  10848. p->removeProp("@buildVersion");
  10849. p->removeProp("@eclVersion");
  10850. p->removeProp("@hash");
  10851. //Remove statistics, and extract them to a separate file
  10852. IPropertyTree * stats = p->queryPropTree("Statistics");
  10853. if (stats)
  10854. {
  10855. StringBuffer statsFilename;
  10856. statsFilename.append(filename).append(".stats");
  10857. saveXML(statsFilename, stats, 0, (XML_Format|XML_SortTags|extraXmlFlags) & ~XML_LineBreakAttributes);
  10858. p->removeProp("Statistics");
  10859. }
  10860. //Now remove timestamps from exceptions
  10861. Owned<IPropertyTreeIterator> elems = p->getElements("Exceptions/Exception", iptiter_sort);
  10862. ForEach(*elems)
  10863. {
  10864. IPropertyTree &elem = elems->query();
  10865. elem.removeProp("@time");
  10866. }
  10867. }
  10868. saveXML(filename, p, 0, XML_Format|XML_SortTags|extraXmlFlags);
  10869. }
  10870. else
  10871. throw makeStringException(0, "Unrecognized workunit format");
  10872. }
  10873. extern WORKUNIT_API void submitWorkUnit(const char *wuid, const char *username, const char *password)
  10874. {
  10875. MemoryBuffer buffer;
  10876. Owned<INamedQueueConnection> conn = createNamedQueueConnection(0); // MORE - security token?
  10877. Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
  10878. Owned<IWorkUnit> workunit = factory->updateWorkUnit(wuid);
  10879. assertex(workunit);
  10880. StringAttr clusterName(workunit->queryClusterName());
  10881. if (!clusterName.length())
  10882. throw MakeStringException(WUERR_InvalidCluster, "No target cluster specified");
  10883. workunit->commit();
  10884. workunit.clear();
  10885. Owned<IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(clusterName.str());
  10886. if (!clusterInfo)
  10887. throw MakeStringException(WUERR_InvalidCluster, "Unknown cluster %s", clusterName.str());
  10888. SCMStringBuffer serverQueue;
  10889. clusterInfo->getServerQueue(serverQueue);
  10890. assertex(serverQueue.length());
  10891. Owned<IJobQueue> queue = createJobQueue(serverQueue.str());
  10892. if (!queue.get())
  10893. throw MakeStringException(WUERR_InvalidQueue, "Could not create workunit queue");
  10894. IJobQueueItem *item = createJobQueueItem(wuid);
  10895. queue->enqueue(item);
  10896. }
  10897. extern WORKUNIT_API void abortWorkUnit(const char *wuid)
  10898. {
  10899. StringBuffer xpath("/WorkUnitAborts/");
  10900. xpath.append(wuid);
  10901. Owned<IRemoteConnection> acon = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE, SDS_LOCK_TIMEOUT);
  10902. acon->queryRoot()->setPropInt(NULL, 1);
  10903. }
  10904. extern WORKUNIT_API void secSubmitWorkUnit(const char *wuid, ISecManager &secmgr, ISecUser &secuser)
  10905. {
  10906. if (checkWuSecAccess(wuid, &secmgr, &secuser, SecAccess_Write, "Submit", true, true))
  10907. submitWorkUnit(wuid, secuser.getName(), secuser.credentials().getPassword());
  10908. }
  10909. extern WORKUNIT_API void secAbortWorkUnit(const char *wuid, ISecManager &secmgr, ISecUser &secuser)
  10910. {
  10911. if (!checkWuSecAccess(wuid, &secmgr, &secuser, SecAccess_Write, "Submit", true, true))
  10912. return;
  10913. abortWorkUnit(wuid);
  10914. Owned<IConstWorkUnit> cw = factory->openWorkUnit(wuid);
  10915. if(!cw)
  10916. return;
  10917. WorkunitUpdate wu(&cw->lock());
  10918. const char *abortBy = secuser.getName();
  10919. if (abortBy && *abortBy)
  10920. wu->setTracingValue("AbortBy", abortBy);
  10921. wu->setTracingValueInt64("AbortTimeStamp", getTimeStampNowValue());
  10922. }
  10923. extern WORKUNIT_API void submitWorkUnit(const char *wuid, ISecManager *secmgr, ISecUser *secuser)
  10924. {
  10925. if (secmgr && secuser)
  10926. return secSubmitWorkUnit(wuid, *secmgr, *secuser);
  10927. if (secuser)
  10928. return submitWorkUnit(wuid, secuser->getName(), secuser->credentials().getPassword());
  10929. submitWorkUnit(wuid, "", "");
  10930. }
  10931. extern WORKUNIT_API void abortWorkUnit(const char *wuid, ISecManager *secmgr, ISecUser *secuser)
  10932. {
  10933. if (secmgr && secuser)
  10934. return secAbortWorkUnit(wuid, *secmgr, *secuser);
  10935. abortWorkUnit(wuid);
  10936. }
  10937. bool CLocalWorkUnit::hasWorkflow() const
  10938. {
  10939. return p->hasProp("Workflow");
  10940. }
  10941. unsigned CLocalWorkUnit::queryEventScheduledCount() const
  10942. {
  10943. CriticalBlock block(crit);
  10944. if (p->hasProp("@eventScheduledCount"))
  10945. return p->getPropInt("@eventScheduledCount", 0);
  10946. else
  10947. return p->getPropInt("Workflow/@eventScheduledCount", 0); // Legacy location for this setting
  10948. }
  10949. void CLocalWorkUnit::incEventScheduledCount()
  10950. {
  10951. CriticalBlock block(crit);
  10952. p->setPropInt("@eventScheduledCount", queryEventScheduledCount()+1);
  10953. }
  10954. IPropertyTree * CLocalWorkUnit::queryWorkflowTree() const
  10955. {
  10956. CriticalBlock block(crit);
  10957. return p->queryPropTree("Workflow");
  10958. }
  10959. IConstWorkflowItemIterator* CLocalWorkUnit::getWorkflowItems() const
  10960. {
  10961. // For this to be legally called, we must have the read-able interface. So we are already locked for (at least) read.
  10962. CriticalBlock block(crit);
  10963. if(!workflowIteratorCached)
  10964. {
  10965. assertex(!workflowIterator);
  10966. Owned<IPropertyTree> s = p->getPropTree("Workflow");
  10967. if(s)
  10968. workflowIterator.setown(createWorkflowItemIterator(s));
  10969. workflowIteratorCached = true;
  10970. }
  10971. return workflowIterator.getLink();
  10972. }
  10973. IWorkflowItemArray * CLocalWorkUnit::getWorkflowClone() const
  10974. {
  10975. unsigned count = 0;
  10976. Owned<IConstWorkflowItemIterator> iter = getWorkflowItems();
  10977. for(iter->first(); iter->isValid(); iter->next())
  10978. count++;
  10979. Owned<IWorkflowItemArray> array = createWorkflowItemArray(count);
  10980. for(iter->first(); iter->isValid(); iter->next())
  10981. array->addClone(iter->query());
  10982. return array.getLink();
  10983. }
  10984. IWorkflowItem * CLocalWorkUnit::addWorkflowItem(unsigned wfid, WFType type, WFMode mode, unsigned success, unsigned failure, unsigned recovery, unsigned retriesAllowed, unsigned contingencyFor)
  10985. {
  10986. // For this to be legally called, we must have the write-able interface. So we are already locked for write.
  10987. CriticalBlock block(crit);
  10988. workflowIterator.clear();
  10989. workflowIteratorCached = false;
  10990. IPropertyTree * s = p->queryPropTree("Workflow");
  10991. if(!s)
  10992. s = p->addPropTree("Workflow");
  10993. return createWorkflowItem(s, wfid, type, mode, success, failure, recovery, retriesAllowed, contingencyFor);
  10994. }
  10995. IWorkflowItemIterator * CLocalWorkUnit::updateWorkflowItems()
  10996. {
  10997. // For this to be legally called, we must have the write-able interface. So we are already locked for write.
  10998. CriticalBlock block(crit);
  10999. if(!workflowIterator)
  11000. {
  11001. IPropertyTree * s = p->queryPropTree("Workflow");
  11002. if(!s)
  11003. s = p->addPropTree("Workflow");
  11004. workflowIterator.setown(createWorkflowItemIterator(s));
  11005. workflowIteratorCached = true;
  11006. }
  11007. return workflowIterator.getLink();
  11008. }
  11009. void CLocalWorkUnit::syncRuntimeWorkflow(IWorkflowItemArray * array)
  11010. {
  11011. Owned<IWorkflowItemIterator> iter = updateWorkflowItems();
  11012. Owned<IWorkflowItem> item;
  11013. for(iter->first(); iter->isValid(); iter->next())
  11014. {
  11015. item.setown(iter->get());
  11016. item->syncRuntimeData(array->queryWfid(item->queryWfid()));
  11017. }
  11018. workflowIterator.clear();
  11019. workflowIteratorCached = false;
  11020. }
  11021. void CLocalWorkUnit::resetWorkflow()
  11022. {
  11023. if (hasWorkflow())
  11024. {
  11025. Owned<IWorkflowItemIterator> iter = updateWorkflowItems();
  11026. Owned<IWorkflowItem> wf;
  11027. for(iter->first(); iter->isValid(); iter->next())
  11028. {
  11029. wf.setown(iter->get());
  11030. wf->reset();
  11031. }
  11032. workflowIterator.clear();
  11033. workflowIteratorCached = false;
  11034. }
  11035. }
  11036. void CLocalWorkUnit::schedule()
  11037. {
  11038. CriticalBlock block(crit);
  11039. if(queryEventScheduledCount() == 0) return;
  11040. switch(getState())
  11041. {
  11042. case WUStateCompleted:
  11043. setState(WUStateWait);
  11044. break;
  11045. case WUStateFailed:
  11046. case WUStateArchived:
  11047. case WUStateAborting:
  11048. case WUStateAborted:
  11049. case WUStateScheduled:
  11050. throw MakeStringException(WUERR_CannotSchedule, "Cannot schedule workunit in this state");
  11051. }
  11052. StringBuffer rootPath;
  11053. rootPath.append("/Schedule/").append(queryClusterName());
  11054. Owned<IRemoteConnection> conn = querySDS().connect(rootPath.str(), myProcessSession(), RTM_LOCK_WRITE | RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
  11055. Owned<IPropertyTree> root = conn->getRoot();
  11056. if(!root->hasChildren())
  11057. {
  11058. StringBuffer addPath;
  11059. addPath.append("/Schedulers/").append(queryClusterName());
  11060. Owned<IRemoteConnection> addConn = querySDS().connect(addPath.str(), myProcessSession(), RTM_LOCK_WRITE | RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
  11061. }
  11062. char const * wuid = p->queryName();
  11063. StringBuffer xpath("*/*/");
  11064. ncnameEscape(wuid, xpath);
  11065. bool more;
  11066. do more = root->removeProp(xpath.str()); while(more);
  11067. Owned<IConstWorkflowItemIterator> iter = getWorkflowItems();
  11068. Owned<IWorkflowEvent> event;
  11069. Owned<IPropertyTree> branch1, branch2;
  11070. for(iter->first(); iter->isValid(); iter->next())
  11071. {
  11072. event.setown(iter->query()->getScheduleEvent());
  11073. if(!event) continue;
  11074. ncnameEscape(event->queryName(), xpath.clear());
  11075. ensurePTree(root, xpath.str());
  11076. branch1.setown(root->getPropTree(xpath.str()));
  11077. ncnameEscape(event->queryText(), xpath.clear());
  11078. ensurePTree(branch1, xpath.str());
  11079. branch2.setown(branch1->getPropTree(xpath.str()));
  11080. ncnameEscape(wuid, xpath.clear());
  11081. ensurePTree(branch2, xpath.str());
  11082. }
  11083. }
  11084. void CLocalWorkUnit::deschedule()
  11085. {
  11086. if(queryEventScheduledCount() == 0) return;
  11087. if(getState() == WUStateWait)
  11088. setState(WUStateCompleted);
  11089. doDescheduleWorkkunit(p->queryName());
  11090. }
  11091. EnumMapping localFileUploadTypes[] = {
  11092. { UploadTypeFileSpray, "FileSpray" },
  11093. { UploadTypeWUResult, "WUResult" },
  11094. { UploadTypeWUResultCsv, "WUResultCsv" },
  11095. { UploadTypeWUResultXml, "WUResultXml" },
  11096. { UploadTypeSize, NULL }
  11097. };
  11098. class CLocalFileUpload : public CInterface, implements IConstLocalFileUpload
  11099. {
  11100. public:
  11101. CLocalFileUpload(IPropertyTree * _tree) : tree(_tree) {}
  11102. CLocalFileUpload(unsigned id, LocalFileUploadType type, char const * source, char const * destination, char const * eventTag)
  11103. {
  11104. tree.setown(createPTree());
  11105. tree->setPropInt("@id", id);
  11106. setEnum(tree, "@type", type, localFileUploadTypes);
  11107. tree->setProp("@source", source);
  11108. tree->setProp("@destination", destination);
  11109. if (eventTag)
  11110. tree->setProp("@eventTag", eventTag);
  11111. }
  11112. IMPLEMENT_IINTERFACE;
  11113. IPropertyTree * getTree() { return tree.getLink(); }
  11114. virtual unsigned queryID() const { return tree->getPropInt("@id"); }
  11115. virtual LocalFileUploadType queryType() const { return (LocalFileUploadType)getEnum(tree, "@type", localFileUploadTypes); }
  11116. virtual IStringVal & getSource(IStringVal & ret) const { ret.set(tree->queryProp("@source")); return ret; }
  11117. virtual IStringVal & getDestination(IStringVal & ret) const { ret.set(tree->queryProp("@destination")); return ret; }
  11118. virtual IStringVal & getEventTag(IStringVal & ret) const { if(tree->hasProp("@eventTag")) ret.set(tree->queryProp("@eventTag")); else ret.clear(); return ret; }
  11119. private:
  11120. Owned<IPropertyTree> tree;
  11121. };
  11122. class CLocalFileUploadIterator : public CInterface, implements IConstLocalFileUploadIterator
  11123. {
  11124. public:
  11125. CLocalFileUploadIterator(IPropertyTree * _tree) : tree(_tree), iter(tree->getElements("LocalFileUpload")) {}
  11126. IMPLEMENT_IINTERFACE;
  11127. bool first() { return iter->first(); }
  11128. bool isValid() { return iter->isValid(); }
  11129. bool next() { return iter->next(); }
  11130. IConstLocalFileUpload * get() { return new CLocalFileUpload(&iter->get()); }
  11131. private:
  11132. Owned<IPropertyTree> tree;
  11133. Owned<IPropertyTreeIterator> iter;
  11134. };
  11135. IConstLocalFileUploadIterator * CLocalWorkUnit::getLocalFileUploads() const
  11136. {
  11137. // For this to be legally called, we must have the read-able interface. So we are already locked for (at least) read.
  11138. CriticalBlock block(crit);
  11139. Owned<IPropertyTree> s = p->getPropTree("LocalFileUploads");
  11140. if(s)
  11141. return new CLocalFileUploadIterator(s.getClear());
  11142. else
  11143. return NULL;
  11144. }
  11145. bool CLocalWorkUnit::requiresLocalFileUpload() const
  11146. {
  11147. SCMStringBuffer dest;
  11148. Owned<IConstWUResult> result;
  11149. Owned<IConstLocalFileUploadIterator> iter(getLocalFileUploads());
  11150. if(!iter)
  11151. return false;
  11152. for(iter->first(); iter->isValid(); iter->next())
  11153. {
  11154. Owned<IConstLocalFileUpload> upload(iter->get());
  11155. switch(upload->queryType())
  11156. {
  11157. case UploadTypeWUResult:
  11158. case UploadTypeWUResultCsv:
  11159. case UploadTypeWUResultXml:
  11160. upload->getDestination(dest);
  11161. result.setown(getResultByName(dest.str()));
  11162. if(!result)
  11163. return true;
  11164. break;
  11165. default:
  11166. throw MakeStringException(WUERR_InvalidUploadFormat, "Unsupported local file upload type %s", getEnumText(upload->queryType(), localFileUploadTypes));
  11167. }
  11168. }
  11169. return false;
  11170. }
  11171. unsigned CLocalWorkUnit::addLocalFileUpload(LocalFileUploadType type, char const * source, char const * destination, char const * eventTag)
  11172. {
  11173. // For this to be legally called, we must have the write-able interface. So we are already locked for write.
  11174. CriticalBlock block(crit);
  11175. IPropertyTree * s = p->queryPropTree("LocalFileUploads");
  11176. if(!s)
  11177. s = p->addPropTree("LocalFileUploads");
  11178. unsigned id = s->numChildren();
  11179. Owned<CLocalFileUpload> upload = new CLocalFileUpload(id, type, source, destination, eventTag);
  11180. s->addPropTree("LocalFileUpload", upload->getTree());
  11181. return id;
  11182. }
  11183. IStringVal & CLocalWorkUnit::getAbortBy(IStringVal & str) const
  11184. {
  11185. CriticalBlock block(crit);
  11186. str.set(p->queryProp("Tracing/AbortBy"));
  11187. return str;
  11188. }
  11189. unsigned __int64 CLocalWorkUnit::getAbortTimeStamp() const
  11190. {
  11191. CriticalBlock block(crit);
  11192. return p->getPropInt64("Tracing/AbortTimeStamp", 0);
  11193. }
  11194. #if 0
  11195. void testConstWorkflow(IConstWorkflowItem * cwf, bool * okay, bool * dep)
  11196. {
  11197. DBGLOG("Test workflow const iface %u", cwf->queryWfid());
  11198. unsigned deps = 0;
  11199. Owned<IWorkflowDependencyIterator> diter;
  11200. switch(cwf->queryWfid())
  11201. {
  11202. case 1:
  11203. assertex(!cwf->isScheduled());
  11204. assertex(cwf->queryType() == WFTypeNormal);
  11205. assertex(cwf->queryState() == WFStateNull);
  11206. diter.setown(cwf->getDependencies());
  11207. for(diter->first(); diter->isValid(); diter->next())
  11208. deps++;
  11209. assertex(deps==0);
  11210. okay[0] = true;
  11211. break;
  11212. case 2:
  11213. assertex(!cwf->isScheduled());
  11214. assertex(cwf->queryType() == WFTypeRecovery);
  11215. assertex(cwf->queryState() == WFStateSkip);
  11216. okay[1] = true;
  11217. break;
  11218. case 3:
  11219. assertex(cwf->queryContingencyFor() == 4);
  11220. okay[2] = true;
  11221. break;
  11222. case 4:
  11223. assertex(cwf->isScheduled());
  11224. assertex(cwf->queryType() == WFTypeNormal);
  11225. assertex(cwf->queryState() == WFStateReqd);
  11226. assertex(cwf->querySuccess() == 0);
  11227. assertex(cwf->queryFailure() == 3);
  11228. assertex(cwf->queryRecovery() == 2);
  11229. assertex(cwf->queryRetriesAllowed() == 10);
  11230. assertex(cwf->queryRetriesRemaining() == 10);
  11231. diter.setown(cwf->getDependencies());
  11232. for(diter->first(); diter->isValid(); diter->next())
  11233. {
  11234. dep[diter->query()-1] = true;
  11235. deps++;
  11236. }
  11237. assertex(deps==2);
  11238. assertex(dep[0]);
  11239. assertex(dep[1]);
  11240. okay[3] = true;
  11241. break;
  11242. case 5:
  11243. assertex(cwf->isScheduled());
  11244. assertex(!cwf->isScheduledNow());
  11245. assertex(cwf->querySchedulePriority() == 75);
  11246. assertex(cwf->queryScheduleCount() == 5);
  11247. assertex(cwf->queryScheduleCountRemaining() == 5);
  11248. okay[4] = true;
  11249. break;
  11250. case 6:
  11251. assertex(cwf->isScheduled());
  11252. assertex(!cwf->isScheduledNow());
  11253. assertex(cwf->querySchedulePriority() == 25);
  11254. assertex(!cwf->hasScheduleCount());
  11255. okay[5] = true;
  11256. break;
  11257. default:
  11258. assertex(!"unknown wfid in test");
  11259. }
  11260. }
  11261. void testRuntimeWorkflow(IRuntimeWorkflowItem * rwf, bool * okay)
  11262. {
  11263. DBGLOG("Test workflow runtime iface %u", rwf->queryWfid());
  11264. switch(rwf->queryWfid())
  11265. {
  11266. case 1:
  11267. case 2:
  11268. case 3:
  11269. okay[rwf->queryWfid()-1] = true;
  11270. break;
  11271. case 4:
  11272. {
  11273. unsigned tries = 0;
  11274. while(rwf->testAndDecRetries())
  11275. tries++;
  11276. assertex(tries == 10);
  11277. assertex(rwf->queryRetriesRemaining() == 0);
  11278. rwf->setState(WFStateFail);
  11279. assertex(rwf->queryState() == WFStateFail);
  11280. rwf->reset();
  11281. assertex(rwf->queryRetriesRemaining() == 10);
  11282. assertex(rwf->queryState() == WFStateReqd);
  11283. }
  11284. okay[3] = true;
  11285. break;
  11286. case 5:
  11287. {
  11288. assertex(rwf->queryScheduleCountRemaining() == 5);
  11289. unsigned count = 0;
  11290. do count++; while(rwf->decAndTestScheduleCountRemaining());
  11291. assertex(count == 5);
  11292. assertex(rwf->queryScheduleCountRemaining() == 0);
  11293. rwf->reset();
  11294. assertex(rwf->queryScheduleCountRemaining() == 5);
  11295. }
  11296. okay[4] = true;
  11297. break;
  11298. case 6:
  11299. {
  11300. assertex(!rwf->hasScheduleCount());
  11301. unsigned count;
  11302. for(count=0; count<20; count++)
  11303. assertex(rwf->decAndTestScheduleCountRemaining());
  11304. }
  11305. okay[5] = true;
  11306. break;
  11307. default:
  11308. assertex(!"unknown wfid in test");
  11309. }
  11310. }
  11311. void testWorkflow()
  11312. {
  11313. DBGLOG("workunit.cpp : testWorkflow");
  11314. CLocalWorkUnit wu("W-WF-TEST", 0, 0, 0);
  11315. Owned<IWorkflowItem> wf;
  11316. wf.setown(wu.addWorkflowItem(1, WFTypeNormal, 0, 0, 0, 0, 0));
  11317. wf.setown(wu.addWorkflowItem(2, WFTypeRecovery, 0, 0, 0, 0, 0));
  11318. wf.setown(wu.addWorkflowItem(3, WFTypeFailure, 0, 0, 0, 0, 4));
  11319. wf.setown(wu.addWorkflowItem(4, WFTypeNormal, 0, 3, 2, 10, 0));
  11320. wf->setScheduledNow();
  11321. wf->addDependency(1);
  11322. wf.setown(wu.addWorkflowItem(5, WFTypeNormal, 0, 0, 0, 0, 0));
  11323. wf->setScheduledOn("test", "foo*");
  11324. wf->setSchedulePriority(75);
  11325. wf->setScheduleCount(5);
  11326. wf.setown(wu.addWorkflowItem(6, WFTypeNormal, 0, 0, 0, 0, 0));
  11327. wf->setScheduledOn("test", "bar*");
  11328. wf->setSchedulePriority(25);
  11329. unsigned const n = 6;
  11330. bool okay[n];
  11331. bool dep[n];
  11332. unsigned i;
  11333. for(i=0; i<n; i++)
  11334. okay[i] = dep[i] = 0;
  11335. Owned<IConstWorkflowItemIterator> citer(wu.getWorkflowItems());
  11336. for(citer->first(); citer->isValid(); citer->next())
  11337. testConstWorkflow(citer->query(), okay, dep);
  11338. for(i=0; i<n; i++)
  11339. {
  11340. assertex(okay[i]);
  11341. okay[i] = false;
  11342. }
  11343. Owned<IWorkflowItemIterator> miter(wu.updateWorkflowItems());
  11344. for(miter->first(); miter->isValid(); miter->next())
  11345. {
  11346. Owned<IRuntimeWorkflowItem> rwf(miter->get());
  11347. testRuntimeWorkflow(rwf, okay);
  11348. }
  11349. for(i=0; i<n; i++)
  11350. {
  11351. assertex(okay[i]);
  11352. okay[i] = dep[i] = false;
  11353. }
  11354. Owned<IWorkflowItemArray> array(wu.getWorkflowClone());
  11355. unsigned wfid;
  11356. for(wfid = 1; array->isValid(wfid); wfid++)
  11357. testConstWorkflow(&array->queryWfid(wfid), okay, dep);
  11358. for(i=0; i<n; i++)
  11359. {
  11360. assertex(okay[i]);
  11361. okay[i] = false;
  11362. }
  11363. for(wfid = 1; array->isValid(wfid); wfid++)
  11364. testRuntimeWorkflow(&array->queryWfid(wfid), okay);
  11365. for(i=0; i<n; i++)
  11366. {
  11367. assertex(okay[i]);
  11368. okay[i] = false;
  11369. }
  11370. }
  11371. #endif
  11372. //------------------------------------------------------------------------------------------
  11373. extern WUState waitForWorkUnitToComplete(const char * wuid, int timeout, bool returnOnWaitState)
  11374. {
  11375. return factory->waitForWorkUnit(wuid, (unsigned) timeout, false, returnOnWaitState);
  11376. }
  11377. extern WORKUNIT_API WUState secWaitForWorkUnitToComplete(const char * wuid, ISecManager &secmgr, ISecUser &secuser, int timeout, bool returnOnWaitState)
  11378. {
  11379. if (checkWuSecAccess(wuid, &secmgr, &secuser, SecAccess_Read, "Wait for Complete", false, true))
  11380. return waitForWorkUnitToComplete(wuid, timeout, returnOnWaitState);
  11381. return WUStateUnknown;
  11382. }
  11383. extern bool waitForWorkUnitToCompile(const char * wuid, int timeout)
  11384. {
  11385. switch(factory->waitForWorkUnit(wuid, (unsigned) timeout, true, true))
  11386. {
  11387. case WUStateCompiled:
  11388. case WUStateCompleted:
  11389. case WUStateWait:
  11390. case WUStateUploadingFiles:
  11391. return true;
  11392. default:
  11393. return false;
  11394. }
  11395. }
  11396. extern WORKUNIT_API bool secWaitForWorkUnitToCompile(const char * wuid, ISecManager &secmgr, ISecUser &secuser, int timeout)
  11397. {
  11398. if (checkWuSecAccess(wuid, &secmgr, &secuser, SecAccess_Read, "Wait for Compile", false, true))
  11399. return waitForWorkUnitToCompile(wuid, timeout);
  11400. return false;
  11401. }
  11402. extern WORKUNIT_API bool secDebugWorkunit(const char * wuid, ISecManager &secmgr, ISecUser &secuser, const char *command, StringBuffer &response)
  11403. {
  11404. if (strnicmp(command, "<debug:", 7) == 0 && checkWuSecAccess(wuid, &secmgr, &secuser, SecAccess_Read, "Debug", false, true))
  11405. {
  11406. Owned<IConstWorkUnit> wu = factory->openWorkUnit(wuid, &secmgr, &secuser);
  11407. SCMStringBuffer ip;
  11408. unsigned port;
  11409. try
  11410. {
  11411. port = wu->getDebugAgentListenerPort();
  11412. wu->getDebugAgentListenerIP(ip);
  11413. SocketEndpoint debugEP(ip.str(), port);
  11414. Owned<ISocket> socket = ISocket::connect_timeout(debugEP, 1000);
  11415. unsigned len = (size32_t)strlen(command);
  11416. unsigned revlen = len;
  11417. _WINREV(revlen);
  11418. socket->write(&revlen, sizeof(revlen));
  11419. socket->write(command, len);
  11420. for (;;)
  11421. {
  11422. socket->read(&len, sizeof(len));
  11423. _WINREV(len);
  11424. if (len == 0)
  11425. break;
  11426. if (len & 0x80000000)
  11427. {
  11428. throwUnexpected();
  11429. }
  11430. char * mem = (char*) response.reserve(len);
  11431. socket->read(mem, len);
  11432. }
  11433. return true;
  11434. }
  11435. catch (IException *E)
  11436. {
  11437. VStringBuffer msg("In secDebugWorkunit wuid %s port %d ip %s command %s", wuid, port, ip.str(), command);
  11438. EXCLOG(E, msg.str());
  11439. throw;
  11440. }
  11441. }
  11442. return false;
  11443. }
  11444. void getSimpleResultType(IWUResult *r, Owned<ITypeInfo> &type)
  11445. {
  11446. TypeInfoArray types;
  11447. StringAttrArray names;
  11448. r->getSchema(types, names, NULL);
  11449. if (types.ordinality()==1)
  11450. type.set(&types.item(0));
  11451. }
  11452. bool isSuppliedParamScalar(IWUResult *r, IPropertyTree &curVal, Owned<ITypeInfo> &type)
  11453. {
  11454. if (!r->isResultScalar())
  11455. return false;
  11456. if (!curVal.hasChildren())
  11457. return true;
  11458. getSimpleResultType(r, type);
  11459. return type && type->isScalar();
  11460. }
  11461. void updateSuppliedXmlParams(IWorkUnit * w)
  11462. {
  11463. Owned<const IPropertyTree> params = w->getXmlParams();
  11464. if (!params)
  11465. return;
  11466. Owned<IPropertyTreeIterator> elems = params->getElements("*");
  11467. ForEach(*elems)
  11468. {
  11469. IPropertyTree & curVal = elems->query();
  11470. const char *name = curVal.queryName();
  11471. Owned<IWUResult> r = updateWorkUnitResult(w, name, -1);
  11472. if (r)
  11473. {
  11474. Owned<ITypeInfo> type;
  11475. StringBuffer s;
  11476. if (isSuppliedParamScalar(r, curVal, type))
  11477. {
  11478. curVal.getProp(".", s);
  11479. r->setResultXML(s);
  11480. r->setResultStatus(ResultStatusSupplied);
  11481. }
  11482. else
  11483. {
  11484. toXML(&curVal, s);
  11485. if (!type)
  11486. getSimpleResultType(r, type);
  11487. bool isSet = (type && type->getTypeCode()==type_set);
  11488. r->setResultRaw(s.length(), s.str(), isSet ? ResultFormatXmlSet : ResultFormatXml);
  11489. }
  11490. }
  11491. else
  11492. DBGLOG("WARNING: no matching variable in workunit for input parameter %s", name);
  11493. }
  11494. }
  11495. IWUResult * updateWorkUnitResult(IWorkUnit * w, const char *name, unsigned sequence)
  11496. {
  11497. switch ((int)sequence)
  11498. {
  11499. case ResultSequenceStored:
  11500. return w->updateVariableByName(name);
  11501. case ResultSequencePersist:
  11502. return w->updateGlobalByName(name);
  11503. case ResultSequenceInternal:
  11504. case ResultSequenceOnce:
  11505. return w->updateTemporaryByName(name);
  11506. default:
  11507. return w->updateResultBySequence(sequence);
  11508. }
  11509. }
  11510. IConstWUResult * getWorkUnitResult(IConstWorkUnit * w, const char *name, unsigned sequence)
  11511. {
  11512. switch ((int)sequence)
  11513. {
  11514. case ResultSequenceStored:
  11515. return w->getVariableByName(name);
  11516. case ResultSequencePersist:
  11517. return w->getGlobalByName(name);
  11518. case ResultSequenceInternal:
  11519. case ResultSequenceOnce:
  11520. return w->getTemporaryByName(name);
  11521. default:
  11522. if (name && name[0])
  11523. return w->getResultByName(name);//name takes precedence over sequence
  11524. else
  11525. return w->getResultBySequence(sequence);
  11526. }
  11527. }
  11528. extern WORKUNIT_API bool getWorkUnitCreateTime(const char *wuid,CDateTime &time)
  11529. {
  11530. if (wuid) {
  11531. char prefchar;
  11532. unsigned year,month,day,hour,min,sec;
  11533. if (sscanf(wuid, "%c%4u%2u%2u-%2u%2u%2u", &prefchar, &year, &month, &day, &hour, &min, &sec)==7) {
  11534. time.set(year, month, day, hour, min, sec, 0, true);
  11535. // time.setDate(year, month, day);
  11536. // time.setTime(hour, min, sec, 0, true); // for some reason time is local
  11537. return true;
  11538. }
  11539. }
  11540. return false;
  11541. }
  11542. extern WORKUNIT_API WUState getWorkUnitState(const char* state)
  11543. {
  11544. return (WUState) getEnum(state, states);
  11545. }
  11546. constexpr LogMsgCategory MCschedconn = MCprogress(1000); // Category used to inform about schedule synchronization
  11547. class CWorkflowScheduleConnection : implements IWorkflowScheduleConnection, public CInterface
  11548. {
  11549. public:
  11550. CWorkflowScheduleConnection(char const * wuid)
  11551. {
  11552. basexpath.append("/WorkflowSchedule/").append(wuid);
  11553. flagxpath.append(basexpath.str()).append("/Active");
  11554. }
  11555. IMPLEMENT_IINTERFACE;
  11556. virtual void lock()
  11557. {
  11558. LOG(MCschedconn, "Locking base schedule connection");
  11559. baseconn.setown(querySDS().connect(basexpath.str(), myProcessSession(), RTM_CREATE_QUERY | RTM_LOCK_WRITE, INFINITE));
  11560. if(!baseconn)
  11561. throw MakeStringException(WUERR_ScheduleLockFailed, "Could not get base workflow schedule lock");
  11562. }
  11563. virtual void unlock()
  11564. {
  11565. LOG(MCschedconn, "Unlocking base schedule connection");
  11566. baseconn.clear();
  11567. }
  11568. virtual void setActive()
  11569. {
  11570. LOG(MCschedconn, "Setting active flag in schedule connection");
  11571. flagconn.setown(querySDS().connect(flagxpath.str(), myProcessSession(), RTM_CREATE | RTM_LOCK_WRITE | RTM_DELETE_ON_DISCONNECT, INFINITE));
  11572. if(!flagconn)
  11573. throw MakeStringException(WUERR_ScheduleLockFailed, "Could not get active workflow schedule lock");
  11574. }
  11575. virtual void resetActive()
  11576. {
  11577. LOG(MCschedconn, "Resetting active flag in schedule connection");
  11578. flagconn.clear();
  11579. }
  11580. virtual bool queryActive()
  11581. {
  11582. return baseconn->queryRoot()->hasProp("Active");
  11583. }
  11584. virtual bool pull(IWorkflowItemArray * workflow)
  11585. {
  11586. assertex(baseconn);
  11587. Owned<IPropertyTree> root = baseconn->getRoot();
  11588. Owned<IPropertyTree> eventQueue = root->getPropTree("EventQueue");
  11589. if(!eventQueue) return false;
  11590. if(!eventQueue->hasProp("Item")) return false;
  11591. {
  11592. Owned<IPropertyTreeIterator> eventItems = eventQueue->getElements("Item");
  11593. Owned<IPropertyTree> eventItem;
  11594. Owned<IRuntimeWorkflowItemIterator> wfItems = workflow->getSequenceIterator();
  11595. Owned<IRuntimeWorkflowItem> wfItem;
  11596. for(eventItems->first(); eventItems->isValid(); eventItems->next())
  11597. {
  11598. eventItem.setown(&eventItems->get());
  11599. const char * eventName = eventItem->queryProp("@name");
  11600. const char * eventText = eventItem->queryProp("@text");
  11601. for(wfItems->first(); wfItems->isValid(); wfItems->next())
  11602. {
  11603. wfItem.setown(wfItems->get());
  11604. if(wfItem->queryState() != WFStateWait)
  11605. continue;
  11606. Owned<IWorkflowEvent> targetEvent = wfItem->getScheduleEvent();
  11607. if(!targetEvent || !targetEvent->matches(eventName, eventText))
  11608. continue;
  11609. wfItem->setEvent(eventName, eventText);
  11610. wfItem->setState(WFStateReqd);
  11611. resetDependentsState(workflow, *wfItem);
  11612. }
  11613. }
  11614. }
  11615. bool more;
  11616. do
  11617. more = eventQueue->removeProp("Item");
  11618. while(more);
  11619. return true;
  11620. }
  11621. virtual void push(char const * name, char const * text)
  11622. {
  11623. assertex(baseconn);
  11624. Owned<IPropertyTree> root = baseconn->getRoot();
  11625. IPropertyTree *eventQueue = ensurePTree(root, "EventQueue");
  11626. IPropertyTree *eventItem = eventQueue->addPropTree("Item");
  11627. eventItem->setProp("@name", name);
  11628. eventItem->setProp("@text", text);
  11629. }
  11630. virtual void remove()
  11631. {
  11632. if (baseconn)
  11633. {
  11634. baseconn->close(true);
  11635. baseconn.clear();
  11636. }
  11637. }
  11638. private:
  11639. void resetItemStateAndDependents(IWorkflowItemArray * workflow, unsigned wfid) const
  11640. {
  11641. if (wfid)
  11642. resetItemStateAndDependents(workflow, workflow->queryWfid(wfid));
  11643. }
  11644. void resetItemStateAndDependents(IWorkflowItemArray * workflow, IRuntimeWorkflowItem & item) const
  11645. {
  11646. switch(item.queryState())
  11647. {
  11648. case WFStateDone:
  11649. case WFStateFail:
  11650. {
  11651. item.setState(WFStateNull);
  11652. resetItemStateAndDependents(workflow, item.queryPersistWfid());
  11653. resetDependentsState(workflow, item);
  11654. break;
  11655. }
  11656. }
  11657. }
  11658. void resetDependentsState(IWorkflowItemArray * workflow, IRuntimeWorkflowItem & item) const
  11659. {
  11660. Owned<IWorkflowDependencyIterator> iter(item.getDependencies());
  11661. for(iter->first(); iter->isValid(); iter->next())
  11662. {
  11663. IRuntimeWorkflowItem & dep = workflow->queryWfid(iter->query());
  11664. resetItemStateAndDependents(workflow, dep);
  11665. }
  11666. }
  11667. private:
  11668. StringBuffer basexpath;
  11669. StringBuffer flagxpath;
  11670. Owned<IRemoteConnection> baseconn;
  11671. Owned<IRemoteConnection> flagconn;
  11672. };
  11673. extern WORKUNIT_API IWorkflowScheduleConnection * getWorkflowScheduleConnection(char const * wuid)
  11674. {
  11675. return new CWorkflowScheduleConnection(wuid);
  11676. }
  11677. extern WORKUNIT_API IExtendedWUInterface * queryExtendedWU(IConstWorkUnit * wu)
  11678. {
  11679. return QUERYINTERFACE(wu, IExtendedWUInterface);
  11680. }
  11681. extern WORKUNIT_API const IExtendedWUInterface * queryExtendedWU(const IConstWorkUnit * wu)
  11682. {
  11683. return QUERYINTERFACE(wu, const IExtendedWUInterface);
  11684. }
  11685. extern WORKUNIT_API void addExceptionToWorkunit(IWorkUnit * wu, ErrorSeverity severity, const char * source, unsigned code, const char * text, const char * filename, unsigned lineno, unsigned column, unsigned activity)
  11686. {
  11687. Owned<IWUException> we = wu->createException();
  11688. we->setSeverity(severity);
  11689. we->setExceptionMessage(text);
  11690. if (source)
  11691. we->setExceptionSource(source);
  11692. if (code)
  11693. we->setExceptionCode(code);
  11694. if (filename)
  11695. we->setExceptionFileName(filename);
  11696. if (lineno)
  11697. {
  11698. we->setExceptionLineNo(lineno);
  11699. if (column)
  11700. we->setExceptionColumn(lineno);
  11701. }
  11702. if (activity)
  11703. we->setActivityId(activity);
  11704. }
  11705. const char * skipLeadingXml(const char * text)
  11706. {
  11707. if (!text)
  11708. return NULL;
  11709. //skip utf8 BOM, probably excessive
  11710. if (memcmp(text, UTF8_BOM, 3) == 0)
  11711. text += 3;
  11712. for (;;)
  11713. {
  11714. if (isspace(*text))
  11715. text++;
  11716. else if (text[0] == '<' && text[1] == '?')
  11717. {
  11718. text += 2;
  11719. for (;;)
  11720. {
  11721. if (!*text) break;
  11722. if (text[0] == '?' && text[1] == '>')
  11723. {
  11724. text += 2;
  11725. break;
  11726. }
  11727. text++;
  11728. }
  11729. }
  11730. else if (text[0] == '<' && text[1] == '!' && text[2] == '-' && text[3] == '-')
  11731. {
  11732. text += 4;
  11733. for (;;)
  11734. {
  11735. if (!*text) break;
  11736. if (text[0] == '-' && text[1] == '-' && text[2] == '>')
  11737. {
  11738. text += 3;
  11739. break;
  11740. }
  11741. text++;
  11742. }
  11743. }
  11744. else
  11745. break;
  11746. }
  11747. return text;
  11748. }
  11749. extern WORKUNIT_API bool isArchiveQuery(const char * text)
  11750. {
  11751. text = skipLeadingXml(text);
  11752. if (!text)
  11753. return false;
  11754. const char * archivePrefix = "<Archive";
  11755. return memicmp(text, archivePrefix, strlen(archivePrefix)) == 0;
  11756. }
  11757. extern WORKUNIT_API bool isQueryManifest(const char * text)
  11758. {
  11759. text = skipLeadingXml(text);
  11760. if (!text)
  11761. return false;
  11762. const char * manifestPrefix = "<Manifest";
  11763. return memicmp(text, manifestPrefix, strlen(manifestPrefix)) == 0;
  11764. }
  11765. //------------------------------------------------------------------------------
  11766. // Named Alias helper function
  11767. static IPropertyTree * resolveQueryByDll(IPropertyTree * queryRegistry, const char * dll)
  11768. {
  11769. StringBuffer xpath;
  11770. xpath.append("Query[@dll=\"").append(dll).append("\"]");
  11771. return queryRegistry->getPropTree(xpath);
  11772. }
  11773. static IPropertyTree * resolveQueryByWuid(IPropertyTree * queryRegistry, const char * wuid)
  11774. {
  11775. StringBuffer xpath;
  11776. xpath.append("Query[@wuid=\"").append(wuid).append("\"]");
  11777. return queryRegistry->getPropTree(xpath);
  11778. }
  11779. static void clearAliases(IPropertyTree * queryRegistry, const char * id)
  11780. {
  11781. StringBuffer lcId(id);
  11782. lcId.toLowerCase();
  11783. StringBuffer xpath;
  11784. xpath.append("Alias[@id=\"").append(lcId).append("\"]");
  11785. Owned<IPropertyTreeIterator> iter = queryRegistry->getElements(xpath);
  11786. ForEach(*iter)
  11787. {
  11788. queryRegistry->removeProp(xpath.str());
  11789. }
  11790. }
  11791. IPropertyTree * addNamedQuery(IPropertyTree * queryRegistry, const char * name, const char * wuid, const char * dll, bool library, const char *userid, const char *snapshot)
  11792. {
  11793. StringBuffer lcName(name);
  11794. lcName.toLowerCase();
  11795. StringBuffer xpath;
  11796. xpath.append("Query[@name=\"").append(lcName.str()).append("\"]");
  11797. Owned<IPropertyTreeIterator> iter = queryRegistry->getElements(xpath);
  11798. unsigned seq = 1;
  11799. ForEach(*iter)
  11800. {
  11801. IPropertyTree &item = iter->query();
  11802. const char *thisWuid = item.queryProp("@wuid");
  11803. if (strieq(wuid, thisWuid))
  11804. return &item;
  11805. unsigned thisSeq = item.getPropInt("@seq");
  11806. if (thisSeq >= seq)
  11807. seq = thisSeq + 1;
  11808. }
  11809. StringBuffer id;
  11810. id.append(lcName).append(".").append(seq);
  11811. IPropertyTree * newEntry = createPTree("Query", ipt_caseInsensitive|ipt_lowmem);
  11812. newEntry->setProp("@name", lcName);
  11813. newEntry->setProp("@wuid", wuid);
  11814. newEntry->setProp("@dll", dll);
  11815. newEntry->setProp("@id", id);
  11816. newEntry->setPropInt("@seq", seq);
  11817. if (library)
  11818. newEntry->setPropBool("@isLibrary", true);
  11819. if (userid && *userid)
  11820. newEntry->setProp("@publishedBy", userid);
  11821. if (snapshot && *snapshot)
  11822. newEntry->setProp("@snapshot", snapshot);
  11823. return queryRegistry->addPropTree("Query", newEntry);
  11824. }
  11825. void removeNamedQuery(IPropertyTree * queryRegistry, const char * id)
  11826. {
  11827. StringBuffer lcId(id);
  11828. lcId.toLowerCase();
  11829. clearAliases(queryRegistry, lcId);
  11830. StringBuffer xpath;
  11831. xpath.append("Query[@id=\"").append(lcId).append("\"]");
  11832. queryRegistry->removeProp(xpath);
  11833. }
  11834. void removeDllFromNamedQueries(IPropertyTree * queryRegistry, const char * dll)
  11835. {
  11836. Owned<IPropertyTree> match = resolveQueryByDll(queryRegistry, dll);
  11837. if (!match)
  11838. return;
  11839. clearAliases(queryRegistry, match->queryProp("@id"));
  11840. queryRegistry->removeTree(match);
  11841. }
  11842. void removeWuidFromNamedQueries(IPropertyTree * queryRegistry, const char * wuid)
  11843. {
  11844. Owned<IPropertyTree> match = resolveQueryByWuid(queryRegistry, wuid);
  11845. if (!match)
  11846. return;
  11847. clearAliases(queryRegistry, match->queryProp("@id"));
  11848. queryRegistry->removeTree(match);
  11849. }
  11850. void removeAliasesFromNamedQuery(IPropertyTree * queryRegistry, const char * id)
  11851. {
  11852. clearAliases(queryRegistry, id);
  11853. }
  11854. void setQueryAlias(IPropertyTree * queryRegistry, const char * name, const char * value)
  11855. {
  11856. StringBuffer lcName(name);
  11857. lcName.toLowerCase();
  11858. StringBuffer xpath;
  11859. xpath.append("Alias[@name=\"").append(lcName).append("\"]");
  11860. IPropertyTree * match = queryRegistry->queryPropTree(xpath);
  11861. if (!match)
  11862. {
  11863. match = queryRegistry->addPropTree("Alias");
  11864. match->setProp("@name", lcName);
  11865. }
  11866. match->setProp("@id", value);
  11867. }
  11868. extern WORKUNIT_API IPropertyTree * getQueryById(IPropertyTree * queryRegistry, const char *queryid)
  11869. {
  11870. if (!queryRegistry || !queryid)
  11871. return NULL;
  11872. StringBuffer xpath;
  11873. xpath.append("Query[@id=\"").append(queryid).append("\"]");
  11874. return queryRegistry->getPropTree(xpath);
  11875. }
  11876. extern WORKUNIT_API IPropertyTree * getQueryById(const char *queryset, const char *queryid, bool readonly)
  11877. {
  11878. Owned<IPropertyTree> queryRegistry = getQueryRegistry(queryset, readonly);
  11879. return getQueryById(queryRegistry, queryid);
  11880. }
  11881. extern WORKUNIT_API IPropertyTree * resolveQueryAlias(IPropertyTree * queryRegistry, const char * alias)
  11882. {
  11883. if (!queryRegistry || !alias)
  11884. return NULL;
  11885. StringBuffer xpath;
  11886. unsigned cnt = 0;
  11887. StringBuffer lc(alias);
  11888. const char * search = lc.toLowerCase().str();
  11889. for (;;)
  11890. {
  11891. xpath.set("Alias[@name='").append(search).append("']/@id");
  11892. const char * queryId = queryRegistry->queryProp(xpath);
  11893. if (!queryId)
  11894. break;
  11895. //Check for too many alias indirections.
  11896. if (cnt++ > 10)
  11897. return NULL;
  11898. search = lc.set(queryId).toLowerCase().str();
  11899. }
  11900. return getQueryById(queryRegistry, search);
  11901. }
  11902. extern WORKUNIT_API IPropertyTree * resolveQueryAlias(const char *queryset, const char *alias, bool readonly)
  11903. {
  11904. Owned<IPropertyTree> queryRegistry = getQueryRegistry(queryset, readonly);
  11905. return resolveQueryAlias(queryRegistry, alias);
  11906. }
  11907. void setQuerySuspendedState(IPropertyTree * queryRegistry, const char *id, bool suspend, const char *userid)
  11908. {
  11909. StringBuffer lcId(id);
  11910. lcId.toLowerCase();
  11911. StringBuffer xpath;
  11912. xpath.append("Query[@id=\"").append(lcId).append("\"]");
  11913. IPropertyTree *tree = queryRegistry->queryPropTree(xpath);
  11914. if (tree)
  11915. {
  11916. if (tree->getPropBool("@suspended", false) == suspend)
  11917. return;
  11918. if (suspend)
  11919. {
  11920. tree->addPropBool("@suspended", true);
  11921. if (userid && *userid)
  11922. tree->addProp("@suspendedBy", userid);
  11923. }
  11924. else
  11925. {
  11926. tree->removeProp("@suspended");
  11927. tree->removeProp("@suspendedBy");
  11928. }
  11929. }
  11930. else
  11931. throw MakeStringException((suspend)? QUERRREG_SUSPEND : QUERRREG_UNSUSPEND, "Modifying query suspended state failed. Could not find query %s", id);
  11932. }
  11933. void setQueryCommentForNamedQuery(IPropertyTree * queryRegistry, const char *id, const char *queryComment)
  11934. {
  11935. if (queryComment)
  11936. {
  11937. StringBuffer lcId(id);
  11938. lcId.toLowerCase();
  11939. StringBuffer xpath;
  11940. xpath.append("Query[@id=\"").append(lcId).append("\"]");
  11941. IPropertyTree *tree = queryRegistry->queryPropTree(xpath);
  11942. if (tree)
  11943. tree->setProp("@queryComment", queryComment);
  11944. else
  11945. throw MakeStringException(QUERRREG_COMMENT, "Could not find query %s", id);
  11946. }
  11947. }
  11948. extern WORKUNIT_API IPropertyTree * getQueryRegistryRoot()
  11949. {
  11950. Owned<IRemoteConnection> conn = querySDS().connect("/QuerySets", myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT);
  11951. if (conn)
  11952. return conn->getRoot();
  11953. else
  11954. return NULL;
  11955. }
  11956. extern WORKUNIT_API void checkAddLibrariesToQueryEntry(IPropertyTree *queryTree, IConstWULibraryIterator *libraries)
  11957. {
  11958. if (!queryTree || !libraries)
  11959. return;
  11960. if (queryTree->hasProp("@libCount")) //already added
  11961. return;
  11962. unsigned libCount=0;
  11963. ForEach(*libraries)
  11964. {
  11965. IConstWULibrary &library = libraries->query();
  11966. SCMStringBuffer libname;
  11967. if (!library.getName(libname).length())
  11968. continue;
  11969. queryTree->addProp("Library", libname.str());
  11970. libCount++;
  11971. }
  11972. queryTree->setPropInt("@libCount", libCount);
  11973. }
  11974. extern WORKUNIT_API void checkAddLibrariesToQueryEntry(IPropertyTree *queryTree, IConstWorkUnit *cw)
  11975. {
  11976. Owned<IConstWULibraryIterator> libraries = &cw->getLibraries();
  11977. checkAddLibrariesToQueryEntry(queryTree, libraries);
  11978. }
  11979. extern WORKUNIT_API IPropertyTree * getQueryRegistry(const char * wsEclId, bool readonly)
  11980. {
  11981. //Only lock the branch for the target we're interested in.
  11982. StringBuffer xpath;
  11983. xpath.append("/QuerySets/QuerySet[@id=\"").append(wsEclId).append("\"]");
  11984. Owned<IRemoteConnection> conn = querySDS().connect(xpath.str(), myProcessSession(), readonly ? RTM_LOCK_READ : RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
  11985. if (conn)
  11986. return conn->getRoot();
  11987. if (readonly)
  11988. return NULL;
  11989. //Lock the QuerySets in case another thread/client wants to check/add the same QuerySet.
  11990. Owned<IRemoteConnection> globalLock = querySDS().connect("/QuerySets/", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
  11991. //Re-check if the QuerySet has been added between checking the 1st time and gaining the globalLock.
  11992. conn.setown(querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT));
  11993. if (conn)
  11994. return conn->getRoot();
  11995. conn.setown(querySDS().connect("/QuerySets/QuerySet", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_ADD, SDS_LOCK_TIMEOUT));
  11996. if (!conn)
  11997. throwUnexpected();
  11998. IPropertyTree * root = conn->queryRoot();
  11999. root->setProp("@id",wsEclId);
  12000. conn->commit();
  12001. return LINK(root);
  12002. }
  12003. IPropertyTree * addNamedPackageSet(IPropertyTree * packageRegistry, const char * name, IPropertyTree *packageInfo, bool overWrite)
  12004. {
  12005. StringBuffer xpath;
  12006. StringBuffer lcName(name);
  12007. lcName.toLowerCase();
  12008. // see if "name" already exists
  12009. xpath.append("Package[@id='").append(name).append("']");
  12010. IPropertyTree *pkgTree = packageRegistry->queryPropTree(xpath.str());
  12011. if (pkgTree)
  12012. {
  12013. if (overWrite)
  12014. packageRegistry->removeTree(pkgTree);
  12015. else
  12016. throw MakeStringException(WUERR_PackageAlreadyExists, "Package name %s already exists, either delete it or specify overwrite",name);
  12017. }
  12018. IPropertyTree *tree = packageRegistry->addPropTree("Package", packageInfo);
  12019. tree->setProp("@id", lcName);
  12020. return tree;
  12021. }
  12022. void removeNamedPackage(IPropertyTree * packageRegistry, const char * id)
  12023. {
  12024. StringBuffer lcId(id);
  12025. lcId.toLowerCase();
  12026. StringBuffer xpath;
  12027. xpath.append("Package[@id=\"").append(lcId).append("\"]");
  12028. packageRegistry->removeProp(xpath);
  12029. }
  12030. extern WORKUNIT_API IPropertyTree * getPackageSetRegistry(const char * wsEclId, bool readonly)
  12031. {
  12032. //Only lock the branch for the target we're interested in.
  12033. StringBuffer xpath;
  12034. xpath.append("/PackageSets/PackageSet[@id=\"").append(wsEclId).append("\"]");
  12035. Owned<IRemoteConnection> conn = querySDS().connect(xpath.str(), myProcessSession(), readonly ? RTM_LOCK_READ : RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
  12036. if (conn)
  12037. return conn->getRoot();
  12038. if (readonly)
  12039. return NULL;
  12040. //Lock the PackageSets in case another thread/client wants to check/add the same PackageSet.
  12041. Owned<IRemoteConnection> globalLock = querySDS().connect("/PackageSets/", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
  12042. //Re-check if the PackageSet has been added between checking the 1st time and gaining the globalLock.
  12043. conn.setown(querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT));
  12044. if (conn)
  12045. return conn->getRoot();
  12046. conn.setown(querySDS().connect("/PackageSets/PackageSet", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_ADD, SDS_LOCK_TIMEOUT));
  12047. if (!conn)
  12048. throwUnexpected();
  12049. IPropertyTree* root = conn->queryRoot();
  12050. root->setProp("@id",wsEclId);
  12051. conn->commit();
  12052. return LINK(root);
  12053. }
  12054. void addQueryToQuerySet(IWorkUnit *workunit, IPropertyTree *queryRegistry, const char *queryName, WUQueryActivationOptions activateOption, StringBuffer &newQueryId, const char *userid)
  12055. {
  12056. StringBuffer cleanQueryName;
  12057. appendUtf8XmlName(cleanQueryName, strlen(queryName), queryName);
  12058. SCMStringBuffer dllName;
  12059. Owned<IConstWUQuery> q = workunit->getQuery();
  12060. q->getQueryDllName(dllName);
  12061. if (!dllName.length())
  12062. throw MakeStringException(WUERR_InvalidDll, "Cannot deploy query - no associated dll.");
  12063. StringBuffer currentTargetClusterType;
  12064. queryRegistry->getProp("@targetclustertype", currentTargetClusterType);
  12065. SCMStringBuffer targetClusterType;
  12066. workunit->getDebugValue("targetclustertype", targetClusterType);
  12067. SCMStringBuffer snapshot;
  12068. workunit->getSnapshot(snapshot);
  12069. if (currentTargetClusterType.length() < 1)
  12070. {
  12071. queryRegistry->setProp("@targetclustertype", targetClusterType.str());
  12072. }
  12073. else
  12074. {
  12075. if (strcmp(currentTargetClusterType.str(), "roxie") == 0 && strcmp(currentTargetClusterType.str(), targetClusterType.str())!=0)
  12076. {
  12077. throw MakeStringException(WUERR_MismatchClusterType, "TargetClusterTypes of workunit and queryset do not match.");
  12078. }
  12079. }
  12080. IPropertyTree *newEntry = addNamedQuery(queryRegistry, cleanQueryName, workunit->queryWuid(), dllName.str(), isLibrary(workunit), userid, snapshot.str());
  12081. Owned<IConstWULibraryIterator> libraries = &workunit->getLibraries();
  12082. checkAddLibrariesToQueryEntry(newEntry, libraries);
  12083. newQueryId.append(newEntry->queryProp("@id"));
  12084. workunit->setIsQueryService(true); //will check querysets before delete
  12085. workunit->commit();
  12086. activateQuery(queryRegistry, activateOption, queryName, newQueryId, userid);
  12087. }
  12088. void activateQuery(IPropertyTree *queryRegistry, WUQueryActivationOptions activateOption, const char *queryName, const char *queryId, const char *userid)
  12089. {
  12090. StringBuffer cleanQueryName;
  12091. appendUtf8XmlName(cleanQueryName, strlen(queryName), queryName);
  12092. if (activateOption == ACTIVATE_SUSPEND_PREVIOUS|| activateOption == ACTIVATE_DELETE_PREVIOUS)
  12093. {
  12094. Owned<IPropertyTree> prevQuery = resolveQueryAlias(queryRegistry, cleanQueryName);
  12095. setQueryAlias(queryRegistry, cleanQueryName, queryId);
  12096. if (prevQuery && !streq(queryId, prevQuery->queryProp("@id")))
  12097. {
  12098. if (activateOption == ACTIVATE_SUSPEND_PREVIOUS)
  12099. setQuerySuspendedState(queryRegistry, prevQuery->queryProp("@id"), true, userid);
  12100. else
  12101. removeNamedQuery(queryRegistry, prevQuery->queryProp("@id"));
  12102. }
  12103. }
  12104. else if (activateOption == MAKE_ACTIVATE || activateOption == MAKE_ACTIVATE_LOAD_DATA_ONLY)
  12105. setQueryAlias(queryRegistry, cleanQueryName, queryId);
  12106. }
  12107. void addQueryToQuerySet(IWorkUnit *workunit, const char *querySetName, const char *queryName, WUQueryActivationOptions activateOption, StringBuffer &newQueryId, const char *userid)
  12108. {
  12109. Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, false);
  12110. addQueryToQuerySet(workunit, queryRegistry, queryName, activateOption, newQueryId, userid);
  12111. }
  12112. bool removeQuerySetAlias(const char *querySetName, const char *alias)
  12113. {
  12114. Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, true);
  12115. StringBuffer xpath;
  12116. xpath.appendf("Alias[@name='%s']", alias);
  12117. IPropertyTree *t = queryRegistry->queryPropTree(xpath);
  12118. return queryRegistry->removeTree(t);
  12119. }
  12120. void addQuerySetAlias(const char *querySetName, const char *alias, const char *id)
  12121. {
  12122. Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, false);
  12123. setQueryAlias(queryRegistry, alias, id);
  12124. }
  12125. void setSuspendQuerySetQuery(const char *querySetName, const char *id, bool suspend, const char *userid)
  12126. {
  12127. Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, true);
  12128. setQuerySuspendedState(queryRegistry, id, suspend, userid);
  12129. }
  12130. void deleteQuerySetQuery(const char *querySetName, const char *id)
  12131. {
  12132. Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, true);
  12133. removeNamedQuery(queryRegistry, id);
  12134. }
  12135. void removeQuerySetAliasesFromNamedQuery(const char *querySetName, const char * id)
  12136. {
  12137. Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, true);
  12138. clearAliases(queryRegistry, id);
  12139. }
  12140. void setQueryCommentForNamedQuery(const char *querySetName, const char *id, const char *queryComment)
  12141. {
  12142. Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, true);
  12143. setQueryCommentForNamedQuery(queryRegistry, id, queryComment);
  12144. }
  12145. const char *queryIdFromQuerySetWuid(IPropertyTree *queryRegistry, const char *wuid, const char *queryName, IStringVal &id)
  12146. {
  12147. if (!queryRegistry)
  12148. return NULL;
  12149. StringBuffer xpath;
  12150. xpath.appendf("Query[@wuid='%s']", wuid);
  12151. if (queryName && *queryName)
  12152. xpath.appendf("[@name='%s']", queryName);
  12153. IPropertyTree *q = queryRegistry->queryPropTree(xpath.str());
  12154. if (q)
  12155. {
  12156. id.set(q->queryProp("@id"));
  12157. }
  12158. return id.str();
  12159. }
  12160. const char *queryIdFromQuerySetWuid(const char *querySetName, const char *wuid, const char *queryName, IStringVal &id)
  12161. {
  12162. Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, true);
  12163. return queryIdFromQuerySetWuid(queryRegistry, wuid, queryName, id);
  12164. }
  12165. extern WORKUNIT_API void gatherLibraryNames(StringArray &names, StringArray &unresolved, IWorkUnitFactory &workunitFactory, IConstWorkUnit &cw, IPropertyTree *queryset)
  12166. {
  12167. Owned<IConstWULibraryIterator> wulibraries = &cw.getLibraries();
  12168. ForEach(*wulibraries)
  12169. {
  12170. SCMStringBuffer libname;
  12171. IConstWULibrary &wulibrary = wulibraries->query();
  12172. wulibrary.getName(libname);
  12173. if (names.contains(libname.str()) || unresolved.contains(libname.str()))
  12174. continue;
  12175. Owned<IPropertyTree> query = resolveQueryAlias(queryset, libname.str());
  12176. if (query && query->getPropBool("@isLibrary"))
  12177. {
  12178. const char *wuid = query->queryProp("@wuid");
  12179. Owned<IConstWorkUnit> libcw = workunitFactory.openWorkUnit(wuid);
  12180. if (libcw)
  12181. {
  12182. names.appendUniq(libname.str());
  12183. gatherLibraryNames(names, unresolved, workunitFactory, *libcw, queryset);
  12184. continue;
  12185. }
  12186. }
  12187. unresolved.appendUniq(libname.str());
  12188. }
  12189. }
  12190. bool looksLikeAWuid(const char * wuid, const char firstChar)
  12191. {
  12192. if (!wuid)
  12193. return false;
  12194. if (wuid[0] != firstChar)
  12195. return false;
  12196. if (!isdigit(wuid[1]) || !isdigit(wuid[2]) || !isdigit(wuid[3]) || !isdigit(wuid[4]))
  12197. return false;
  12198. if (!isdigit(wuid[5]) || !isdigit(wuid[6]) || !isdigit(wuid[7]) || !isdigit(wuid[8]))
  12199. return false;
  12200. return (wuid[9]=='-');
  12201. }
  12202. IPropertyTree * resolveDefinitionInArchive(IPropertyTree * archive, const char * path)
  12203. {
  12204. IPropertyTree * module = archive;
  12205. const char * dot = strrchr(path, '.');
  12206. StringBuffer xpath;
  12207. if (dot)
  12208. {
  12209. xpath.clear().append("Module[@key='").appendLower(dot-path, path).append("']");
  12210. module = archive->queryPropTree(xpath);
  12211. path = dot+1;
  12212. }
  12213. else
  12214. module = archive->queryPropTree("Module[@key='']");
  12215. if (!module)
  12216. return NULL;
  12217. xpath.clear().append("Attribute[@key='").appendLower(strlen(path), path).append("']");
  12218. return module->queryPropTree(xpath);
  12219. }
  12220. extern WORKUNIT_API void associateLocalFile(IWUQuery * query, WUFileType type, const char * name, const char * description, unsigned crc, unsigned minActivity, unsigned maxActivity)
  12221. {
  12222. StringBuffer hostname;
  12223. queryHostIP().getIpText(hostname);
  12224. StringBuffer fullPathname;
  12225. makeAbsolutePath(name, fullPathname);
  12226. query->addAssociatedFile(type, fullPathname, hostname, description, crc, minActivity, maxActivity);
  12227. }
  12228. extern WORKUNIT_API void descheduleWorkunit(char const * wuid)
  12229. {
  12230. Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
  12231. Owned<IWorkUnit> workunit = factory->updateWorkUnit(wuid);
  12232. if(workunit)
  12233. workunit->deschedule();
  12234. else
  12235. doDescheduleWorkkunit(wuid);
  12236. }
  12237. extern WORKUNIT_API void updateWorkunitStat(IWorkUnit * wu, StatisticScopeType scopeType, const char * scope, StatisticKind kind, const char * description, unsigned __int64 value, unsigned wfid)
  12238. {
  12239. StringBuffer scopestr;
  12240. if (wfid && scope && *scope)
  12241. scopestr.append(WorkflowScopePrefix).append(wfid).append(":").append(scope);
  12242. else
  12243. scopestr.set(scope);
  12244. wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), scopeType, scopestr, kind, description, value, 1, 0, StatsMergeReplace);
  12245. }
  12246. class WuTimingUpdater : implements ITimeReportInfo
  12247. {
  12248. public:
  12249. WuTimingUpdater(IWorkUnit * _wu, StatisticScopeType _scopeType, StatisticKind _kind)
  12250. : wu(_wu), scopeType(_scopeType), kind(_kind)
  12251. { }
  12252. virtual void report(const char * scope, const __int64 totaltime, const __int64 maxtime, const unsigned count)
  12253. {
  12254. wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), scopeType, scope, kind, nullptr, totaltime, count, maxtime, StatsMergeReplace);
  12255. }
  12256. protected:
  12257. IWorkUnit * wu;
  12258. StatisticScopeType scopeType;
  12259. StatisticKind kind;
  12260. };
  12261. extern WORKUNIT_API void updateWorkunitTimings(IWorkUnit * wu, ITimeReporter *timer)
  12262. {
  12263. WuTimingUpdater target(wu, SSTsection, StTimeTotalExecute);
  12264. timer->report(target);
  12265. }
  12266. extern WORKUNIT_API void updateWorkunitTimings(IWorkUnit * wu, StatisticScopeType scopeType, StatisticKind kind, ITimeReporter *timer)
  12267. {
  12268. WuTimingUpdater target(wu, scopeType, kind);
  12269. timer->report(target);
  12270. }
  12271. extern WORKUNIT_API void addTimeStamp(IWorkUnit * wu, StatisticScopeType scopeType, const char * scope, StatisticKind kind, unsigned wfid)
  12272. {
  12273. StringBuffer scopestr;
  12274. if (wfid && scope && *scope)
  12275. scopestr.append(WorkflowScopePrefix).append(wfid).append(":").append(scope);
  12276. else
  12277. scopestr.set(scope);
  12278. wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), scopeType, scopestr, kind, NULL, getTimeStampNowValue(), 1, 0, StatsMergeAppend);
  12279. }
  12280. void aggregateStatistic(StatsAggregation & result, IConstWorkUnit * wu, const WuScopeFilter & filter, StatisticKind search)
  12281. {
  12282. SimpleReferenceAggregator aggregator(search, result);
  12283. Owned<IConstWUScopeIterator> it = &wu->getScopeIterator(filter);
  12284. ForEach(*it)
  12285. it->playProperties(aggregator);
  12286. }
  12287. class GlobalStatisticGatherer : public CInterfaceOf<IStatisticGatherer>
  12288. {
  12289. public:
  12290. GlobalStatisticGatherer(IWorkUnit * _wu) : wu(_wu) {}
  12291. virtual void beginScope(const StatsScopeId & id)
  12292. {
  12293. prevLenStack.append(scope.length());
  12294. if (scope.length())
  12295. scope.append(":");
  12296. id.getScopeText(scope);
  12297. scopeTypeStack.append(id.queryScopeType());
  12298. }
  12299. virtual void beginSubGraphScope(unsigned id)
  12300. {
  12301. StatsScopeId scopeId(SSTsubgraph, id);
  12302. beginScope(scopeId);
  12303. }
  12304. virtual void beginActivityScope(unsigned id)
  12305. {
  12306. StatsScopeId scopeId(SSTactivity, id);
  12307. beginScope(scopeId);
  12308. }
  12309. virtual void beginEdgeScope(unsigned id, unsigned oid)
  12310. {
  12311. StatsScopeId scopeId(SSTedge, id, oid);
  12312. beginScope(scopeId);
  12313. }
  12314. virtual void beginChildGraphScope(unsigned id)
  12315. {
  12316. StatsScopeId scopeId(SSTchildgraph, id);
  12317. beginScope(scopeId);
  12318. }
  12319. virtual void endScope()
  12320. {
  12321. scope.setLength(prevLenStack.popGet());
  12322. scopeTypeStack.pop();
  12323. }
  12324. virtual void addStatistic(StatisticKind kind, unsigned __int64 value)
  12325. {
  12326. StatisticScopeType scopeType = scopeTypeStack.ordinality() ? (StatisticScopeType)scopeTypeStack.tos() : SSTglobal;
  12327. wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), scopeType, scope, kind, NULL, value, 1, 0, StatsMergeAppend);
  12328. }
  12329. virtual void updateStatistic(StatisticKind kind, unsigned __int64 value, StatsMergeAction mergeAction)
  12330. {
  12331. StatisticScopeType scopeType = scopeTypeStack.ordinality() ? (StatisticScopeType)scopeTypeStack.tos() : SSTglobal;
  12332. wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), scopeType, scope, kind, NULL, value, 1, 0, mergeAction);
  12333. }
  12334. virtual IStatisticCollection * getResult()
  12335. {
  12336. return NULL;
  12337. }
  12338. protected:
  12339. Linked<IWorkUnit> wu;
  12340. StringBuffer scope;
  12341. UnsignedArray prevLenStack;
  12342. UnsignedArray scopeTypeStack;
  12343. };
  12344. IStatisticGatherer * createGlobalStatisticGatherer(IWorkUnit * wu)
  12345. {
  12346. return new GlobalStatisticGatherer(wu);
  12347. }
  12348. extern WORKUNIT_API IPropertyTree * getWUGraphProgress(const char * wuid, bool readonly)
  12349. {
  12350. if (!wuid || !*wuid)
  12351. return NULL;
  12352. VStringBuffer path("/GraphProgress/%s", wuid);
  12353. Owned<IRemoteConnection> conn = querySDS().connect(path.str(),myProcessSession(),readonly ? RTM_LOCK_READ : RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
  12354. if (conn)
  12355. return conn->getRoot();
  12356. else
  12357. return NULL;
  12358. }
  12359. void addWorkunitException(IWorkUnit * wu, IError * error, bool removeTimeStamp)
  12360. {
  12361. ErrorSeverity wuSeverity = SeverityInformation;
  12362. ErrorSeverity severity = error->getSeverity();
  12363. switch (severity)
  12364. {
  12365. case SeverityIgnore:
  12366. return;
  12367. case SeverityInformation:
  12368. break;
  12369. case SeverityWarning:
  12370. wuSeverity = SeverityWarning;
  12371. break;
  12372. case SeverityError:
  12373. case SeverityFatal:
  12374. wuSeverity = SeverityError;
  12375. break;
  12376. }
  12377. Owned<IWUException> exception = wu->createException();
  12378. exception->setSeverity(wuSeverity);
  12379. StringBuffer msg;
  12380. exception->setExceptionCode(error->errorCode());
  12381. exception->setExceptionMessage(error->errorMessage(msg).str());
  12382. const char * source = queryCreatorTypeName(queryStatisticsComponentType());
  12383. exception->setExceptionSource(source);
  12384. exception->setExceptionFileName(error->getFilename());
  12385. exception->setExceptionLineNo(error->getLine());
  12386. exception->setExceptionColumn(error->getColumn());
  12387. if (removeTimeStamp)
  12388. exception->setTimeStamp(nullptr);
  12389. if (error->getActivity())
  12390. exception->setActivityId(error->getActivity());
  12391. if (error->queryScope())
  12392. exception->setScope(error->queryScope());
  12393. }
  12394. IError * WorkUnitErrorReceiver::mapError(IError * error)
  12395. {
  12396. return LINK(error);
  12397. }
  12398. void WorkUnitErrorReceiver::report(IError* eclError)
  12399. {
  12400. addWorkunitException(wu, eclError, removeTimeStamp);
  12401. }
  12402. size32_t WorkUnitErrorReceiver::errCount()
  12403. {
  12404. unsigned count = 0;
  12405. Owned<IConstWUExceptionIterator> exceptions = &wu->getExceptions();
  12406. ForEach(*exceptions)
  12407. if (exceptions->query().getSeverity() == SeverityError)
  12408. count++;
  12409. return count;
  12410. }
  12411. size32_t WorkUnitErrorReceiver::warnCount()
  12412. {
  12413. unsigned count = 0;
  12414. Owned<IConstWUExceptionIterator> exceptions = &wu->getExceptions();
  12415. ForEach(*exceptions)
  12416. if (exceptions->query().getSeverity() == SeverityWarning)
  12417. count++;
  12418. return count;
  12419. }
  12420. bool isValidPriorityValue(const char *priority)
  12421. {
  12422. if (isEmptyString(priority))
  12423. return false;
  12424. if (strieq("SLA", priority) || strieq("LOW", priority) || strieq("HIGH", priority) || strieq("NONE", priority))
  12425. return true;
  12426. return false;
  12427. }
  12428. bool isValidMemoryValue(const char *memoryUnit)
  12429. {
  12430. if (isEmptyString(memoryUnit) || !isdigit(*memoryUnit))
  12431. return false;
  12432. while (isdigit(*++memoryUnit));
  12433. if (!*memoryUnit)
  12434. return true;
  12435. switch (toupper(*memoryUnit++))
  12436. {
  12437. case 'E':
  12438. case 'P':
  12439. case 'T':
  12440. case 'G':
  12441. case 'M':
  12442. case 'K':
  12443. if (!*memoryUnit || strieq("B", memoryUnit))
  12444. return true;
  12445. break;
  12446. case 'B':
  12447. if (!*memoryUnit)
  12448. return true;
  12449. break;
  12450. }
  12451. return false;
  12452. }