dasds.cpp 331 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305730673077308730973107311731273137314731573167317731873197320732173227323732473257326732773287329733073317332733373347335733673377338733973407341734273437344734573467347734873497350735173527353735473557356735773587359736073617362736373647365736673677368736973707371737273737374737573767377737873797380738173827383738473857386738773887389739073917392739373947395739673977398739974007401740274037404740574067407740874097410741174127413741474157416741774187419742074217422742374247425742674277428742974307431743274337434743574367437743874397440744174427443744474457446744774487449745074517452745374547455745674577458745974607461746274637464746574667467746874697470747174727473747474757476747774787479748074817482748374847485748674877488748974907491749274937494749574967497749874997500750175027503750475057506750775087509751075117512751375147515751675177518751975207521752275237524752575267527752875297530753175327533753475357536753775387539754075417542754375447545754675477548754975507551755275537554755575567557755875597560756175627563756475657566756775687569757075717572757375747575757675777578757975807581758275837584758575867587758875897590759175927593759475957596759775987599760076017602760376047605760676077608760976107611761276137614761576167617761876197620762176227623762476257626762776287629763076317632763376347635763676377638763976407641764276437644764576467647764876497650765176527653765476557656765776587659766076617662766376647665766676677668766976707671767276737674767576767677767876797680768176827683768476857686768776887689769076917692769376947695769676977698769977007701770277037704770577067707770877097710771177127713771477157716771777187719772077217722772377247725772677277728772977307731773277337734773577367737773877397740774177427743774477457746774777487749775077517752775377547755775677577758775977607761776277637764776577667767776877697770777177727773777477757776777777787779778077817782778377847785778677877788778977907791779277937794779577967797779877997800780178027803780478057806780778087809781078117812781378147815781678177818781978207821782278237824782578267827782878297830783178327833783478357836783778387839784078417842784378447845784678477848784978507851785278537854785578567857785878597860786178627863786478657866786778687869787078717872787378747875787678777878787978807881788278837884788578867887788878897890789178927893789478957896789778987899790079017902790379047905790679077908790979107911791279137914791579167917791879197920792179227923792479257926792779287929793079317932793379347935793679377938793979407941794279437944794579467947794879497950795179527953795479557956795779587959796079617962796379647965796679677968796979707971797279737974797579767977797879797980798179827983798479857986798779887989799079917992799379947995799679977998799980008001800280038004800580068007800880098010801180128013801480158016801780188019802080218022802380248025802680278028802980308031803280338034803580368037803880398040804180428043804480458046804780488049805080518052805380548055805680578058805980608061806280638064806580668067806880698070807180728073807480758076807780788079808080818082808380848085808680878088808980908091809280938094809580968097809880998100810181028103810481058106810781088109811081118112811381148115811681178118811981208121812281238124812581268127812881298130813181328133813481358136813781388139814081418142814381448145814681478148814981508151815281538154815581568157815881598160816181628163816481658166816781688169817081718172817381748175817681778178817981808181818281838184818581868187818881898190819181928193819481958196819781988199820082018202820382048205820682078208820982108211821282138214821582168217821882198220822182228223822482258226822782288229823082318232823382348235823682378238823982408241824282438244824582468247824882498250825182528253825482558256825782588259826082618262826382648265826682678268826982708271827282738274827582768277827882798280828182828283828482858286828782888289829082918292829382948295829682978298829983008301830283038304830583068307830883098310831183128313831483158316831783188319832083218322832383248325832683278328832983308331833283338334833583368337833883398340834183428343834483458346834783488349835083518352835383548355835683578358835983608361836283638364836583668367836883698370837183728373837483758376837783788379838083818382838383848385838683878388838983908391839283938394839583968397839883998400840184028403840484058406840784088409841084118412841384148415841684178418841984208421842284238424842584268427842884298430843184328433843484358436843784388439844084418442844384448445844684478448844984508451845284538454845584568457845884598460846184628463846484658466846784688469847084718472847384748475847684778478847984808481848284838484848584868487848884898490849184928493849484958496849784988499850085018502850385048505850685078508850985108511851285138514851585168517851885198520852185228523852485258526852785288529853085318532853385348535853685378538853985408541854285438544854585468547854885498550855185528553855485558556855785588559856085618562856385648565856685678568856985708571857285738574857585768577857885798580858185828583858485858586858785888589859085918592859385948595859685978598859986008601860286038604860586068607860886098610861186128613861486158616861786188619862086218622862386248625862686278628862986308631863286338634863586368637863886398640864186428643864486458646864786488649865086518652865386548655865686578658865986608661866286638664866586668667866886698670867186728673867486758676867786788679868086818682868386848685868686878688868986908691869286938694869586968697869886998700870187028703870487058706870787088709871087118712871387148715871687178718871987208721872287238724872587268727872887298730873187328733873487358736873787388739874087418742874387448745874687478748874987508751875287538754875587568757875887598760876187628763876487658766876787688769877087718772877387748775877687778778877987808781878287838784878587868787878887898790879187928793879487958796879787988799880088018802880388048805880688078808880988108811881288138814881588168817881888198820882188228823882488258826882788288829883088318832883388348835883688378838883988408841884288438844884588468847884888498850885188528853885488558856885788588859886088618862886388648865886688678868886988708871887288738874887588768877887888798880888188828883888488858886888788888889889088918892889388948895889688978898889989008901890289038904890589068907890889098910891189128913891489158916891789188919892089218922892389248925892689278928892989308931893289338934893589368937893889398940894189428943894489458946894789488949895089518952895389548955895689578958895989608961896289638964896589668967896889698970897189728973897489758976897789788979898089818982898389848985898689878988898989908991899289938994899589968997899889999000900190029003900490059006900790089009901090119012901390149015901690179018901990209021902290239024902590269027902890299030903190329033903490359036903790389039904090419042904390449045904690479048904990509051905290539054905590569057905890599060906190629063906490659066906790689069907090719072907390749075907690779078907990809081908290839084908590869087908890899090909190929093909490959096909790989099
  1. /*##############################################################################
  2. HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. ############################################################################## */
  13. #define da_decl DECL_EXPORT
  14. #include "platform.h"
  15. #include "jhash.hpp"
  16. #include "jlib.hpp"
  17. #include "jfile.hpp"
  18. #include "jregexp.hpp"
  19. #include "jthread.hpp"
  20. #include "javahash.hpp"
  21. #include "javahash.tpp"
  22. #include "jmisc.hpp"
  23. #include "jlog.hpp"
  24. #include "mplog.hpp"
  25. #include "jptree.ipp"
  26. #include "jqueue.tpp"
  27. #include "dautils.hpp"
  28. #include "dadfs.hpp"
  29. #define DEBUG_DIR "debug"
  30. #define DEFAULT_KEEP_LASTN_STORES 1
  31. #define MAXDELAYS 5
  32. static const char *deltaHeader = "<CRC>0000000000</CRC><SIZE>0000000000000000</SIZE>"; // fill in later
  33. static unsigned deltaHeaderCrcOff = 5;
  34. static unsigned deltaHeaderSizeStart = 21;
  35. static unsigned deltaHeaderSizeOff = 27;
  36. static unsigned readWriteSlowTracing = 10000; // 10s default
  37. static bool readWriteStackTracing = false;
  38. static unsigned fakeCritTimeout = 60000;
  39. static unsigned readWriteTimeout = 60000;
  40. // #define NODELETE
  41. // #define DISABLE_COALESCE_QUIETTIME
  42. #define LEGACY_CLIENT_RESPONSE
  43. #define ENABLE_INSPOS
  44. #include "mpbuff.hpp"
  45. #include "mpcomm.hpp"
  46. #include "mputil.hpp"
  47. #include "dacoven.hpp"
  48. #include "daserver.hpp"
  49. #include "daclient.hpp"
  50. #include "dacsds.ipp"
  51. #include "dasds.ipp"
  52. #define ALWAYSLAZY_NOTUSED
  53. #define NoMoreChildrenMarker ((__int64)-1)
  54. #define DEFAULT_MAXCLOSEDOWNRETRIES 20 // really do not want to give up.
  55. #define DEFAULT_MAXRETRIES 3
  56. #define DEFAULT_RETRYDELAY (2) // (seconds)
  57. #define DELTANAME "daliinc"
  58. #define DELTADETACHED "dalidet"
  59. #define DELTAINPROGRESS "delta.progress"
  60. #define DETACHINPROGRESS "detach.progress"
  61. #define TMPSAVENAME "dali_store_tmp.xml"
  62. #define INIT_NODETABLE_SIZE 0x380000
  63. #define DEFAULT_LCIDLE_PERIOD (60*10) // time has to be quiet for before blocking/saving (when using @lightweightCoalesce)
  64. #define DEFAULT_LCMIN_TIME (24*60*60) // don't save more than once a 'DEFAULT_LCMIN_TIME' period.
  65. #define DEFAULT_LCIDLE_RATE 1 // 1 write transactions per idle period. <= this rate is deemed idle (suitable for save)
  66. #define STORENOTSAVE_WARNING_PERIOD 72 // hours
  67. static unsigned msgCount=(unsigned)-1;
  68. // #define TEST_NOTIFY_HANDLER
  69. #define TRACE_QWAITING
  70. #define CRC_VALIDATION
  71. #define SUBNTFY_POOL_SIZE 400
  72. #define SUBSCAN_POOL_SIZE 100
  73. #define RTM_INTERNAL 0x80000000 // marker for internal connection (performed within a transaction)
  74. #define DEFAULT_EXTERNAL_SIZE_THRESHOLD (10*1024)
  75. #define NOTIFY_ATTR "@sds:notify"
  76. #define FETCH_ENTIRE -1
  77. #define FETCH_ENTIRE_COND -2
  78. #define TIMEOUT_ON_CLOSEDOWN 120000 // On closedown, give up on trying to join a thread in CSDSTransactionServer after two minutes
  79. #define _POOLED_SERVER_REMOTE_TREE // use a pool for CServerRemoteTree allocations
  80. #ifdef _POOLED_SERVER_REMOTE_TREE
  81. static CFixedSizeAllocator *CServerRemoteTree_Allocator;
  82. #endif
  83. enum notifications { notify_delete=1 };
  84. static const char *notificationStr(notifications n)
  85. {
  86. switch (n)
  87. {
  88. case notify_delete: return "Notify Delete";
  89. default: return "UNKNOWN NOTIFY TYPE";
  90. }
  91. }
  92. const char *queryNotifyHandlerName(IPropertyTree *tree)
  93. {
  94. return tree->queryProp(NOTIFY_ATTR);
  95. }
  96. bool setNotifyHandlerName(const char *handlerName, IPropertyTree *tree)
  97. {
  98. #ifdef _DEBUG
  99. CClientRemoteTree *_tree = QUERYINTERFACE(tree, CClientRemoteTree);
  100. if (!_tree) return false; // has to a SDS tree!
  101. #endif
  102. tree->setProp(NOTIFY_ATTR, handlerName);
  103. return true;
  104. }
  105. StringBuffer &getSdsCmdText(SdsCommand cmd, StringBuffer &ret)
  106. {
  107. switch (cmd)
  108. {
  109. case DAMP_SDSCMD_CONNECT:
  110. return ret.append("DAMP_SDSCMD_CONNECT");
  111. case DAMP_SDSCMD_GET:
  112. return ret.append("DAMP_SDSCMD_GET");
  113. case DAMP_SDSCMD_GETCHILDREN:
  114. return ret.append("DAMP_SDSCMD_GETCHILDREN");
  115. case DAMP_SDSCMD_REVISIONS:
  116. return ret.append("DAMP_SDSCMD_REVISIONS");
  117. case DAMP_SDSCMD_DATA:
  118. return ret.append("DAMP_SDSCMD_DATA");
  119. case DAMP_SDSCMD_DISCONNECT:
  120. return ret.append("DAMP_SDSCMD_DISCONNECT");
  121. case DAMP_SDSCMD_CONNECTSERVER:
  122. return ret.append("DAMP_SDSCMD_CONNECTSERVER");
  123. case DAMP_SDSCMD_DATASERVER:
  124. return ret.append("DAMP_SDSCMD_DATASERVER");
  125. case DAMP_SDSCMD_DISCONNECTSERVER:
  126. return ret.append("DAMP_SDSCMD_DISCONNECTSERVER");
  127. case DAMP_SDSCMD_CHANGEMODE:
  128. return ret.append("DAMP_SDSCMD_CHANGEMODE");
  129. case DAMP_SDSCMD_CHANGEMODESERVER:
  130. return ret.append("DAMP_SDSCMD_CHANGEMODESERVER");
  131. case DAMP_SDSCMD_EDITION:
  132. return ret.append("DAMP_SDSCMD_EDITION");
  133. case DAMP_SDSCMD_GETSTORE:
  134. return ret.append("DAMP_SDSCMD_GETSTORE");
  135. case DAMP_SDSCMD_VERSION:
  136. return ret.append("DAMP_SDSCMD_VERSION");
  137. case DAMP_SDSCMD_DIAGNOSTIC:
  138. return ret.append("DAMP_SDSCMD_DIAGNOSTIC");
  139. case DAMP_SDSCMD_GETELEMENTS:
  140. return ret.append("DAMP_SDSCMD_GETELEMENTS");
  141. case DAMP_SDSCMD_MCONNECT:
  142. return ret.append("DAMP_SDSCMD_MCONNECT");
  143. case DAMP_SDSCMD_GETCHILDREN2:
  144. return ret.append("DAMP_SDSCMD_GETCHILDREN2");
  145. case DAMP_SDSCMD_GET2:
  146. return ret.append("DAMP_SDSCMD_GET2");
  147. case DAMP_SDSCMD_GETPROPS:
  148. return ret.append("DAMP_SDSCMD_GETPROPS");
  149. case DAMP_SDSCMD_GETXPATHS:
  150. return ret.append("DAMP_SDSCMD_GETXPATHS");
  151. case DAMP_SDSCMD_GETEXTVALUE:
  152. return ret.append("DAMP_SDSCMD_GETEXTVALUE");
  153. case DAMP_SDSCMD_GETXPATHSPLUSIDS:
  154. return ret.append("DAMP_SDSCMD_GETXPATHSPLUSIDS");
  155. case DAMP_SDSCMD_GETXPATHSCRITERIA:
  156. return ret.append("DAMP_SDSCMD_GETXPATHSCRITERIA");
  157. case DAMP_SDSCMD_GETELEMENTSRAW:
  158. return ret.append("DAMP_SDSCMD_GETELEMENTSRAW");
  159. case DAMP_SDSCMD_GETCOUNT:
  160. return ret.append("DAMP_SDSCMD_GETCOUNT");
  161. default:
  162. return ret.append("UNKNOWN");
  163. };
  164. return ret;
  165. }
  166. #ifdef USECHECKEDCRITICALSECTIONS
  167. class LinkingCriticalBlock : public CheckedCriticalBlock, public CInterface
  168. {
  169. public:
  170. LinkingCriticalBlock(CheckedCriticalSection &crit, const char *file, unsigned line) : CheckedCriticalBlock(crit, fakeCritTimeout, file, line) { }
  171. };
  172. class CLCLockBlock : public CInterface
  173. {
  174. ReadWriteLock &lock;
  175. bool readLocked; // false == writeLocked
  176. unsigned got, lnum;
  177. public:
  178. CLCLockBlock(ReadWriteLock &_lock, bool readLock, unsigned timeout, const char *fname, unsigned _lnum) : lock(_lock), lnum(_lnum)
  179. {
  180. got = msTick();
  181. for (;;)
  182. {
  183. if (readLock)
  184. {
  185. if (lock.lockRead(timeout))
  186. break;
  187. }
  188. else
  189. {
  190. if (lock.lockWrite(timeout))
  191. break;
  192. }
  193. PROGLOG("CLCLockBlock(write=%d) timeout %s(%d), took %d ms",!readLock,fname,lnum,got-msTick());
  194. PrintStackReport();
  195. }
  196. got = msTick();
  197. readLocked = readLock; // false == writeLocked
  198. };
  199. ~CLCLockBlock()
  200. {
  201. if (readLocked)
  202. lock.unlockRead();
  203. else
  204. lock.unlockWrite();
  205. unsigned e=msTick()-got;
  206. if (e>readWriteSlowTracing)
  207. {
  208. StringBuffer s("TIME: CLCLockBlock(write=");
  209. s.append(!readLocked).append(",lnum=").append(lnum).append(") took ").append(e).append(" ms");
  210. DBGLOG("%s", s.str());
  211. if (readWriteStackTracing)
  212. PrintStackReport();
  213. }
  214. }
  215. };
  216. #else
  217. class LinkingCriticalBlock : public CriticalBlock, public CInterface, implements IInterface
  218. {
  219. public:
  220. IMPLEMENT_IINTERFACE;
  221. LinkingCriticalBlock(CriticalSection &crit, const char *file, unsigned line) : CriticalBlock(crit) { }
  222. };
  223. class CLCLockBlock : public CInterface
  224. {
  225. ReadWriteLock &lock;
  226. bool readLocked; // false == writeLocked
  227. public:
  228. CLCLockBlock(ReadWriteLock &_lock, bool readLock, unsigned timeout, const char *fname, unsigned lnum) : lock(_lock)
  229. {
  230. if (readLock)
  231. lock.lockRead();
  232. else
  233. lock.lockWrite();
  234. readLocked = readLock; // false == writeLocked
  235. };
  236. ~CLCLockBlock()
  237. {
  238. if (readLocked)
  239. lock.unlockRead();
  240. else
  241. lock.unlockWrite();
  242. }
  243. };
  244. #endif
  245. class CLCReadLockBlock : public CLCLockBlock
  246. {
  247. public:
  248. CLCReadLockBlock(ReadWriteLock &lock, unsigned timeout, const char *fname, unsigned lnum) : CLCLockBlock(lock, true, timeout, fname, lnum) { }
  249. };
  250. class CLCWriteLockBlock : public CLCLockBlock
  251. {
  252. public:
  253. CLCWriteLockBlock(ReadWriteLock &lock, unsigned timeout, const char *fname, unsigned lnum) : CLCLockBlock(lock, false, timeout, fname, lnum) { }
  254. };
  255. #ifdef USECHECKEDCRITICALSECTIONS
  256. #define CHECKEDDALIREADLOCKBLOCK(l,timeout) Owned<CLCReadLockBlock> glue(block,__LINE__) = new CLCReadLockBlock(l,timeout,__FILE__,__LINE__)
  257. #define CHECKEDDALIWRITELOCKBLOCK(l,timeout) Owned<CLCWriteLockBlock> glue(block,__LINE__) = new CLCWriteLockBlock(l,timeout,__FILE__,__LINE__)
  258. #else
  259. #define CHECKEDDALIREADLOCKBLOCK(l,timeout) ReadLockBlock glue(block,__LINE__)(l)
  260. #define CHECKEDDALIWRITELOCKBLOCK(l,timeout) WriteLockBlock glue(block,__LINE__)(l)
  261. #endif
  262. #define OVERFLOWSIZE 50000
  263. class CFitArray
  264. {
  265. unsigned size;
  266. CRemoteTreeBase **ptrs; // offset 0 not used
  267. CriticalSection crit;
  268. unsigned nextId, chk;
  269. unsigned freeChainHead;
  270. inline void _ensure(unsigned _size)
  271. {
  272. if (size<_size) {
  273. ptrs = (CRemoteTreeBase **)checked_realloc(ptrs, _size * sizeof(CRemoteTreeBase *), size * sizeof(CRemoteTreeBase *), -100);
  274. memset(&ptrs[size], 0, (_size-size)*sizeof(CRemoteTreeBase *));
  275. size = _size;
  276. }
  277. }
  278. unsigned getId()
  279. {
  280. if (freeChainHead)
  281. {
  282. unsigned nF = freeChainHead;
  283. freeChainHead = (CRemoteTreeBase **)ptrs[nF]-ptrs;
  284. return nF;
  285. }
  286. if (++nextId >= size)
  287. _ensure(nextId+OVERFLOWSIZE);
  288. return nextId;
  289. }
  290. CRemoteTreeBase *_queryElem(__int64 id)
  291. {
  292. unsigned i = (unsigned) id;
  293. if (i>=size)
  294. return NULL;
  295. CRemoteTreeBase *ret = ptrs[i];
  296. if (!ret)
  297. return NULL;
  298. if ((memsize_t)ret >= (memsize_t)&ptrs[0] && (memsize_t)ret < (memsize_t)&ptrs[size])
  299. return NULL; // a ptr within the table is part of the free chain
  300. if (id != ret->queryServerId()) // then obj at index is not the same object.
  301. return NULL;
  302. return ret;
  303. }
  304. public:
  305. CFitArray() : ptrs(NULL), chk(0) { reset(); }
  306. CFitArray(unsigned initSize) : ptrs(NULL), chk(0) { reset(); ensure(initSize); }
  307. ~CFitArray() { free(ptrs); }
  308. void reset()
  309. {
  310. if (ptrs)
  311. free(ptrs);
  312. size = 0;
  313. ptrs = NULL;
  314. nextId = 1;
  315. freeChainHead = 0;
  316. }
  317. void ensure(unsigned size)
  318. {
  319. CriticalBlock b(crit);
  320. _ensure(size);
  321. }
  322. void addElem(CRemoteTreeBase *member)
  323. {
  324. CriticalBlock b(crit);
  325. __int64 id = getId();
  326. ptrs[id] = member;
  327. if (++chk==0x80000000)
  328. chk = 0;
  329. id |= ((__int64)chk << 32);
  330. member->setServerId(id);
  331. }
  332. void freeElem(__int64 id)
  333. {
  334. CriticalBlock b(crit);
  335. unsigned i = (unsigned) id;
  336. assertex(i<size);
  337. ptrs[i] = (CRemoteTreeBase *)&ptrs[freeChainHead];
  338. freeChainHead = i;
  339. }
  340. CRemoteTreeBase *queryElem(__int64 id)
  341. {
  342. CriticalBlock b(crit);
  343. return _queryElem(id);
  344. }
  345. CRemoteTreeBase *getElem(__int64 id)
  346. {
  347. CriticalBlock b(crit);
  348. return LINK(_queryElem(id));
  349. }
  350. unsigned maxElements() const { return nextId; } // actual is nextId - however many in free chain
  351. };
  352. ////////////////
  353. enum IncInfo { IINull=0x00, IncData=0x01, IncDetails=0x02, IncConnect=0x04, IncDisconnect=0x08, IncDisconnectDelete=0x24 };
  354. StringBuffer &constructStoreName(const char *storeBase, unsigned e, StringBuffer &res)
  355. {
  356. res.append(storeBase);
  357. if (e)
  358. res.append(e);
  359. res.append(".xml");
  360. return res;
  361. }
  362. ////////////////
  363. static CheckedCriticalSection loadStoreCrit, saveStoreCrit, saveIncCrit, nfyTableCrit, extCrit, blockedSaveCrit;
  364. class CCovenSDSManager;
  365. static CCovenSDSManager *SDSManager;
  366. static StringAttr remoteBackupLocation;
  367. /////////////////
  368. class TimingStats
  369. {
  370. public:
  371. TimingStats() : count(0), totalTime(0), maxTime(0), minTime((unsigned long)-1), totalSize(0) {}
  372. inline void record(unsigned long interval)
  373. {
  374. count++;
  375. totalTime += interval;
  376. if(interval>maxTime) maxTime = interval;
  377. if(interval<minTime) minTime = interval;
  378. }
  379. inline void recordSize(unsigned long size)
  380. {
  381. totalSize += size;
  382. }
  383. unsigned long queryCount() const { return count; }
  384. unsigned long queryMeanTime() const { return count ? (unsigned long)(((double)totalTime*0.1)/count + 0.5) : 0; }
  385. unsigned long queryMaxTime() const { return maxTime/10; }
  386. unsigned long queryMinTime() const { return count ? minTime/10 : 0; }
  387. unsigned long queryMeanSize() const { return count ? (unsigned long)(((double)totalSize)/count + 0.5) : 0; }
  388. private:
  389. unsigned long count;
  390. unsigned long totalTime;
  391. unsigned long maxTime;
  392. unsigned long minTime;
  393. unsigned long totalSize;
  394. };
  395. class TimingBlock
  396. {
  397. public:
  398. TimingBlock(TimingStats & _stats) : stats(_stats) { start = msTick(); }
  399. ~TimingBlock() { stats.record(msTick()-start); }
  400. protected:
  401. TimingStats & stats;
  402. private:
  403. unsigned long start;
  404. };
  405. class TimingSizeBlock : public TimingBlock
  406. {
  407. public:
  408. TimingSizeBlock(TimingStats & _stats) : TimingBlock(_stats), size(0) {}
  409. ~TimingSizeBlock() { stats.recordSize(size); }
  410. inline void recordSize(unsigned long _size) { size = _size; }
  411. private:
  412. unsigned long size;
  413. };
  414. class CSDSTransactionServer : public Thread, public CTransactionLogTracker
  415. {
  416. public:
  417. CSDSTransactionServer(CCovenSDSManager &_manager);
  418. void stop();
  419. void processMessage(CMessageBuffer &mb);
  420. const bool &queryStopped() const { return stopped; }
  421. inline TimingStats const & queryXactTimingStats() const { return xactTimingStats; }
  422. inline TimingStats const & queryConnectTimingStats() const { return connectTimingStats; }
  423. inline TimingStats const & queryCommitTimingStats() const { return commitTimingStats; }
  424. // Thread
  425. virtual int run();
  426. // CTransactionLogTracker
  427. virtual StringBuffer &getCmdText(unsigned cmd, StringBuffer &ret) const
  428. {
  429. return getSdsCmdText((SdsCommand)cmd, ret);
  430. }
  431. private:
  432. TimingStats xactTimingStats;
  433. TimingStats connectTimingStats;
  434. TimingStats commitTimingStats;
  435. bool stopped;
  436. CCovenSDSManager &manager;
  437. };
  438. //////////////
  439. class CSubscriberContainerBase : public CInterfaceOf<IInterface>
  440. {
  441. DECL_NAMEDCOUNT;
  442. protected:
  443. bool unsubscribed;
  444. Owned<ISubscription> subscriber;
  445. SubscriptionId id;
  446. public:
  447. CSubscriberContainerBase(ISubscription *_subscriber, SubscriptionId _id) :
  448. subscriber(_subscriber), id(_id)
  449. {
  450. INIT_NAMEDCOUNT;
  451. unsubscribed = false;
  452. }
  453. bool notify(MemoryBuffer &mb) const
  454. {
  455. try
  456. {
  457. subscriber->notify(mb);
  458. return true;
  459. }
  460. catch (IException *e)
  461. {
  462. LOG(MCuserWarning, e, "SDS: Error notifying subscriber");
  463. e->Release();
  464. }
  465. return false; // unsubscribe
  466. }
  467. const SubscriptionId &queryId() const { return id; }
  468. const void *queryFindParam() const
  469. {
  470. return (const void *) &id;
  471. }
  472. bool isUnsubscribed() { return unsubscribed || subscriber->aborted(); }
  473. void setUnsubscribed() { unsubscribed = true; }
  474. };
  475. /////////////////
  476. class CConnectionSubscriberContainer : public CSubscriberContainerBase
  477. {
  478. public:
  479. CConnectionSubscriberContainer(ISubscription *subscriber, SubscriptionId id) : CSubscriberContainerBase(subscriber, id) { }
  480. bool notify() { MemoryBuffer mb; return CSubscriberContainerBase::notify(mb); }
  481. };
  482. //////////////
  483. enum ConnInfoFlags { ci_newParent = 0x01 };
  484. //////////////
  485. class CServerConnection : public CConnectionBase, implements ISessionNotify
  486. {
  487. DECL_NAMEDCOUNT;
  488. public:
  489. IMPLEMENT_IINTERFACE;
  490. CServerConnection(ISDSConnectionManager &manager, ConnectionId connectionId, const char *xpath, SessionId id, unsigned mode, unsigned timeout, IPropertyTree *_parent, ConnInfoFlags _connInfoFlags)
  491. : CConnectionBase(manager, connectionId, xpath, id, mode, timeout), parent(_parent), connInfoFlags(_connInfoFlags)
  492. {
  493. INIT_NAMEDCOUNT;
  494. subsid = 0;
  495. established = false;
  496. }
  497. ~CServerConnection();
  498. void initPTreePath(PTree &root, PTree &tail)
  499. {
  500. if (&root != &tail)
  501. {
  502. StringBuffer head;
  503. const char *_tail = splitXPath(xpath, head);
  504. if (_tail == xpath)
  505. ptreePath.append(*LINK(&root));
  506. else
  507. ptreePath.fill(root, head.str(), tail);
  508. #define DEBUG_HPCC_11202
  509. #ifdef DEBUG_HPCC_11202
  510. PTree &parent = ptreePath.tos();
  511. aindex_t pos = parent.queryChildIndex(&tail);
  512. if (pos == NotFound)
  513. {
  514. StringBuffer msg;
  515. msg.append("ConnectionId=").appendf("%" I64F "x", connectionId).append(", xpath=").append(xpath).append(", sessionId=").appendf("%" I64F "x", sessionId).append(", mode=").append(mode).append(", timeout=");
  516. if (INFINITE == timeout)
  517. msg.append("INFINITE");
  518. else
  519. msg.append(timeout);
  520. ERRLOG("Invalid connection: %s", msg.str());
  521. ForEachItemIn(i, ptreePath)
  522. {
  523. PTree &tree = ptreePath.item(i);
  524. DBGLOG("PTree path item %d = %s", i, tree.queryName());
  525. }
  526. assertex(false); // stack report may be useful
  527. }
  528. #endif
  529. }
  530. ptreePath.append(*LINK(&tail));
  531. }
  532. CPTStack &queryPTreePath() { return ptreePath; }
  533. IPropertyTree *queryRootUnvalidated()
  534. {
  535. return root;
  536. }
  537. MemoryBuffer &getInfo(MemoryBuffer &out)
  538. {
  539. out.append(connectionId).append(xpath).append(sessionId).append(mode).append(timeout).append(established);
  540. return out;
  541. }
  542. void subscribe(SessionId id)
  543. {
  544. subsid = querySessionManager().subscribeSession(id, this);
  545. }
  546. void addConnInfoFlag(ConnInfoFlags flag) { connInfoFlags = (ConnInfoFlags) (((unsigned)connInfoFlags) | ((unsigned) flag)); }
  547. void closed(SessionId id)
  548. {
  549. LOG(MCwarning, unknownJob, "Connection (%" I64F "x) was leaked by exiting client (%" I64F "x) path=%s", connectionId, id, queryXPath());
  550. aborted(id);
  551. subsid=0;
  552. }
  553. void aborted(SessionId id);
  554. void unsubscribeSession()
  555. {
  556. if (subsid) {
  557. querySessionManager().unsubscribeSession(subsid);
  558. subsid = 0;
  559. }
  560. }
  561. void notify()
  562. {
  563. ForEachItemInRev(s, subscriptions)
  564. {
  565. CConnectionSubscriberContainer &sub = subscriptions.item(s);
  566. if (!sub.notify())
  567. subscriptions.remove(s);
  568. }
  569. }
  570. void addSubscriber(CConnectionSubscriberContainer &sub)
  571. {
  572. subscriptions.append(sub);
  573. }
  574. void removeSubscriber(SubscriptionId id)
  575. {
  576. ForEachItemIn(s, subscriptions) // do not expect a lot of subscribers per connection - probably ~ 1.
  577. {
  578. if (id == subscriptions.item(s).queryId())
  579. {
  580. subscriptions.remove(s);
  581. break;
  582. }
  583. }
  584. }
  585. bool queryEstablished() { return established; }
  586. void setEstablished() { established = true; }
  587. IPropertyTree *queryParent() { return parent; }
  588. void removeRoot()
  589. {
  590. if (parent)
  591. parent->removeTree(root);
  592. }
  593. virtual IPropertyTree *queryRoot();
  594. private:
  595. ConnInfoFlags connInfoFlags;
  596. IPropertyTree *parent;
  597. SubscriptionId subsid;
  598. CPTStack ptreePath;
  599. IArrayOf<CConnectionSubscriberContainer> subscriptions;
  600. bool established;
  601. };
  602. /////////////////
  603. class CQualifiers : public CInterfaceOf<IInterface>
  604. {
  605. StringArray qualifiers;
  606. public:
  607. inline void add(const char *qualifier) { qualifiers.append(qualifier); }
  608. inline unsigned count() const { return qualifiers.ordinality(); }
  609. inline const char *item(unsigned i) const { return qualifiers.item(i); }
  610. };
  611. class CSubscriberContainer : public CSubscriberContainerBase
  612. {
  613. StringAttr xpath, fullXpath;
  614. IPointerArrayOf<CQualifiers> qualifierStack;
  615. bool sub, sendValue;
  616. unsigned depth;
  617. public:
  618. CSubscriberContainer(ISubscription *subscriber, SubscriptionId id) : CSubscriberContainerBase(subscriber, id)
  619. {
  620. const MemoryAttr &ma = subscriber->queryData();
  621. MemoryBuffer mb(ma.length(), ma.get());
  622. mb.read(xpath);
  623. mb.read(sub);
  624. if (mb.remaining()) // remaining
  625. mb.read(sendValue);
  626. else
  627. sendValue = false;
  628. const char *path = xpath;
  629. const char *nextSep = path+1;
  630. StringBuffer head;
  631. depth = 1; // root
  632. for (;;)
  633. {
  634. nextSep = queryHead(nextSep, head.clear());
  635. ++depth; // inc last
  636. if (!nextSep)
  637. break;
  638. }
  639. StringBuffer strippedXpath;
  640. for (;;)
  641. {
  642. const char *startQ;
  643. if (NULL == (startQ = queryNextUnquoted(path, '['))) // escaped '[]' chars??
  644. {
  645. if (strippedXpath.length()) strippedXpath.append(path);
  646. break;
  647. }
  648. const char *nextSep = path+1;
  649. for (;;)
  650. {
  651. nextSep = queryHead(nextSep, head.clear());
  652. if (!nextSep || startQ < nextSep)
  653. break;
  654. qualifierStack.append(NULL); // no qualifier for this segment.
  655. }
  656. Owned<CQualifiers> qualifiers = new CQualifiers;
  657. strippedXpath.append(startQ-path, path);
  658. for (;;)
  659. {
  660. const char *endQ = queryNextUnquoted(startQ+1, ']');
  661. if (!endQ)
  662. throw MakeSDSException(SDSExcpt_SubscriptionParseError, "Missing closing brace: %s", xpath.get());
  663. StringAttr qualifier(startQ+1, endQ-startQ-1);
  664. qualifiers->add(qualifier);
  665. path = endQ+1;
  666. if ('[' != *path)
  667. break;
  668. startQ = path;
  669. }
  670. qualifierStack.append(qualifiers.getClear());
  671. }
  672. fullXpath.set(xpath);
  673. if (strippedXpath.length()) // some qualifications
  674. xpath.set(strippedXpath.str());
  675. // Notes on qualified (e.g. 'a/b[x="test"]/c' etc.)
  676. // 1) strip out all qualifies from xpath into 'qualifierStack' and translate xpath to simple absolute path.
  677. // 2) collate IPropertyTree stack in processData; to pass to querySubscribers.
  678. // 3) querySubscribers becomes getSubscribers, builds up a list of matching subscribers,
  679. // initially obtained with findList, but pruned based on IPT stack and qualifierStack.
  680. // by walking IPT stack performing either PTree->checkPattern or for index's parent->findChild etc.
  681. }
  682. const char *queryXPath() const { return xpath; }
  683. bool querySub() const { return sub; }
  684. bool querySendValue() const { return sendValue; }
  685. unsigned queryDepth() const { return depth; }
  686. bool qualify(CPTStack &stack, bool matchIfPartial)
  687. {
  688. ForEachItemIn(q, qualifierStack)
  689. {
  690. if (stack.ordinality() <= q+1)
  691. {
  692. // No more stack available (e.g. because deleted below this point)
  693. return matchIfPartial; // NB: return true if matchIfPartial=true (meaning head of subscriber path matched commit stack)
  694. }
  695. PTree &item = stack.item(q+1); // stack +1, top is root unqualified.
  696. CQualifiers *qualifiers = qualifierStack.item(q);
  697. if (qualifiers)
  698. {
  699. for (unsigned q2=0; q2<qualifiers->count(); q2++)
  700. {
  701. const char *qualifier = qualifiers->item(q2);
  702. const char *q = qualifier;
  703. bool numeric = true;
  704. for (;;)
  705. {
  706. if ('\0' == *q) break;
  707. else if (!isdigit(*q)) { numeric = false; break; }
  708. else q++;
  709. }
  710. if (numeric)
  711. {
  712. unsigned qnum = atoi(qualifier);
  713. if (!item.queryParent())
  714. {
  715. if (qnum != 1)
  716. return false;
  717. }
  718. else if (((PTree *)item.queryParent())->findChild(&item) != qnum-1)
  719. return false;
  720. }
  721. else if (!item.checkPattern(qualifier))
  722. return false;
  723. }
  724. }
  725. }
  726. return true;
  727. }
  728. MemoryBuffer &getInfo(MemoryBuffer &out)
  729. {
  730. out.append(id).append(sub).append(fullXpath);
  731. return out;
  732. }
  733. };
  734. typedef IArrayOf<CSubscriberContainer> CSubscriberArray;
  735. typedef ICopyArrayOf<CSubscriberContainer> CSubscriberCopyArray;
  736. class CSubscriberContainerList : public CInterface, public CSubscriberArray
  737. {
  738. public:
  739. CSubscriberContainerList() { }
  740. CSubscriberContainerList(const char *_xpath) : xpath(_xpath) { _xpath = xpath.get(); }
  741. const char *queryXPath() const { return xpath; }
  742. const char *queryFindString() const { return queryXPath(); }
  743. private:
  744. StringAttr xpath;
  745. };
  746. typedef OwningStringSuperHashTableOf<CSubscriberContainerList> CSubscriberXPathTable;
  747. class CSubscriberTable : public ThreadSafeSimpleHashTableOf<CSubscriberContainer, SubscriptionId>
  748. {
  749. public:
  750. ~CSubscriberTable() { _releaseAll(); }
  751. virtual void onAdd(void *et)
  752. {
  753. CSubscriberContainer *subscriber = (CSubscriberContainer *) et;
  754. CSubscriberContainerList *list = xpathTable.find(subscriber->queryXPath());
  755. if (!list)
  756. {
  757. list = new CSubscriberContainerList(subscriber->queryXPath());
  758. xpathTable.replace(*list);
  759. }
  760. list->append(*subscriber); // give over ownership.
  761. }
  762. virtual void onRemove(void *et)
  763. {
  764. CSubscriberContainer *subscriber = (CSubscriberContainer *) et;
  765. subscriber->setUnsubscribed();
  766. CSubscriberContainerList *list = xpathTable.find(subscriber->queryXPath());
  767. assertex(list);
  768. verifyex(list->zap(*subscriber));
  769. if (!list->ordinality())
  770. xpathTable.removeExact(list);
  771. }
  772. CSubscriberContainerList *getQualifiedList(const char *xpath, CPTStack &stack)
  773. {
  774. CriticalBlock b(crit);
  775. CSubscriberContainerList *list = xpathTable.find(xpath);
  776. if (!list) return NULL;
  777. CSubscriberContainerList *results = NULL;
  778. ForEachItemIn(s, *list)
  779. {
  780. CSubscriberContainer &subscriber = list->item(s);
  781. if (subscriber.qualify(stack, false))
  782. {
  783. if (!results) results = new CSubscriberContainerList(xpath);
  784. subscriber.Link();
  785. results->append(subscriber);
  786. }
  787. }
  788. return results;
  789. }
  790. void getSubscribers(CSubscriberArray &subs)
  791. {
  792. CriticalBlock b(crit);
  793. SuperHashIteratorOf<CSubscriberContainer> iter(queryBaseTable());
  794. ForEach(iter)
  795. {
  796. CSubscriberContainer &sub = iter.query();
  797. sub.Link();
  798. subs.append(sub);
  799. }
  800. }
  801. private:
  802. CSubscriberXPathTable xpathTable;
  803. };
  804. #ifdef _DEBUG
  805. struct DebugInfo
  806. {
  807. DebugInfo() { clearExclusive(); }
  808. void clearExclusive() { ExclOwningThread =0; ExclOwningConnection=0; ExclOwningSession=0; }
  809. ThreadId ExclOwningThread;
  810. ConnectionId ExclOwningConnection;
  811. SessionId ExclOwningSession;
  812. };
  813. #endif
  814. class BoolSetBlock
  815. {
  816. public:
  817. BoolSetBlock(bool &_b, bool state=true) : b(_b) { o = b; b = state; }
  818. ~BoolSetBlock() { b = o; }
  819. private:
  820. bool o, &b;
  821. };
  822. //////////////
  823. template <class ET>
  824. class LinkedStringHTMapping : public OwningStringHTMapping<ET>
  825. {
  826. public:
  827. LinkedStringHTMapping(const char *fp, ET &et) : OwningStringHTMapping<ET>(fp, et) { this->et.Link(); }
  828. };
  829. typedef LinkedStringHTMapping<IExternalHandler> CExternalHandlerMapping;
  830. typedef OwningStringSuperHashTableOf<CExternalHandlerMapping> CExternalHandlerTable;
  831. void serializeVisibleAttributes(IPropertyTree &tree, MemoryBuffer &mb)
  832. {
  833. IAttributeIterator *aIter = tree.getAttributes();
  834. if (aIter->first())
  835. {
  836. for (;;)
  837. {
  838. const char *attr = aIter->queryName();
  839. if (0 != strcmp(EXT_ATTR, attr))
  840. {
  841. mb.append(attr);
  842. mb.append(aIter->queryValue());
  843. }
  844. if (!aIter->next())
  845. break;
  846. }
  847. }
  848. aIter->Release();
  849. mb.append(""); // attribute terminator. i.e. blank attr name.
  850. }
  851. void writeDelta(StringBuffer &xml, IFile &iFile, const char *msg="", unsigned retrySecs=0, unsigned retryAttempts=10)
  852. {
  853. Owned<IException> exception;
  854. OwnedIFileIO iFileIO;
  855. unsigned _retryAttempts = retryAttempts;
  856. Owned<IFileIOStream> stream;
  857. offset_t lastGood = 0;
  858. unsigned startCrc = ~0;
  859. MemoryBuffer header;
  860. char strNum[17];
  861. for (;;)
  862. {
  863. header.append(deltaHeader);
  864. try
  865. {
  866. iFileIO.setown(iFile.open(IFOreadwrite));
  867. stream.setown(createIOStream(iFileIO));
  868. if (lastGood)
  869. {
  870. PROGLOG("Resetting delta file size");
  871. iFileIO->setSize(lastGood);
  872. }
  873. else
  874. {
  875. if (iFileIO->size())
  876. {
  877. iFileIO->read(deltaHeaderCrcOff, 10, strNum);
  878. startCrc = ~(unsigned)atoi64_l(strNum, 10);
  879. }
  880. else
  881. stream->write(strlen(deltaHeader), deltaHeader);
  882. lastGood = iFileIO->size();
  883. }
  884. stream->seek(0, IFSend);
  885. stream->write(xml.length(), xml.str());
  886. stream->flush();
  887. stream.clear();
  888. offset_t fLen = lastGood + xml.length();
  889. unsigned crc = crc32(xml.str(), xml.length(), startCrc);
  890. char *headerPtr = (char *)header.bufferBase();
  891. sprintf(strNum, "%010u", ~crc);
  892. memcpy(headerPtr + deltaHeaderCrcOff, strNum, 10);
  893. sprintf(strNum, "%016" I64F "X", fLen);
  894. memcpy(headerPtr + deltaHeaderSizeOff, strNum, 16);
  895. iFileIO->write(0, strlen(deltaHeader), headerPtr);
  896. }
  897. catch (IException *e)
  898. {
  899. exception.setown(e);
  900. StringBuffer s(msg);
  901. LOG(MCoperatorError, unknownJob, e, s.append("writeDelta, failed").str());
  902. }
  903. if (!exception.get())
  904. break;
  905. if (0 == retrySecs)
  906. return;
  907. if (0 == --_retryAttempts)
  908. {
  909. WARNLOG("writeDelta, too many retry attempts [%d]", retryAttempts);
  910. return;
  911. }
  912. exception.clear();
  913. WARNLOG("writeDelta, retrying");
  914. MilliSleep(retrySecs*1000);
  915. }
  916. }
  917. struct BackupQueueItem
  918. {
  919. static unsigned typeMask;
  920. enum flagt { f_delta=0x1, f_addext=0x2, f_delext=0x3, f_first=0x10 };
  921. BackupQueueItem() : edition((unsigned)-1), flags(0) { text = new StringBuffer; dataLength = 0; data = NULL; }
  922. ~BackupQueueItem()
  923. {
  924. delete text;
  925. if (data) free(data);
  926. }
  927. StringBuffer *text;
  928. unsigned edition;
  929. unsigned dataLength;
  930. void *data;
  931. byte flags;
  932. };
  933. unsigned BackupQueueItem::typeMask = 0x0f;
  934. class CBackupHandler : public CInterface, implements IThreaded
  935. {
  936. typedef QueueOf<BackupQueueItem, false> BackupQueue;
  937. CThreaded threaded;
  938. BackupQueue itemQueue, freeQueue;
  939. Semaphore pending, softQueueLimitSem;
  940. bool aborted, waiting, addWaiting, async;
  941. unsigned currentEdition, throttleCounter;
  942. CriticalSection queueCrit, freeQueueCrit;
  943. StringAttr backupPath;
  944. unsigned freeQueueLimit; // how many BackupQueueItems to cache for reuse
  945. unsigned largeWarningThreshold; // point at which to start warning about large queue
  946. unsigned softQueueLimit; // threshold over which primary transactions will be delay by small delay, to allow backup catchup.
  947. unsigned softQueueLimitDelay; // delay for above
  948. CTimeMon warningTime;
  949. unsigned recentTimeThrottled;
  950. unsigned lastNumWarnItems;
  951. IPropertyTree &config;
  952. const unsigned defaultFreeQueueLimit = 50;
  953. const unsigned defaultLargeWarningThreshold = 50;
  954. const unsigned defaultSoftQueueLimit = 200;
  955. const unsigned defaultSoftQueueLimitDelay = 200;
  956. BackupQueueItem *getFreeItem()
  957. {
  958. BackupQueueItem *item;
  959. {
  960. CriticalBlock b(freeQueueCrit);
  961. item = freeQueue.dequeue();
  962. }
  963. if (!item)
  964. item = new BackupQueueItem;
  965. return item;
  966. }
  967. void clearQueue(BackupQueue &queue)
  968. {
  969. for (;;)
  970. {
  971. BackupQueueItem *item = queue.dequeue();
  972. if (!item) break;
  973. delete item;
  974. }
  975. }
  976. void writeExt(const char *name, const unsigned length, const void *data, unsigned retrySecs=0, unsigned retryAttempts=10)
  977. {
  978. Owned<IException> exception;
  979. unsigned _retryAttempts = retryAttempts;
  980. StringBuffer rL(remoteBackupLocation);
  981. for (;;)
  982. {
  983. try
  984. {
  985. rL.append(name);
  986. Owned<IFile> iFile = createIFile(rL.str());
  987. Owned<IFileIO> fileIO = iFile->open(IFOcreate);
  988. fileIO->write(0, length, data);
  989. }
  990. catch (IException *e)
  991. {
  992. exception.setown(e);
  993. StringBuffer err("Saving external (backup): ");
  994. LOG(MCoperatorError, unknownJob, e, err.append(rL).str());
  995. }
  996. if (!exception.get())
  997. break;
  998. if (0 == retrySecs)
  999. return;
  1000. if (0 == --_retryAttempts)
  1001. {
  1002. WARNLOG("writeExt, too many retry attempts [%d]", retryAttempts);
  1003. return;
  1004. }
  1005. exception.clear();
  1006. WARNLOG("writeExt, retrying");
  1007. MilliSleep(retrySecs*1000);
  1008. }
  1009. }
  1010. void deleteExt(const char *name, unsigned retrySecs=0, unsigned retryAttempts=10)
  1011. {
  1012. Owned<IException> exception;
  1013. unsigned _retryAttempts = retryAttempts;
  1014. StringBuffer rL(remoteBackupLocation);
  1015. for (;;)
  1016. {
  1017. try
  1018. {
  1019. rL.append(name);
  1020. Owned<IFile> iFile = createIFile(rL.str());
  1021. iFile->remove();
  1022. }
  1023. catch (IException *e)
  1024. {
  1025. exception.setown(e);
  1026. StringBuffer err("Removing external (backup): ");
  1027. LOG(MCoperatorWarning, unknownJob, e, err.append(rL).str());
  1028. }
  1029. if (!exception.get())
  1030. break;
  1031. if (0 == retrySecs)
  1032. return;
  1033. if (0 == --_retryAttempts)
  1034. {
  1035. WARNLOG("deleteExt, too many retry attempts [%d]", retryAttempts);
  1036. return;
  1037. }
  1038. exception.clear();
  1039. WARNLOG("deleteExt, retrying");
  1040. MilliSleep(retrySecs*1000);
  1041. }
  1042. }
  1043. bool writeDelta(StringBuffer &xml, unsigned edition, bool first)
  1044. {
  1045. StringBuffer deltaFilename(backupPath);
  1046. constructStoreName(DELTANAME, edition, deltaFilename);
  1047. OwnedIFile iFile = createIFile(deltaFilename.str());
  1048. if (!first && !iFile->exists())
  1049. return false; // discard
  1050. ::writeDelta(xml, *iFile, "CBackupHandler - ", 60, 30);
  1051. return true;
  1052. }
  1053. void clearOld()
  1054. {
  1055. CriticalBlock b(queueCrit);
  1056. for (;;)
  1057. {
  1058. BackupQueueItem *item = itemQueue.dequeue();
  1059. if (!item) break;
  1060. if (BackupQueueItem::f_delta == (item->flags & BackupQueueItem::typeMask))
  1061. {
  1062. item->text->clear();
  1063. if (freeQueue.ordinality() < freeQueueLimit)
  1064. freeQueue.enqueue(item);
  1065. else
  1066. delete item;
  1067. }
  1068. }
  1069. if (addWaiting && itemQueue.ordinality()<softQueueLimit)
  1070. softQueueLimitSem.signal();
  1071. }
  1072. public:
  1073. CBackupHandler(IPropertyTree &_config) : config(_config), threaded("CBackupHandler")
  1074. {
  1075. currentEdition = (unsigned)-1;
  1076. addWaiting = waiting = async = false;
  1077. aborted = true;
  1078. throttleCounter = 0;
  1079. recentTimeThrottled = 0;
  1080. lastNumWarnItems = 0;
  1081. freeQueueLimit = config.getPropInt("@backupFreeQueueLimit", defaultFreeQueueLimit);
  1082. largeWarningThreshold = config.getPropInt("@backupLargeWarningThreshold", defaultLargeWarningThreshold);
  1083. softQueueLimit = config.getPropInt("@backupSoftQueueLimit", defaultSoftQueueLimit);
  1084. softQueueLimitDelay = config.getPropInt("@backupSoftQueueLimitDelay", defaultSoftQueueLimitDelay);
  1085. }
  1086. ~CBackupHandler()
  1087. {
  1088. clearQueue(freeQueue);
  1089. clearQueue(itemQueue);
  1090. }
  1091. void init(const char *_backupPath, bool _async)
  1092. {
  1093. backupPath.set(_backupPath);
  1094. async = _async;
  1095. aborted = false;
  1096. PROGLOG("BackupHandler started, async=%s", async?"true":"false");
  1097. threaded.init(this);
  1098. }
  1099. void stop()
  1100. {
  1101. if (!aborted)
  1102. {
  1103. aborted = true;
  1104. pending.signal();
  1105. threaded.join();
  1106. }
  1107. }
  1108. void removeExt(const char *fname)
  1109. {
  1110. if (aborted) return;
  1111. if (!async)
  1112. {
  1113. deleteExt(fname);
  1114. return;
  1115. }
  1116. BackupQueueItem *item = getFreeItem();
  1117. item->text->append(fname);
  1118. item->flags = BackupQueueItem::f_delext;
  1119. add(item);
  1120. }
  1121. void addExt(const char *fname, unsigned length, void *data)
  1122. {
  1123. if (aborted) return;
  1124. if (!async)
  1125. {
  1126. writeExt(fname, length, data);
  1127. free(data);
  1128. return;
  1129. }
  1130. BackupQueueItem *item = getFreeItem();
  1131. item->text->append(fname);
  1132. item->dataLength = length;
  1133. item->data = data; // take ownership
  1134. item->flags = BackupQueueItem::f_addext;
  1135. add(item);
  1136. }
  1137. void addDelta(StringBuffer &xml, unsigned edition, bool first)
  1138. {
  1139. if (aborted) return;
  1140. if (!async)
  1141. {
  1142. writeDelta(xml, edition, first);
  1143. if (xml.length() > 0x100000)
  1144. xml.kill();
  1145. else
  1146. xml.clear();
  1147. return;
  1148. }
  1149. if (edition != currentEdition)
  1150. {
  1151. clearOld();
  1152. currentEdition = edition;
  1153. }
  1154. BackupQueueItem *item = getFreeItem();
  1155. xml.swapWith(*item->text);
  1156. item->edition = edition;
  1157. item->flags = BackupQueueItem::f_delta | BackupQueueItem::f_first;
  1158. add(item);
  1159. }
  1160. void add(BackupQueueItem *item)
  1161. {
  1162. CriticalBlock b(queueCrit);
  1163. itemQueue.enqueue(item);
  1164. unsigned items=itemQueue.ordinality();
  1165. if (0==items%largeWarningThreshold)
  1166. {
  1167. if (items>lastNumWarnItems) // track as they go up
  1168. {
  1169. LOG(MCoperatorWarning, "Backup thread has a high # (%d) of pending transaction queued to write", items);
  1170. lastNumWarnItems = items;
  1171. }
  1172. else if (warningTime.elapsed() >= 60000) // if falling, avoid logging too much
  1173. {
  1174. LOG(MCoperatorWarning, "Backup thread has a high # (%d) of pending transaction queued to write", items);
  1175. lastNumWarnItems = 0;
  1176. warningTime.reset(0);
  1177. }
  1178. }
  1179. if (items>=softQueueLimit)
  1180. {
  1181. addWaiting = true;
  1182. unsigned ms = msTick();
  1183. {
  1184. CriticalUnblock b(queueCrit);
  1185. softQueueLimitSem.wait(softQueueLimitDelay);
  1186. }
  1187. addWaiting = false;
  1188. recentTimeThrottled += (msTick()-ms); // reset when queue < largeWarningThreshold
  1189. if (recentTimeThrottled >= softQueueLimitDelay && (0 == throttleCounter % 10)) // softQueueLimit exceeded - log every 10 transactions if recentTimeThrottled >= softQueueLimitDelay (1 unsignalled delay)
  1190. LOG(MCoperatorWarning, "Primary transactions are being delayed by lagging backup, currently %d queued, recent total throttle delay=%d", items, recentTimeThrottled);
  1191. ++throttleCounter; // also reset when queue < largeWarningThreshold
  1192. }
  1193. if (waiting)
  1194. pending.signal();
  1195. }
  1196. bool doIt(BackupQueueItem &item)
  1197. {
  1198. try
  1199. {
  1200. switch (item.flags & BackupQueueItem::typeMask)
  1201. {
  1202. case BackupQueueItem::f_delta:
  1203. return writeDelta(*item.text, item.edition, 0 != (item.flags & BackupQueueItem::f_first));
  1204. case BackupQueueItem::f_addext:
  1205. writeExt(item.text->str(), item.dataLength, item.data, 60, 30);
  1206. return true;
  1207. case BackupQueueItem::f_delext:
  1208. deleteExt(item.text->str(), 60, 30);
  1209. return true;
  1210. }
  1211. }
  1212. catch (IException *e)
  1213. {
  1214. LOG(MCoperatorWarning, e, "BackupHandler(async) write operation failed, possible backup data loss");
  1215. e->Release();
  1216. }
  1217. return false;
  1218. }
  1219. // IThreaded
  1220. void main()
  1221. {
  1222. for (;;)
  1223. {
  1224. BackupQueueItem *item=NULL;
  1225. do
  1226. {
  1227. CriticalBlock b(queueCrit);
  1228. if (itemQueue.ordinality())
  1229. {
  1230. item = itemQueue.dequeue();
  1231. if (addWaiting && itemQueue.ordinality()<softQueueLimit)
  1232. softQueueLimitSem.signal();
  1233. if (itemQueue.ordinality() < largeWarningThreshold) // reset stats when falls below
  1234. {
  1235. recentTimeThrottled = 0;
  1236. throttleCounter = 0;
  1237. }
  1238. }
  1239. else
  1240. {
  1241. waiting = true;
  1242. {
  1243. CriticalUnblock b(queueCrit);
  1244. if (!aborted)
  1245. pending.wait();
  1246. }
  1247. waiting = false;
  1248. }
  1249. if (aborted)
  1250. {
  1251. if (item) delete item;
  1252. PROGLOG("BackupHandler stopped");
  1253. return;
  1254. }
  1255. }
  1256. while (!item);
  1257. if (!doIt(*item))
  1258. clearOld();
  1259. CriticalBlock b(freeQueueCrit);
  1260. if (freeQueue.ordinality() < freeQueueLimit)
  1261. {
  1262. if (item->text->length() > 0x100000)
  1263. item->text->kill();
  1264. else
  1265. item->text->clear();
  1266. if (item->data)
  1267. {
  1268. free(item->data);
  1269. item->data = NULL;
  1270. item->dataLength = 0;
  1271. }
  1272. freeQueue.enqueue(item);
  1273. }
  1274. else
  1275. delete item;
  1276. }
  1277. }
  1278. };
  1279. class CExternalFile : public CInterface
  1280. {
  1281. StringAttr ext, dataPath;
  1282. protected:
  1283. CBackupHandler &backupHandler;
  1284. public:
  1285. CExternalFile(const char *_ext, const char *_dataPath, CBackupHandler &_backupHandler) : ext(_ext), dataPath(_dataPath), backupHandler(_backupHandler) { }
  1286. const char *queryExt() { return ext; }
  1287. StringBuffer &getName(StringBuffer &fName, const char *base)
  1288. {
  1289. return fName.append(base).append(ext);
  1290. }
  1291. StringBuffer &getFilename(StringBuffer &fName, const char *base)
  1292. {
  1293. return fName.append(dataPath).append(base).append(ext);
  1294. }
  1295. bool isValid(const char *name)
  1296. {
  1297. StringBuffer filename;
  1298. getFilename(filename, name);
  1299. Owned<IFile> iFile = createIFile(filename.str());
  1300. return iFile->exists();
  1301. }
  1302. void remove(const char *name)
  1303. {
  1304. StringBuffer filename;
  1305. getFilename(filename, name);
  1306. Owned<IFile> iFile = createIFile(filename.str());
  1307. iFile->remove();
  1308. if (remoteBackupLocation.length())
  1309. {
  1310. StringBuffer fname(name);
  1311. backupHandler.removeExt(fname.append(queryExt()).str());
  1312. }
  1313. }
  1314. };
  1315. class CLegacyBinaryFileExternal : public CExternalFile, implements IExternalHandler
  1316. {
  1317. public:
  1318. IMPLEMENT_IINTERFACE;
  1319. CLegacyBinaryFileExternal(const char *dataPath, CBackupHandler &backupHandler) : CExternalFile("." EF_LegacyBinaryValue, dataPath, backupHandler) { }
  1320. virtual void resetAsExternal(IPropertyTree &tree)
  1321. {
  1322. tree.setProp(NULL, (char *)NULL);
  1323. }
  1324. virtual void readValue(const char *name, MemoryBuffer &mb)
  1325. {
  1326. StringBuffer filename;
  1327. getFilename(filename, name);
  1328. Owned<IFile> iFile = createIFile(filename.str());
  1329. size32_t sz = (size32_t)iFile->size();
  1330. if ((unsigned)-1 == sz)
  1331. {
  1332. StringBuffer s("Missing external file ");
  1333. Owned<IException> e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str());
  1334. LOG(MCoperatorWarning, unknownJob, e, s.str());
  1335. StringBuffer str("EXTERNAL BINARY FILE: \"");
  1336. str.append(filename.str()).append("\" MISSING");
  1337. CPTValue v(str.length()+1, str.str(), false);
  1338. v.serialize(mb);
  1339. }
  1340. else
  1341. {
  1342. Owned<IFileIO> fileIO = iFile->open(IFOread);
  1343. MemoryBuffer vmb;
  1344. verifyex(sz == ::read(fileIO, 0, sz, vmb));
  1345. CPTValue v(sz, vmb.toByteArray(), true);
  1346. v.serialize(mb);
  1347. }
  1348. }
  1349. virtual void read(const char *name, IPropertyTree &owner, MemoryBuffer &mb, bool withValue)
  1350. {
  1351. StringBuffer filename;
  1352. getFilename(filename, name);
  1353. const char *_name = owner.queryName();
  1354. if (!_name) _name = "";
  1355. mb.append(_name);
  1356. byte flags = ((PTree &)owner).queryFlags();
  1357. mb.append(IptFlagSet(flags, ipt_binary));
  1358. serializeVisibleAttributes(owner, mb);
  1359. Owned<IFile> iFile = createIFile(filename.str());
  1360. size32_t sz = (size32_t)iFile->size();
  1361. if ((unsigned)-1 == sz)
  1362. {
  1363. StringBuffer s("Missing external file ");
  1364. if (*_name)
  1365. s.append("in property ").append(_name);
  1366. Owned<IException> e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str());
  1367. LOG(MCoperatorWarning, unknownJob, e, s.str());
  1368. if (withValue)
  1369. {
  1370. StringBuffer str("EXTERNAL BINARY FILE: \"");
  1371. str.append(filename.str()).append("\" MISSING");
  1372. CPTValue v(str.length()+1, str.str(), false);
  1373. v.serialize(mb);
  1374. }
  1375. else
  1376. mb.append((size32_t)0);
  1377. }
  1378. else
  1379. {
  1380. if (withValue)
  1381. {
  1382. MemoryBuffer vmb;
  1383. Owned<IFileIO> fileIO = iFile->open(IFOread);
  1384. verifyex(sz == ::read(fileIO, 0, sz, vmb));
  1385. CPTValue v(sz, vmb.toByteArray(), true);
  1386. v.serialize(mb);
  1387. }
  1388. else
  1389. mb.append((size32_t)0);
  1390. }
  1391. }
  1392. virtual void write(const char *name, IPropertyTree &tree)
  1393. {
  1394. StringBuffer filename;
  1395. getFilename(filename, name);
  1396. Owned<IFile> iFile = createIFile(filename.str());
  1397. Owned<IFileIO> fileIO = iFile->open(IFOcreate);
  1398. MemoryBuffer out;
  1399. ((PTree &)tree).queryValue()->serialize(out);
  1400. const char *data = out.toByteArray();
  1401. unsigned length = out.length();
  1402. fileIO->write(0, length, data);
  1403. if (remoteBackupLocation.length())
  1404. {
  1405. StringBuffer fname(name);
  1406. backupHandler.addExt(fname.append(queryExt()).str(), length, out.detach());
  1407. }
  1408. }
  1409. virtual void remove(const char *name) { CExternalFile::remove(name); }
  1410. virtual bool isValid(const char *name) { return CExternalFile::isValid(name); }
  1411. virtual StringBuffer &getName(StringBuffer &fName, const char *base) { return CExternalFile::getName(fName, base); }
  1412. virtual StringBuffer &getFilename(StringBuffer &fName, const char *base) { return CExternalFile::getFilename(fName, base); }
  1413. };
  1414. class CBinaryFileExternal : public CExternalFile, implements IExternalHandler
  1415. {
  1416. public:
  1417. IMPLEMENT_IINTERFACE;
  1418. CBinaryFileExternal(const char *dataPath, CBackupHandler &backupHandler) : CExternalFile("." EF_BinaryValue, dataPath, backupHandler) { }
  1419. virtual void resetAsExternal(IPropertyTree &tree)
  1420. {
  1421. tree.setProp(NULL, (char *)NULL);
  1422. }
  1423. virtual void readValue(const char *name, MemoryBuffer &mb)
  1424. {
  1425. StringBuffer filename;
  1426. getFilename(filename, name);
  1427. Owned<IFile> iFile = createIFile(filename.str());
  1428. size32_t sz = (size32_t)iFile->size();
  1429. if ((unsigned)-1 == sz)
  1430. {
  1431. StringBuffer s("Missing external file ");
  1432. Owned<IException> e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str());
  1433. LOG(MCoperatorWarning, unknownJob, e, s.str());
  1434. StringBuffer str("EXTERNAL BINARY FILE: \"");
  1435. str.append(filename.str()).append("\" MISSING");
  1436. CPTValue v(str.length()+1, str.str(), false);
  1437. v.serialize(mb);
  1438. }
  1439. else
  1440. {
  1441. Owned<IFileIO> fileIO = iFile->open(IFOread);
  1442. verifyex(sz == ::read(fileIO, 0, sz, mb));
  1443. }
  1444. }
  1445. virtual void read(const char *name, IPropertyTree &owner, MemoryBuffer &mb, bool withValue)
  1446. {
  1447. StringBuffer filename;
  1448. getFilename(filename, name);
  1449. const char *_name = owner.queryName();
  1450. if (!_name) _name = "";
  1451. mb.append(_name);
  1452. Owned<IFile> iFile = createIFile(filename.str());
  1453. size32_t sz = (size32_t)iFile->size();
  1454. if ((unsigned)-1 == sz)
  1455. {
  1456. byte flags = ((PTree &)owner).queryFlags();
  1457. IptFlagClr(flags, ipt_binary);
  1458. mb.append(flags);
  1459. serializeVisibleAttributes(owner, mb);
  1460. StringBuffer s("Missing external file ");
  1461. if (*_name)
  1462. s.append("in property ").append(_name);
  1463. Owned<IException> e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str());
  1464. LOG(MCoperatorWarning, unknownJob, e, s.str());
  1465. if (withValue)
  1466. {
  1467. StringBuffer str("EXTERNAL BINARY FILE: \"");
  1468. str.append(filename.str()).append("\" MISSING");
  1469. CPTValue v(str.length()+1, str.str(), false);
  1470. v.serialize(mb);
  1471. }
  1472. else
  1473. mb.append((size32_t)0);
  1474. }
  1475. else
  1476. {
  1477. byte flags = ((PTree &)owner).queryFlags();
  1478. mb.append(flags);
  1479. serializeVisibleAttributes(owner, mb);
  1480. if (withValue)
  1481. {
  1482. Owned<IFileIO> fileIO = iFile->open(IFOread);
  1483. verifyex(sz == ::read(fileIO, 0, sz, mb));
  1484. }
  1485. else
  1486. mb.append((size32_t)0);
  1487. }
  1488. }
  1489. virtual void write(const char *name, IPropertyTree &tree)
  1490. {
  1491. StringBuffer filename;
  1492. getFilename(filename, name);
  1493. Owned<IFile> iFile = createIFile(filename.str());
  1494. Owned<IFileIO> fileIO = iFile->open(IFOcreate);
  1495. MemoryBuffer out;
  1496. ((PTree &)tree).queryValue()->serialize(out);
  1497. const char *data = out.toByteArray();
  1498. unsigned length = out.length();
  1499. fileIO->write(0, length, data);
  1500. if (remoteBackupLocation.length())
  1501. {
  1502. StringBuffer fname(name);
  1503. backupHandler.addExt(fname.append(queryExt()).str(), length, out.detach());
  1504. }
  1505. }
  1506. virtual void remove(const char *name) { CExternalFile::remove(name); }
  1507. virtual bool isValid(const char *name) { return CExternalFile::isValid(name); }
  1508. virtual StringBuffer &getName(StringBuffer &fName, const char *base) { return CExternalFile::getName(fName, base); }
  1509. virtual StringBuffer &getFilename(StringBuffer &fName, const char *base) { return CExternalFile::getFilename(fName, base); }
  1510. };
  1511. class CXMLFileExternal : public CExternalFile, implements IExternalHandler
  1512. {
  1513. public:
  1514. IMPLEMENT_IINTERFACE;
  1515. CXMLFileExternal(const char *dataPath, CBackupHandler &backupHandler) : CExternalFile("." EF_XML, dataPath, backupHandler) { }
  1516. virtual void resetAsExternal(IPropertyTree &_tree)
  1517. {
  1518. PTree &tree = *QUERYINTERFACE(&_tree, PTree);
  1519. ::Release(tree.detach());
  1520. }
  1521. virtual void readValue(const char *name, MemoryBuffer &mb)
  1522. {
  1523. StringBuffer filename;
  1524. getFilename(filename, name);
  1525. OwnedIFile ifile = createIFile(filename.str());
  1526. if (!ifile->exists())
  1527. {
  1528. StringBuffer s("Missing external file ");
  1529. Owned<IException> e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str());
  1530. LOG(MCoperatorWarning, unknownJob, e, s.str());
  1531. StringBuffer str("EXTERNAL XML FILE: \"");
  1532. str.append(filename.str()).append("\" MISSING");
  1533. CPTValue v(str.length()+1, str.str(), false);
  1534. v.serialize(mb);
  1535. }
  1536. else
  1537. {
  1538. Owned<IPropertyTree> tree;
  1539. tree.setown(createPTreeFromXMLFile(filename.str()));
  1540. IPTArrayValue *v = ((PTree *)tree.get())->queryValue();
  1541. if (v)
  1542. v->serialize(mb);
  1543. else
  1544. mb.append((size32_t)0);
  1545. }
  1546. }
  1547. virtual void read(const char *name, IPropertyTree &owner, MemoryBuffer &mb, bool withValue)
  1548. {
  1549. StringBuffer filename;
  1550. getFilename(filename, name);
  1551. Owned<IPropertyTree> tree;
  1552. OwnedIFile ifile = createIFile(filename.str());
  1553. if (!ifile->exists())
  1554. {
  1555. StringBuffer s("Missing external file ");
  1556. const char *name = owner.queryName();
  1557. if (name && *name)
  1558. s.append("in property ").append(name);
  1559. Owned<IException> e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str());
  1560. LOG(MCoperatorWarning, unknownJob, e, s.str());
  1561. StringBuffer str("EXTERNAL XML FILE: \"");
  1562. str.append(filename.str()).append("\" MISSING");
  1563. tree.setown(createPTree(owner.queryName()));
  1564. if (withValue)
  1565. tree->setProp(NULL, str.str());
  1566. }
  1567. else
  1568. {
  1569. tree.setown(createPTreeFromXMLFile(filename.str()));
  1570. if (!withValue)
  1571. tree->setProp(NULL, (char *)NULL);
  1572. }
  1573. ((PTree *)tree.get())->serializeSelf(mb);
  1574. }
  1575. virtual void write(const char *name, IPropertyTree &tree)
  1576. {
  1577. StringBuffer filename;
  1578. getFilename(filename, name);
  1579. Owned<IFile> iFile = createIFile(filename.str());
  1580. Owned<IFileIO> fileIO = iFile->open(IFOcreate);
  1581. Owned<IFileIOStream> fstream = createBufferedIOStream(fileIO);
  1582. toXML(&tree, *fstream);
  1583. if (remoteBackupLocation.length())
  1584. {
  1585. StringBuffer fname(name);
  1586. StringBuffer str;
  1587. toXML(&tree, str);
  1588. unsigned l = str.length();
  1589. backupHandler.addExt(fname.append(queryExt()).str(), l, str.detach());
  1590. }
  1591. }
  1592. virtual void remove(const char *name) { CExternalFile::remove(name); }
  1593. virtual bool isValid(const char *name) { return CExternalFile::isValid(name); }
  1594. virtual StringBuffer &getName(StringBuffer &fName, const char *base) { return CExternalFile::getName(fName, base); }
  1595. virtual StringBuffer &getFilename(StringBuffer &fName, const char *base) { return CExternalFile::getFilename(fName, base); }
  1596. };
  1597. //////////////
  1598. class CBranchChange : public CInterface
  1599. {
  1600. DECL_NAMEDCOUNT;
  1601. typedef CIArrayOf<CBranchChange> CBranchChangeChildren;
  1602. public:
  1603. CBranchChange(CRemoteTreeBase &_tree) : tree(&_tree), local(PDS_None), state(PDS_None) { INIT_NAMEDCOUNT; }
  1604. void noteChange(PDState _local, PDState _state) { local = _local; state = _state; }
  1605. void addChildBranch(CBranchChange &child) { children.append(child); }
  1606. const void *queryFindParam() const { return (const void *) &tree; }
  1607. CBranchChangeChildren children;
  1608. Linked<CRemoteTreeBase> tree;
  1609. PDState local, state; // change info
  1610. };
  1611. SDSNotifyFlags translatePDState(PDState state)
  1612. {
  1613. return (SDSNotifyFlags) state; // mirrored for now.
  1614. }
  1615. void buildNotifyData(MemoryBuffer &notifyData, PDState state, CPTStack *stack, MemoryBuffer *data)
  1616. {
  1617. if (stack)
  1618. {
  1619. notifyData.append('/');
  1620. PTree *parent = &stack->item(0); // root
  1621. unsigned n = stack->ordinality();
  1622. if (n>1)
  1623. {
  1624. unsigned s = 1;
  1625. for (;;)
  1626. {
  1627. PTree &child = stack->item(s);
  1628. const char *str = child.queryName();
  1629. notifyData.append(strlen(str), str);
  1630. if (child.queryParent())
  1631. {
  1632. char temp[12];
  1633. unsigned written = numtostr(temp, parent->findChild(&child)+1);
  1634. notifyData.append('[').append(written, temp).append(']');
  1635. }
  1636. else
  1637. notifyData.append(3, "[1]");
  1638. parent = &child;
  1639. s++;
  1640. if (s<n)
  1641. notifyData.append('/');
  1642. else
  1643. break;
  1644. }
  1645. }
  1646. notifyData.append('\0');
  1647. }
  1648. notifyData.append((int)translatePDState(state));
  1649. if (data)
  1650. {
  1651. notifyData.append(true);
  1652. notifyData.append(data->length());
  1653. notifyData.append(*data);
  1654. }
  1655. else
  1656. notifyData.append(false);
  1657. }
  1658. class CSubscriberNotifier;
  1659. typedef SimpleHashTableOf<CSubscriberNotifier, SubscriptionId> CSubscriberNotifierTable;
  1660. class CSubscriberNotifier : public CInterface
  1661. {
  1662. DECL_NAMEDCOUNT;
  1663. class CChange : public CInterface
  1664. {
  1665. public:
  1666. CChange(MemoryBuffer &_notifyData) : notifyData(_notifyData.length(), _notifyData.toByteArray()) { }
  1667. MemoryBuffer notifyData;
  1668. };
  1669. public:
  1670. CSubscriberNotifier(CSubscriberNotifierTable &_table, CSubscriberContainerBase &_subscriber, MemoryBuffer &notifyData)
  1671. : table(_table), subscriber(_subscriber) //NB: takes ownership of subscriber
  1672. {
  1673. INIT_NAMEDCOUNT;
  1674. change.setown(new CChange(notifyData));
  1675. }
  1676. ~CSubscriberNotifier() { subscriber.Release(); }
  1677. void queueChange(MemoryBuffer &notifyData)
  1678. {
  1679. changeQueue.append(*new CChange(notifyData));
  1680. }
  1681. void notify()
  1682. {
  1683. for (;;)
  1684. {
  1685. if (!subscriber.notify(change->notifyData))
  1686. {
  1687. subscriber.setUnsubscribed();
  1688. break;
  1689. }
  1690. else if (subscriber.isUnsubscribed())
  1691. break;
  1692. CHECKEDCRITICALBLOCK(nfyTableCrit, fakeCritTimeout);
  1693. if (changeQueue.ordinality())
  1694. {
  1695. change.set(&changeQueue.item(0));
  1696. changeQueue.remove(0);
  1697. }
  1698. else
  1699. {
  1700. table.removeExact(this);
  1701. break;
  1702. }
  1703. }
  1704. if (subscriber.isUnsubscribed())
  1705. {
  1706. { CHECKEDCRITICALBLOCK(nfyTableCrit, fakeCritTimeout);
  1707. table.removeExact(this);
  1708. }
  1709. querySubscriptionManager(SDS_PUBLISHER)->remove(subscriber.queryId());
  1710. }
  1711. }
  1712. const void *queryFindParam() const
  1713. {
  1714. return &(subscriber.queryId());
  1715. }
  1716. private:
  1717. Linked<CChange> change;
  1718. CIArrayOf<CChange> changeQueue;
  1719. CSubscriberContainerBase &subscriber;
  1720. MemoryAttr notifyData;
  1721. CSubscriberNotifierTable &table;
  1722. };
  1723. ////////////////
  1724. typedef MapBetween<SubscriptionId, SubscriptionId, ConnectionId, ConnectionId> SubConnMap;
  1725. class CConnectionSubscriptionManager : implements ISubscriptionManager, CInterface
  1726. {
  1727. public:
  1728. IMPLEMENT_IINTERFACE;
  1729. CConnectionSubscriptionManager()
  1730. {
  1731. }
  1732. unsigned querySubscribers()
  1733. {
  1734. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  1735. return subConnMap.count();
  1736. }
  1737. virtual void add(ISubscription *sub, SubscriptionId id);
  1738. virtual void remove(SubscriptionId id);
  1739. private:
  1740. SubConnMap subConnMap;
  1741. CheckedCriticalSection crit;
  1742. };
  1743. //////////
  1744. class CLock;
  1745. typedef ThreadSafeOwningSimpleHashTableOf<CLock, __int64> CLockTable;
  1746. //////////
  1747. interface IUnlockCallback
  1748. {
  1749. virtual void block() = 0;
  1750. virtual void unblock() = 0;
  1751. };
  1752. //////////
  1753. typedef LinkedStringHTMapping<ISDSNotifyHandler> CSDSNotifyHandlerMapping;
  1754. typedef OwningStringSuperHashTableOf<CSDSNotifyHandlerMapping> CNotifyHandlerTable;
  1755. class CServerRemoteTree;
  1756. typedef CServerRemoteTree *CServerRemoteTreePtr;
  1757. typedef MapBetween<__int64, __int64, CServerRemoteTreePtr, CServerRemoteTreePtr> UniqueTreeMapping;
  1758. struct CStoreInfo
  1759. {
  1760. unsigned edition;
  1761. unsigned crc;
  1762. StringAttr cache;
  1763. };
  1764. interface ICoalesce : extends IInterface
  1765. {
  1766. virtual void start() = 0;
  1767. virtual void stop() = 0;
  1768. };
  1769. //////////////////////
  1770. class CNodeSubscriberContainer;
  1771. interface INodeSubscriptionManager : extends ISubscriptionManager
  1772. {
  1773. virtual void associateSubscriber(CNodeSubscriberContainer &subscriber) = 0;
  1774. virtual void removeSubscriberAssociation(SubscriptionId id) = 0;
  1775. virtual void notifyDelete(CServerRemoteTree *node) = 0;
  1776. virtual void notify(CServerRemoteTree &node, PDState state) = 0;
  1777. virtual MemoryBuffer &collectSubscribers(MemoryBuffer &out) const = 0;
  1778. };
  1779. //////////////////////
  1780. enum LockStatus { LockFailed, LockHeld, LockTimedOut, LockSucceeded };
  1781. class CCovenSDSManager : public CSDSManagerBase, implements ISDSManagerServer, implements ISubscriptionManager, implements IExceptionHandler
  1782. {
  1783. public:
  1784. IMPLEMENT_IINTERFACE;
  1785. CCovenSDSManager(ICoven &_coven, IPropertyTree &config, const char *dataPath);
  1786. ~CCovenSDSManager();
  1787. void start();
  1788. void stop();
  1789. void restart(IException * e);
  1790. void loadStore(const char *store=NULL, const bool *abort=NULL);
  1791. void saveStore(const char *store=NULL, bool currentEdition=false);
  1792. bool unlock(__int64 treeId, ConnectionId connectionId, bool delayDelete=false);
  1793. void unlockAll(__int64 treeId);
  1794. void changeLockMode(CServerConnection &connection, unsigned newMode, unsigned timeout);
  1795. void clearSDSLocks();
  1796. void lock(CServerRemoteTree &tree, const char *xpath, ConnectionId connectionId, SessionId sessionId, unsigned mode, unsigned timeout, IUnlockCallback &callback);
  1797. CLock *queryLock(__int64 id) { return lockTable.find(&id); }
  1798. CSubscriberTable &querySubscriberTable() { return subscribers; }
  1799. IExternalHandler *queryExternalHandler(const char *handler) { if (!handler) return NULL; CExternalHandlerMapping *mapping = externalHandlers.find(handler); return mapping ? &mapping->query() : NULL; }
  1800. void handleNotify(CSubscriberContainerBase *subscriber, MemoryBuffer &notifyData);
  1801. void startNotification(IPropertyTree &changeTree, CPTStack &stack, CBranchChange &changes); // subscription notification
  1802. MemoryBuffer &collectUsageStats(MemoryBuffer &out);
  1803. MemoryBuffer &collectConnections(MemoryBuffer &out);
  1804. MemoryBuffer &collectSubscribers(MemoryBuffer &out);
  1805. void blockingSave(unsigned *writeTransactions=NULL);
  1806. bool queryStopped() { return server.queryStopped(); }
  1807. void handleNodeNotify(notifications n, CServerRemoteTree &tree); // server node notification
  1808. void deleteExternal(__int64 index);
  1809. void serializeExternal(__int64 index, IPropertyTree &owner, MemoryBuffer &mb, bool withValue);
  1810. void writeExternal(CServerRemoteTree &tree, bool direct=false, __int64 existing=0);
  1811. inline unsigned queryExternalThreshold() { return externalSizeThreshold; }
  1812. CServerConnection *queryConnection(ConnectionId id);
  1813. CServerConnection *getConnection(ConnectionId id);
  1814. inline CFitArray &queryAllNodes() { return allNodes; }
  1815. unsigned __int64 getNextExternal() { return nextExternal++; }
  1816. CServerConnection *createConnectionInstance(CRemoteTreeBase *root, SessionId sessionId, unsigned mode, unsigned timeout, const char *xpath, CRemoteTreeBase *&tree, ConnectionId connectionId, StringAttr *deltaPath, Owned<IPropertyTree> &deltaChange, Owned<CBranchChange> &branchChange, unsigned &additions);
  1817. void createConnection(SessionId sessionId, unsigned mode, unsigned timeout, const char *xpath, CServerRemoteTree *&tree, ConnectionId &connectionId, bool primary, Owned<LinkingCriticalBlock> &connectCritBlock);
  1818. void disconnect(ConnectionId connectionId, bool deleteRoot=false, Owned<CLCLockBlock> *lockBlock=NULL);
  1819. void registerTree(__int64 serverId, CServerRemoteTree &tree);
  1820. void unregisterTree(__int64 uniqId);
  1821. CServerRemoteTree *queryRegisteredTree(__int64 uniqId);
  1822. CServerRemoteTree *getRegisteredTree(__int64 uniqId);
  1823. CServerRemoteTree *queryRoot();
  1824. void saveDelta(const char *path, IPropertyTree &changeTree);
  1825. CSubscriberContainerList *getSubscribers(const char *xpath, CPTStack &stack);
  1826. void getExternalValue(__int64 index, MemoryBuffer &mb);
  1827. IPropertyTree *getXPathsSortLimitMatchTree(const char *baseXPath, const char *matchXPath, const char *sortby, bool caseinsensitive, bool ascending, unsigned from, unsigned limit);
  1828. void addNodeSubscriber(ISubscription *sub, SubscriptionId id);
  1829. void removeNodeSubscriber(SubscriptionId id);
  1830. void notifyNodeDelete(CServerRemoteTree &node);
  1831. void notifyNode(CServerRemoteTree &node, PDState state);
  1832. // ISDSConnectionManager
  1833. virtual CRemoteTreeBase *get(CRemoteConnection &connection, __int64 serverId);
  1834. virtual void getChildren(CRemoteTreeBase &parent, CRemoteConnection &connection, unsigned levels);
  1835. virtual void getChildrenFor(CRTArray &childLessList, CRemoteConnection &connection, unsigned levels);
  1836. virtual void ensureLocal(CRemoteConnection &connection, CRemoteTreeBase &_parent, IPropertyTree *serverMatchTree, IPTIteratorCodes flags=iptiter_null);
  1837. virtual IPropertyTreeIterator *getElements(CRemoteConnection &connection, const char *xpath);
  1838. virtual void commit(CRemoteConnection &connection, bool *disconnectDeleteRoot);
  1839. virtual void changeMode(CRemoteConnection &connection, unsigned mode, unsigned timeout, bool suppressReload);
  1840. virtual IPropertyTree *getXPaths(__int64 serverId, const char *xpath, bool getServerIds=false);
  1841. virtual IPropertyTreeIterator *getXPathsSortLimit(const char *baseXPath, const char *matchXPath, const char *sortby, bool caseinsensitive, bool ascending, unsigned from, unsigned limit);
  1842. virtual void getExternalValueFromServerId(__int64 serverId, MemoryBuffer &mb);
  1843. virtual bool unlock(__int64 connectionId, bool closeConn, StringBuffer &connectionInfo);
  1844. // ISDSManagerServer
  1845. virtual IRemoteConnections *connect(IMultipleConnector *mConnect, SessionId id, unsigned timeout);
  1846. virtual IRemoteConnection *connect(const char *xpath, SessionId id, unsigned mode, unsigned timeout);
  1847. virtual SubscriptionId subscribe(const char *xpath, ISDSSubscription &notify, bool sub=true, bool sendValue=false);
  1848. virtual SubscriptionId subscribeExact(const char *xpath, ISDSNodeSubscription &notify, bool sendValue=false);
  1849. virtual void unsubscribe(SubscriptionId id);
  1850. virtual void unsubscribeExact(SubscriptionId id);
  1851. virtual ILockInfoCollection *getLocks(const char *ipPattern, const char *xpathPattern);
  1852. virtual StringBuffer &getUsageStats(StringBuffer &out);
  1853. virtual StringBuffer &getConnections(StringBuffer &out);
  1854. virtual StringBuffer &getSubscribers(StringBuffer &out);
  1855. virtual StringBuffer &getExternalReport(StringBuffer &out);
  1856. virtual void installNotifyHandler(const char *handlerKey, ISDSNotifyHandler *handler);
  1857. virtual bool removeNotifyHandler(const char *handlerKey);
  1858. virtual IPropertyTree *lockStoreRead() const;
  1859. virtual void unlockStoreRead() const;
  1860. virtual unsigned countConnections();
  1861. virtual bool setSDSDebug(StringArray &params, StringBuffer &reply);
  1862. virtual unsigned countActiveLocks();
  1863. virtual unsigned queryExternalSizeThreshold() const { return externalSizeThreshold; }
  1864. virtual void setExternalSizeThreshold(unsigned _size) { externalSizeThreshold = _size; }
  1865. virtual bool queryRestartOnError() const { return restartOnError; }
  1866. virtual void setRestartOnError(bool _restart) { restartOnError = _restart; }
  1867. unsigned queryRequestsPending() const { return coven.probe(RANK_ALL,MPTAG_DALI_SDS_REQUEST,NULL); }
  1868. unsigned queryXactCount() const { return server.queryXactTimingStats().queryCount(); }
  1869. unsigned queryXactMeanTime() const { return server.queryXactTimingStats().queryMeanTime(); }
  1870. unsigned queryXactMaxTime() const { return server.queryXactTimingStats().queryMaxTime(); }
  1871. unsigned queryXactMinTime() const { return server.queryXactTimingStats().queryMinTime(); }
  1872. unsigned queryConnectMeanTime() const { return server.queryConnectTimingStats().queryMeanTime(); }
  1873. unsigned queryConnectMaxTime() const { return server.queryConnectTimingStats().queryMaxTime(); }
  1874. unsigned queryCommitMeanTime() const { return server.queryCommitTimingStats().queryMeanTime(); }
  1875. unsigned queryCommitMaxTime() const { return server.queryCommitTimingStats().queryMaxTime(); }
  1876. unsigned queryCommitMeanSize() const { return server.queryCommitTimingStats().queryMeanSize(); }
  1877. virtual void saveRequest();
  1878. virtual IPropertyTree &queryProperties() const;
  1879. virtual IPropertyTreeIterator *getElementsRaw(const char *xpath,INode *remotedali=NULL, unsigned timeout=MP_WAIT_FOREVER);
  1880. virtual void setConfigOpt(const char *opt, const char *value);
  1881. virtual unsigned queryCount(const char *xpath);
  1882. virtual bool updateEnvironment(IPropertyTree *newEnv, bool forceGroupUpdate, StringBuffer &response);
  1883. // ISubscriptionManager impl.
  1884. virtual void add(ISubscription *subs,SubscriptionId id);
  1885. virtual void remove(SubscriptionId id);
  1886. // IExceptionHandler
  1887. virtual bool fireException(IException *e);
  1888. public: // data
  1889. mutable ReadWriteLock dataRWLock;
  1890. CheckedCriticalSection connectCrit;
  1891. CheckedCriticalSection connDestructCrit;
  1892. CheckedCriticalSection cTableCrit;
  1893. CheckedCriticalSection sTableCrit;
  1894. CheckedCriticalSection lockCrit;
  1895. CheckedCriticalSection treeRegCrit;
  1896. Owned<Thread> unhandledThread;
  1897. unsigned writeTransactions;
  1898. bool ignoreExternals;
  1899. StringAttr dataPath;
  1900. Owned<IPropertyTree> properties;
  1901. private:
  1902. void validateBackup();
  1903. void validateDeltaBackup();
  1904. LockStatus establishLock(CLock &lock, __int64 treeId, ConnectionId connectionId, SessionId sessionId, unsigned mode, unsigned timeout, IUnlockCallback &lockCallback);
  1905. void _getChildren(CRemoteTreeBase &parent, CServerRemoteTree &serverParent, CRemoteConnection &connection, unsigned levels);
  1906. void matchServerTree(CClientRemoteTree *local, IPropertyTree &matchTree, bool allTail);
  1907. CSubscriberTable subscribers;
  1908. CSDSTransactionServer server;
  1909. ICoven &coven;
  1910. CServerRemoteTree *root;
  1911. CFitArray allNodes;
  1912. IPropertyTree &config;
  1913. MemoryBuffer incrementBuffer;
  1914. Owned<ICoalesce> coalesce;
  1915. unsigned __int64 nextExternal;
  1916. unsigned externalSizeThreshold;
  1917. CLockTable lockTable;
  1918. CNotifyHandlerTable nodeNotifyHandlers;
  1919. Owned<IThreadPool> scanNotifyPool, notifyPool;
  1920. CExternalHandlerTable externalHandlers;
  1921. CSubscriberNotifierTable subscriberNotificationTable;
  1922. Owned<CConnectionSubscriptionManager> connectionSubscriptionManager;
  1923. Owned<INodeSubscriptionManager> nodeSubscriptionManager;
  1924. bool restartOnError, externalEnvironment;
  1925. IStoreHelper *iStoreHelper;
  1926. bool doTimeComparison;
  1927. StringBuffer blockedDelta;
  1928. CBackupHandler backupHandler;
  1929. bool backupOutOfSync = false;
  1930. };
  1931. ISDSManagerServer &querySDSServer()
  1932. {
  1933. assertex(queryCoven().inCoven());
  1934. return *SDSManager;
  1935. }
  1936. /////////////////
  1937. void CConnectionSubscriptionManager::add(ISubscription *sub, SubscriptionId id)
  1938. {
  1939. MemoryBuffer mb;
  1940. mb.setBuffer(sub->queryData().length(), (void *)sub->queryData().get());
  1941. ConnectionId connId;
  1942. mb.read(connId);
  1943. Owned<CServerConnection> conn = SDSManager->getConnection(connId);
  1944. if (conn)
  1945. {
  1946. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  1947. conn->addSubscriber(* new CConnectionSubscriberContainer(sub, id));
  1948. subConnMap.setValue(id, connId);
  1949. }
  1950. // else assume connection has since been disconnected.
  1951. }
  1952. void CConnectionSubscriptionManager::remove(SubscriptionId id)
  1953. {
  1954. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  1955. ConnectionId *connId = subConnMap.getValue(id);
  1956. assertex(connId);
  1957. Owned<CServerConnection> conn = SDSManager->getConnection(*connId);
  1958. if (conn)
  1959. conn->removeSubscriber(id);
  1960. subConnMap.remove(id);
  1961. }
  1962. /////////////////
  1963. CDisableFetchChangeBlock::CDisableFetchChangeBlock(CRemoteConnection &_connection) : connection(_connection)
  1964. {
  1965. stateChanges = connection.queryStateChanges();
  1966. connection.setStateChanges(false);
  1967. lazyFetch = connection.setLazyFetch(false);
  1968. }
  1969. CDisableFetchChangeBlock::~CDisableFetchChangeBlock()
  1970. {
  1971. connection.setLazyFetch(lazyFetch);
  1972. connection.setStateChanges(stateChanges);
  1973. }
  1974. CDisableLazyFetchBlock::CDisableLazyFetchBlock(CRemoteConnection &_connection) : connection(_connection)
  1975. {
  1976. lazyFetch = connection.setLazyFetch(false);
  1977. }
  1978. CDisableLazyFetchBlock::~CDisableLazyFetchBlock()
  1979. {
  1980. connection.setLazyFetch(lazyFetch);
  1981. }
  1982. /////////////////
  1983. /// CPTStack impl.
  1984. bool CPTStack::_fill(IPropertyTree &current, const char *xpath, IPropertyTree &tail)
  1985. {
  1986. const char *nextSep = '/' == *xpath ? xpath+1 : xpath;
  1987. StringBuffer head;
  1988. const char *_nextSep = queryHead(nextSep, head);
  1989. if (!_nextSep)
  1990. {
  1991. Owned<IPropertyTreeIterator> iter = current.getElements(nextSep);
  1992. ForEach (*iter)
  1993. {
  1994. if (NotFound != iter->query().queryChildIndex(&tail))
  1995. {
  1996. append(*LINK((PTree *)&iter->query()));
  1997. append(*LINK((PTree *)&current));
  1998. return true;
  1999. }
  2000. }
  2001. }
  2002. else
  2003. {
  2004. Owned<IPropertyTreeIterator> iter = current.getElements(head.str());
  2005. ForEach (*iter)
  2006. {
  2007. if (&tail==&iter->query()) // Afaics, this should not be possible (so this test/block should really be removed)
  2008. {
  2009. ERRLOG("_fill() - tail (%s) found at intermediate level: %s", tail.queryName(), head.str());
  2010. append(*LINK((PTree *)&iter->query()));
  2011. append(*LINK((PTree *)&current));
  2012. return true;
  2013. }
  2014. else if (_fill(iter->query(), _nextSep, tail))
  2015. {
  2016. append(*LINK((PTree *)&current));
  2017. return true;
  2018. }
  2019. }
  2020. }
  2021. return false;
  2022. }
  2023. bool CPTStack::fill(IPropertyTree &root, const char *xpath, IPropertyTree &tail)
  2024. {
  2025. kill();
  2026. bool res = _fill(root, xpath, tail);
  2027. unsigned elems = ordinality();
  2028. if (elems<2) return res;
  2029. elems--;
  2030. unsigned to = 0;
  2031. while (to < elems)
  2032. swap(elems--, to++);
  2033. return res;
  2034. }
  2035. /////////////////////
  2036. StringBuffer &CPTStack::getAbsolutePath(StringBuffer &str)
  2037. {
  2038. str.append('/');
  2039. IPropertyTree *parent = &item(0);
  2040. if (ordinality()>1)
  2041. {
  2042. unsigned i = 1;
  2043. for (;;)
  2044. {
  2045. IPropertyTree *child = &item(i);
  2046. str.append(child->queryName());
  2047. str.append('[');
  2048. aindex_t pos = parent->queryChildIndex(child);
  2049. #ifdef DEBUG_HPCC_11202
  2050. if (NotFound == pos)
  2051. {
  2052. ERRLOG("Invalid CPTStack detected");
  2053. ForEachItemIn(i, *this)
  2054. {
  2055. PTree &tree = item(i);
  2056. DBGLOG("PTree path item %d = %s", i, tree.queryName());
  2057. }
  2058. PrintStackReport();
  2059. }
  2060. #endif
  2061. str.append(pos+1);
  2062. str.append(']');
  2063. if (++i >= ordinality())
  2064. break;
  2065. str.append('/');
  2066. parent = child;
  2067. }
  2068. }
  2069. return str;
  2070. }
  2071. /////////////////////
  2072. CServerConnection::~CServerConnection()
  2073. {
  2074. Owned<LinkingCriticalBlock> checkedCritBlock;
  2075. if (!RTM_MODE(mode, RTM_INTERNAL))
  2076. {
  2077. ForEachItemIn(s, subscriptions)
  2078. SDSManager->remove(subscriptions.item(s).queryId());
  2079. checkedCritBlock.setown(new LinkingCriticalBlock(SDSManager->connDestructCrit, __FILE__, __LINE__));
  2080. }
  2081. ptreePath.kill();
  2082. root.clear();
  2083. }
  2084. void CServerConnection::aborted(SessionId id)
  2085. {
  2086. LOG(MCdebugInfo(100), unknownJob, "CServerConnection: connection aborted (%" I64F "x) sessId=%" I64F "x",connectionId, id);
  2087. #if 0 // JCSMORE - think this is ok, but concerned about deadlock, change later.
  2088. Owned<CLCLockBlock> lockBlock = new CLCWriteLockBlock(((CCovenSDSManager &)manager).dataRWLock, readWriteTimeout, __FILE__, __LINE__);
  2089. SDSManager->disconnect(connectionId, false);
  2090. #else
  2091. Owned<CLCLockBlock> lockBlock = new CLCReadLockBlock(((CCovenSDSManager &)manager).dataRWLock, readWriteTimeout, __FILE__, __LINE__);
  2092. SDSManager->disconnect(connectionId, false, &lockBlock);
  2093. #endif
  2094. }
  2095. ///////////////////
  2096. enum IncCmd { None, PropDelete, AttrDelete, PropChange, PropNew, PropExisting, ChildEndMarker, PropRename, AttrChange };
  2097. CRemoteTreeBase::CRemoteTreeBase(const char *name, IPTArrayValue *value, ChildMap *children)
  2098. : SDS_PTREE(name, ipt_none, value, children)
  2099. {
  2100. serverId = 0;
  2101. }
  2102. CRemoteTreeBase::CRemoteTreeBase(MemoryBuffer &mb)
  2103. {
  2104. serverId = 0;
  2105. }
  2106. void CRemoteTreeBase::deserializeRT(MemoryBuffer &src)
  2107. {
  2108. deserializeSelfRT(src);
  2109. deserializeChildrenRT(src);
  2110. }
  2111. void CRemoteTreeBase::deserializeChildrenRT(MemoryBuffer &src)
  2112. {
  2113. StringAttr eName;
  2114. for (;;)
  2115. {
  2116. size32_t pos = src.getPos();
  2117. src.read(eName);
  2118. if (!eName.length())
  2119. break;
  2120. src.reset(pos); // reset to re-read tree name
  2121. CRemoteTreeBase *child = (CRemoteTreeBase *) create(NULL);
  2122. child->deserializeRT(src);
  2123. addPropTree(eName, child);
  2124. }
  2125. }
  2126. void CRemoteTreeBase::deserializeSelfRT(MemoryBuffer &mb)
  2127. {
  2128. deserializeSelf(mb);
  2129. assertex(!isnocase());
  2130. __int64 _serverId;
  2131. mb.read(_serverId);
  2132. if (_serverId)
  2133. setServerId(_serverId); // ignore deserializing 0 serverId (indicated new)
  2134. }
  2135. void CRemoteTreeBase::setServerId(__int64 _serverId)
  2136. {
  2137. serverId = _serverId;
  2138. }
  2139. void CRemoteTreeBase::clearChildren()
  2140. {
  2141. if (children)
  2142. {
  2143. children->Release();
  2144. children=NULL;
  2145. }
  2146. }
  2147. CRemoteTreeBase *CRemoteTreeBase::createChild(int pos, const char *childName)
  2148. {
  2149. CRemoteTreeBase *child = (CRemoteTreeBase *) create(NULL);
  2150. if (-1 == pos)
  2151. child = (CRemoteTreeBase *) addPropTree(childName, child);
  2152. else
  2153. {
  2154. unsigned e = 0;
  2155. if (children)
  2156. {
  2157. PTree *match = (PTree *) children->query(childName);
  2158. if (match)
  2159. {
  2160. IPTArrayValue *value = match->queryValue();
  2161. e = value && value->isArray()?value->elements() : 1;
  2162. }
  2163. }
  2164. if ((unsigned)pos == e)
  2165. child = (CRemoteTreeBase *) addPropTree(childName, child);
  2166. else
  2167. {
  2168. StringBuffer childPos(childName);
  2169. childPos.append("[").append(pos+1).append("]");
  2170. child = (CRemoteTreeBase *) addPropTree(childPos.str(), child);
  2171. }
  2172. }
  2173. return child;
  2174. }
  2175. ///////////
  2176. static CheckedCriticalSection suppressedOrphanUnlockCrit; // to temporarily suppress unlockall
  2177. static bool suppressedOrphanUnlock=false;
  2178. #if defined(new)
  2179. #define __old_new new
  2180. #undef new
  2181. #endif
  2182. //Do not override the packing for this class - otherwise the fixed size allocator will allocate
  2183. //misaligned objects, which can cause problems on some architectures (especially for atomic operations)
  2184. class CServerRemoteTree : public CRemoteTreeBase
  2185. {
  2186. DECL_NAMEDCOUNT;
  2187. class COrphanHandler : public ChildMap
  2188. {
  2189. public:
  2190. COrphanHandler() : ChildMap() { }
  2191. ~COrphanHandler() { _releaseAll(); }
  2192. static void setOrphans(CServerRemoteTree &tree, bool tf)
  2193. {
  2194. if (tf)
  2195. IptFlagSet(tree.flags, ipt_ext5);
  2196. else
  2197. IptFlagClr(tree.flags, ipt_ext5);
  2198. IPTArrayValue *v = tree.queryValue();
  2199. if (v && v->isArray())
  2200. {
  2201. unsigned e;
  2202. for(e=0; e<v->elements(); e++)
  2203. setOrphans(*(CServerRemoteTree *)v->queryElement(e), tf);
  2204. }
  2205. if (tree.queryChildren())
  2206. {
  2207. if (SDSManager->queryStopped()) return;
  2208. SuperHashIteratorOf<IPropertyTree> iter(*tree.queryChildren());
  2209. ForEach (iter)
  2210. setOrphans((CServerRemoteTree &)iter.query(), tf);
  2211. }
  2212. }
  2213. virtual void onAdd(void *e) // ensure memory of constructed multi value elements are no longer orphaned.
  2214. {
  2215. ChildMap::onAdd(e);
  2216. CServerRemoteTree &tree = *((CServerRemoteTree *)(IPropertyTree *)e);
  2217. setOrphans(tree, false);
  2218. }
  2219. virtual void onRemove(void *e)
  2220. {
  2221. CServerRemoteTree &tree = *((CServerRemoteTree *)(IPropertyTree *)e);
  2222. bool dounlock;
  2223. {
  2224. CHECKEDCRITICALBLOCK(suppressedOrphanUnlockCrit, fakeCritTimeout);
  2225. dounlock = !suppressedOrphanUnlock;
  2226. }
  2227. if (dounlock) // element is moving, don't orphan or unlock
  2228. {
  2229. setOrphans(tree, true);
  2230. SDSManager->unlockAll(tree.queryServerId());
  2231. }
  2232. ChildMap::onRemove(e);
  2233. }
  2234. virtual bool replace(const char *key, IPropertyTree *tree) // provides different semantics, used if element being replaced is not to be treated as deleted.
  2235. {
  2236. CHECKEDCRITICALBLOCK(suppressedOrphanUnlockCrit, fakeCritTimeout);
  2237. BoolSetBlock bblock(suppressedOrphanUnlock);
  2238. bool ret = ChildMap::replace(key, tree);
  2239. return ret;
  2240. }
  2241. virtual bool set(const char *key, IPropertyTree *tree)
  2242. {
  2243. // NB: be careful if replacing, to remove element first, because may self-destruct if lastref in middle of SuperHashTable::replace, leaving tablecount wrong.
  2244. unsigned vs = getHashFromElement(tree);
  2245. const void *fp = getFindParam(tree);
  2246. IPropertyTree *et = (IPropertyTree *)SuperHashTable::find(vs, fp);
  2247. if (et)
  2248. removeExact(et);
  2249. return ChildMap::set(key, tree);
  2250. }
  2251. };
  2252. static void clearTree(CServerRemoteTree &tree)
  2253. {
  2254. Owned<IPropertyTreeIterator> iter = tree.getElements("*");
  2255. ForEach(*iter)
  2256. {
  2257. CServerRemoteTree &child = (CServerRemoteTree &)iter->query();
  2258. clearTree(child);
  2259. }
  2260. tree.clear();
  2261. }
  2262. public:
  2263. #ifdef _POOLED_SERVER_REMOTE_TREE
  2264. void * operator new(memsize_t sz)
  2265. {
  2266. #ifdef _DEBUG
  2267. assertex(sz==sizeof(CServerRemoteTree));
  2268. #endif
  2269. return CServerRemoteTree_Allocator->alloc();
  2270. }
  2271. void operator delete( void * p )
  2272. {
  2273. CServerRemoteTree_Allocator->dealloc(p);
  2274. }
  2275. #endif
  2276. CServerRemoteTree(MemoryBuffer &mb) : CRemoteTreeBase(mb) { init(); }
  2277. CServerRemoteTree(const char *name=NULL, IPTArrayValue *value=NULL, ChildMap *children=NULL)
  2278. : CRemoteTreeBase(name, value, children) { init(); }
  2279. ~CServerRemoteTree()
  2280. {
  2281. if (hasProp(NOTIFY_ATTR))
  2282. SDSManager->handleNodeNotify(notify_delete, *this);
  2283. __int64 index = getPropInt64(EXT_ATTR);
  2284. if (index)
  2285. {
  2286. try { SDSManager->deleteExternal(index); }
  2287. catch (IException *e)
  2288. {
  2289. LOG(MCoperatorWarning, unknownJob, e, StringBuffer("Deleting external reference for ").append(queryName()).str());
  2290. e->Release();
  2291. }
  2292. }
  2293. if (SDSManager->queryStopped()) return; // don't bother building up free list that will never be used hence (could get v. big/slow)
  2294. if (isSubscribed())
  2295. SDSManager->notifyNodeDelete(*this);
  2296. CHECKEDCRITICALBLOCK(SDSManager->treeRegCrit, fakeCritTimeout);
  2297. clear(); // NB: *should* already be cleared, when tree is removed via removeTree
  2298. }
  2299. virtual bool isEquivalent(IPropertyTree *tree) const override { return (NULL != QUERYINTERFACE(tree, CServerRemoteTree)); }
  2300. PDState processData(CServerConnection &connection, IPropertyTree &changeTree, MemoryBuffer &newIds);
  2301. void init()
  2302. {
  2303. INIT_NAMEDCOUNT;
  2304. assertex(!isnocase());
  2305. CHECKEDCRITICALBLOCK(SDSManager->treeRegCrit, fakeCritTimeout);
  2306. SDSManager->queryAllNodes().addElem(this);
  2307. }
  2308. void clear()
  2309. {
  2310. // always called inside SDSManager->treeRegCrit
  2311. if (serverId)
  2312. {
  2313. SDSManager->queryAllNodes().freeElem(serverId);
  2314. serverId = 0;
  2315. }
  2316. }
  2317. virtual bool removeTree(IPropertyTree *_child)
  2318. {
  2319. if (!_child)
  2320. return false;
  2321. dbgassertex(QUERYINTERFACE(_child, CServerRemoteTree));
  2322. Linked<CServerRemoteTree> child = static_cast<CServerRemoteTree *>(_child);
  2323. if (!CRemoteTreeBase::removeTree(child))
  2324. return false;
  2325. CHECKEDCRITICALBLOCK(SDSManager->treeRegCrit, fakeCritTimeout);
  2326. clearTree(*child);
  2327. return true;
  2328. }
  2329. virtual bool isOrphaned() const { return IptFlagTst(flags, ipt_ext5); }
  2330. virtual void setServerId(__int64 _serverId)
  2331. {
  2332. if (serverId && serverId != _serverId)
  2333. WARNLOG("Unexpected - client server id mismatch in %s, id=%" I64F "x", queryName(), _serverId);
  2334. CRemoteTreeBase::setServerId(_serverId);
  2335. }
  2336. virtual CSubscriberContainerList *getSubscribers(const char *xpath, CPTStack &stack)
  2337. {
  2338. return SDSManager->getSubscribers(xpath, stack);
  2339. }
  2340. IPropertyTree *create(const char *name=NULL, IPTArrayValue *value=NULL, ChildMap *children=NULL, bool existing=false)
  2341. {
  2342. return new CServerRemoteTree(name, value, children);
  2343. }
  2344. IPropertyTree *create(MemoryBuffer &mb)
  2345. {
  2346. return new CServerRemoteTree(mb);
  2347. }
  2348. virtual void createChildMap() { children = new COrphanHandler(); }
  2349. inline bool testExternalCandidate()
  2350. {
  2351. // maybe be other cases.
  2352. return (value && value->queryValueSize() >= SDSManager->queryExternalThreshold());
  2353. }
  2354. void serializeCutOffRT(MemoryBuffer &tgt, int cutoff=-1, int depth=0, bool extValues=true)
  2355. {
  2356. serializeSelfRT(tgt, extValues);
  2357. serializeCutOffChildrenRT(tgt, cutoff, depth, extValues);
  2358. }
  2359. void serializeCutOffChildrenRT(MemoryBuffer &tgt, int cutoff=-1, int depth=0, bool extValues=true)
  2360. {
  2361. #ifndef ALWAYSLAZY_NOTUSED
  2362. if (cutoff < 0)
  2363. {
  2364. if (FETCH_ENTIRE_COND == cutoff && getPropBool("@alwaysLazy"))
  2365. {
  2366. tgt.append("");
  2367. return; // i.e. truncate here, this already serialized, lazy fetch children
  2368. }
  2369. }
  2370. else if (getPropBool("@fetchEntire"))
  2371. cutoff = FETCH_ENTIRE_COND;
  2372. #else
  2373. if (cutoff >= 0 && getPropBool("@fetchEntire"))
  2374. cutoff = FETCH_ENTIRE_COND; // NB: can change all _COND references to FETCH_ENTIRE if not using alwaysFetch anymore
  2375. #endif
  2376. if (cutoff < 0 || depth<cutoff)
  2377. {
  2378. IPropertyTreeIterator *iter = getElements("*");
  2379. iter->first();
  2380. while (iter->isValid())
  2381. {
  2382. IPropertyTree *_child = &iter->query();
  2383. CServerRemoteTree *child = QUERYINTERFACE(_child, CServerRemoteTree); assertex(child);
  2384. child->serializeCutOffRT(tgt, cutoff, depth+1, extValues);
  2385. iter->next();
  2386. }
  2387. iter->Release();
  2388. }
  2389. tgt.append(""); // element terminator. i.e. blank child name.
  2390. }
  2391. void serializeSelfRT(MemoryBuffer &mb, bool extValues)
  2392. {
  2393. __int64 index = getPropInt64(EXT_ATTR);
  2394. if (index)
  2395. {
  2396. SDSManager->serializeExternal(index, *this, mb, extValues);
  2397. mb.append(serverId);
  2398. }
  2399. else
  2400. {
  2401. serializeSelf(mb);
  2402. mb.append(serverId);
  2403. }
  2404. byte STIInfo = 0;
  2405. if (children && children->count())
  2406. STIInfo += STI_HaveChildren;
  2407. if (index)
  2408. STIInfo += STI_External;
  2409. mb.append(STIInfo);
  2410. }
  2411. virtual void deserializeSelfRT(MemoryBuffer &src)
  2412. {
  2413. CRemoteTreeBase::deserializeSelfRT(src);
  2414. assertex(!isnocase());
  2415. byte STIInfo;
  2416. src.read(STIInfo);
  2417. }
  2418. virtual void removingElement(IPropertyTree *tree, unsigned pos)
  2419. {
  2420. COrphanHandler::setOrphans(*(CServerRemoteTree *)tree, true);
  2421. CRemoteTreeBase::removingElement(tree, pos);
  2422. }
  2423. virtual bool isCompressed(const char *xpath=NULL) const
  2424. {
  2425. if (isAttribute(xpath)) return false;
  2426. if (CRemoteTreeBase::isCompressed(xpath)) return true;
  2427. if (SDSManager->ignoreExternals) return false;
  2428. CServerRemoteTree *child = (CServerRemoteTree *)queryPropTree(xpath);
  2429. return child->hasProp(EXT_ATTR);
  2430. }
  2431. bool getProp(const char *xpath, StringBuffer &ret) const
  2432. {
  2433. if (xpath)
  2434. return CRemoteTreeBase::getProp(xpath, ret);
  2435. if (SDSManager->ignoreExternals)
  2436. return CRemoteTreeBase::getProp(xpath, ret);
  2437. CHECKEDCRITICALBLOCK(extCrit, fakeCritTimeout);
  2438. __int64 index = getPropInt64(EXT_ATTR);
  2439. if (!index)
  2440. return CRemoteTreeBase::getProp(xpath, ret);
  2441. MemoryBuffer mbv, mb;
  2442. SDSManager->getExternalValue(index, mbv);
  2443. CPTValue v(mbv);
  2444. v.getValue(mb, true);
  2445. unsigned len = mb.length();
  2446. char *mem = (char *)mb.detach();
  2447. mem[len-1] = '\0';
  2448. ret.setBuffer(len, mem, len-1);
  2449. return true;
  2450. }
  2451. void setSubscribed(bool tf)
  2452. {
  2453. if (tf)
  2454. IptFlagSet(flags, ipt_ext4);
  2455. else
  2456. IptFlagClr(flags, ipt_ext4);
  2457. }
  2458. inline bool isSubscribed() const
  2459. {
  2460. return IptFlagTst(flags, ipt_ext4);
  2461. }
  2462. private:
  2463. PDState processData(IPropertyTree &changeTree, Owned<CBranchChange> &parentBranchChange, MemoryBuffer &newIds);
  2464. PDState checkChange(IPropertyTree &tree, CBranchChange &parentBranchChange);
  2465. friend class COrphanHandler;
  2466. };
  2467. class CNodeSubscriberContainer : public CSubscriberContainerBase
  2468. {
  2469. StringAttr xpath;
  2470. bool sendValue;
  2471. ICopyArrayOf<CServerRemoteTree> nodes; // never linked, node must signal the unsubscription and removal of subscriber and these references
  2472. public:
  2473. CNodeSubscriberContainer(ISubscription *subscriber, SubscriptionId id, bool _sendValue, const char *_xpath)
  2474. : CSubscriberContainerBase(subscriber, id), sendValue(_sendValue), xpath(_xpath)
  2475. {
  2476. }
  2477. bool querySendValue() const { return sendValue; }
  2478. void add(CServerRemoteTree &node) { nodes.append(node); }
  2479. ICopyArrayOf<CServerRemoteTree> &queryNodes() { return nodes; }
  2480. MemoryBuffer &getInfo(MemoryBuffer &out) const
  2481. {
  2482. out.append(id).append(xpath).append(nodes.ordinality());
  2483. return out;
  2484. }
  2485. };
  2486. class CNodeSubscriptionManager : implements INodeSubscriptionManager, public CSimpleInterface
  2487. {
  2488. public:
  2489. class CNodeSubscriberContainerList : public CSimpleInterfaceOf<IInterface>
  2490. {
  2491. CServerRemoteTree *node;
  2492. ICopyArrayOf<CNodeSubscriberContainer> subscribers;
  2493. public:
  2494. CNodeSubscriberContainerList(CServerRemoteTree *_node) : node(_node)
  2495. {
  2496. }
  2497. const void *queryFindParam() const { return &node; }
  2498. ICopyArrayOf<CNodeSubscriberContainer> &querySubscribers() { return subscribers; }
  2499. void getSubscribers(IArrayOf<CNodeSubscriberContainer> &linkedSubscribers)
  2500. {
  2501. ForEachItemIn(s, subscribers)
  2502. linkedSubscribers.append(*LINK(&subscribers.item(s)));
  2503. }
  2504. void add(CNodeSubscriberContainer &subscriber) { subscribers.append(subscriber); }
  2505. };
  2506. CCovenSDSManager &owner;
  2507. OwningSimpleHashTableOf<CNodeSubscriberContainer, SubscriptionId> subscribersById;
  2508. OwningSimpleHashTableOf<CNodeSubscriberContainerList, CServerRemoteTree *> subscriberListByNode;
  2509. mutable CriticalSection subscriberListCrit;
  2510. void _notify(CServerRemoteTree *node, PDState state, IArrayOf<CNodeSubscriberContainer> &subscribers)
  2511. {
  2512. MemoryBuffer sendValueNotifyData;
  2513. int lastSendValue = -1;
  2514. while (subscribers.ordinality())
  2515. {
  2516. Owned<CNodeSubscriberContainer> subscriber = &subscribers.popGet();
  2517. if (subscriber->querySendValue())
  2518. {
  2519. if (1 != lastSendValue) // overkill unless many subscribers to same node
  2520. {
  2521. MemoryBuffer mb;
  2522. node->getPropBin(NULL, mb);
  2523. buildNotifyData(sendValueNotifyData.clear(), state, NULL, &mb);
  2524. lastSendValue = 1;
  2525. }
  2526. SDSManager->handleNotify(subscriber.getClear(), sendValueNotifyData);
  2527. }
  2528. else
  2529. {
  2530. if (0 != lastSendValue) // overkill unless many subscribers to same node
  2531. {
  2532. buildNotifyData(sendValueNotifyData.clear(), state, NULL, NULL);
  2533. lastSendValue = 0;
  2534. }
  2535. SDSManager->handleNotify(subscriber.getClear(), sendValueNotifyData);
  2536. }
  2537. }
  2538. }
  2539. void _notify(CServerRemoteTree *node, PDState state)
  2540. {
  2541. CNodeSubscriberContainerList *subscriberList = subscriberListByNode.find(node);
  2542. assertex(subscriberList);
  2543. IArrayOf<CNodeSubscriberContainer> subscribers;
  2544. subscriberList->getSubscribers(subscribers);
  2545. _notify(node, state, subscribers);
  2546. }
  2547. void _removeNode(CServerRemoteTree *node, SubscriptionId id)
  2548. {
  2549. CNodeSubscriberContainerList *subscriberList = subscriberListByNode.find(node);
  2550. assertex(subscriberList);
  2551. ICopyArrayOf<CNodeSubscriberContainer> &subscribers = subscriberList->querySubscribers();
  2552. ForEachItemInRev(s, subscribers)
  2553. {
  2554. CNodeSubscriberContainer &subscriber = subscribers.item(s);
  2555. if (0 == id) // remove all associated subscribers (node being deleted)
  2556. {
  2557. ICopyArrayOf<CServerRemoteTree> &nodes = subscriber.queryNodes();
  2558. verifyex(nodes.zap(*node));
  2559. subscribers.remove(s);
  2560. if (0 == nodes.ordinality()) // IOW this was the last node this subscriber was associated with
  2561. subscribersById.removeExact(&subscriber);
  2562. }
  2563. else if (subscriber.queryId() == id)
  2564. subscribers.remove(s);
  2565. }
  2566. if (0 == subscribers.ordinality())
  2567. {
  2568. node->setSubscribed(false);
  2569. subscriberListByNode.removeExact(subscriberList);
  2570. }
  2571. }
  2572. public:
  2573. IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
  2574. CNodeSubscriptionManager(CCovenSDSManager &_owner) : owner(_owner) { }
  2575. void notify(CServerRemoteTree &node, PDState state)
  2576. {
  2577. // shouldn't be here, unless node is in subscribers table
  2578. CriticalBlock b(subscriberListCrit);
  2579. _notify(&node, state);
  2580. }
  2581. void notifyDelete(CServerRemoteTree *node)
  2582. {
  2583. // shouldn't be here, unless node is in subscribers table
  2584. CriticalBlock b(subscriberListCrit);
  2585. /* Need to be careful not to release subscribers here (on this thread)
  2586. * 1) gather subscribers(linked)
  2587. * 2) remove nodes and lists, so no longer in use by SDS
  2588. * 3) Hand ownership over to notification mechanism
  2589. *
  2590. * Subscribers will be released when notification is done with them.
  2591. */
  2592. CNodeSubscriberContainerList *subscriberList = subscriberListByNode.find(node);
  2593. assertex(subscriberList);
  2594. IArrayOf<CNodeSubscriberContainer> linkedSubscribers;
  2595. subscriberList->getSubscribers(linkedSubscribers);
  2596. _removeNode(node, 0);
  2597. // NB: Notification will take ownership of subscribers being notified.
  2598. _notify(node, PDS_Deleted, linkedSubscribers);
  2599. }
  2600. // ISubscriptionManager impl.
  2601. virtual void add(ISubscription *sub, SubscriptionId id)
  2602. {
  2603. CHECKEDDALIREADLOCKBLOCK(owner.dataRWLock, readWriteTimeout);
  2604. CHECKEDCRITICALBLOCK(owner.treeRegCrit, fakeCritTimeout);
  2605. CriticalBlock b(subscriberListCrit);
  2606. /* calls back out to owner to scan for match, so that SDSManager can protect root/treereg.
  2607. * It calls back (associateSubscriber) in this class to add subscribers based on matches.
  2608. */
  2609. owner.addNodeSubscriber(sub, id);
  2610. }
  2611. virtual void remove(SubscriptionId id)
  2612. {
  2613. /* important to ensure have exclusive data lock on removal, ahead of subscriber lock
  2614. * as can get notifications whilst holding data lock, e.g. notifyDelete on node destruction.
  2615. */
  2616. CHECKEDDALIREADLOCKBLOCK(owner.dataRWLock, readWriteTimeout);
  2617. CHECKEDCRITICALBLOCK(owner.treeRegCrit, fakeCritTimeout);
  2618. CriticalBlock b(subscriberListCrit);
  2619. /* calls back out to owner to protect root/treereg.
  2620. * It calls back into removeSubscriberAssociation.
  2621. */
  2622. owner.removeNodeSubscriber(id);
  2623. }
  2624. virtual void removeSubscriberAssociation(SubscriptionId id) // Always called back from within remove() above.
  2625. {
  2626. CNodeSubscriberContainer *subscriber = subscribersById.find(id);
  2627. if (!subscriber)
  2628. return; // may not exist if removed already
  2629. ICopyArrayOf<CServerRemoteTree> &nodes = subscriber->queryNodes();
  2630. ForEachItemIn(n, nodes)
  2631. {
  2632. CServerRemoteTree &node = nodes.item(n);
  2633. _removeNode(&node, id);
  2634. }
  2635. verifyex(subscribersById.removeExact(subscriber));
  2636. }
  2637. void associateSubscriber(CNodeSubscriberContainer &subscriber) // Always called back from within add() above.
  2638. {
  2639. /* caller has established there are matches and added them to 'subscriber'
  2640. * add to HT's
  2641. */
  2642. verifyex(subscribersById.add(*LINK(&subscriber)));
  2643. ICopyArrayOf<CServerRemoteTree> &nodes = subscriber.queryNodes();
  2644. ForEachItemIn(n, nodes)
  2645. {
  2646. CServerRemoteTree *node = &nodes.item(n);
  2647. CNodeSubscriberContainerList *subscriberList = subscriberListByNode.find(node);
  2648. if (!subscriberList)
  2649. {
  2650. subscriberList = new CNodeSubscriberContainerList(node);
  2651. verifyex(subscriberListByNode.add(* subscriberList));
  2652. }
  2653. subscriberList->add(subscriber);
  2654. }
  2655. }
  2656. MemoryBuffer &collectSubscribers(MemoryBuffer &out) const
  2657. {
  2658. /* important to ensure have exclusive data lock on removal, ahead of subscriber lock
  2659. * as can get notifications whilst holding data lock, e.g. notifyDelete on node destruction.
  2660. */
  2661. CHECKEDDALIREADLOCKBLOCK(owner.dataRWLock, readWriteTimeout);
  2662. CHECKEDCRITICALBLOCK(owner.treeRegCrit, fakeCritTimeout);
  2663. CriticalBlock b(subscriberListCrit);
  2664. out.append(subscribersById.count());
  2665. SuperHashIteratorOf<CNodeSubscriberContainer> sdsNodeIter(subscribersById);
  2666. ForEach(sdsNodeIter)
  2667. sdsNodeIter.query().getInfo(out);
  2668. return out;
  2669. }
  2670. };
  2671. #if defined(_WIN32) && defined(__old_new)
  2672. #define new __old_new
  2673. #endif
  2674. void populateWithServerIds(IPropertyTree *matchParent, CRemoteTreeBase *parent)
  2675. {
  2676. matchParent->setPropInt64("@serverId", parent->queryServerId());
  2677. Owned<IPropertyTreeIterator> iter = matchParent->getElements("*");
  2678. ForEach (*iter)
  2679. {
  2680. IPropertyTree *matchChild = &iter->query();
  2681. StringBuffer path(matchChild->queryName());
  2682. path.append("[").append(matchChild->queryProp("@pos")).append("]");
  2683. CRemoteTreeBase *child = (CRemoteTreeBase *)parent->queryPropTree(path.str());
  2684. assertex(child);
  2685. populateWithServerIds(matchChild, child);
  2686. }
  2687. }
  2688. IPropertyTree *createServerTree(const char *tag=NULL)
  2689. {
  2690. return new CServerRemoteTree(tag);
  2691. }
  2692. // JCSMORE - these should be error conditions, for consistency with previous release not so for now.
  2693. #define consistencyCheck(TEXT, IDTREE, LOCALTREE, PATH, PARENTNAME, ID) \
  2694. if (!IDTREE) \
  2695. { \
  2696. StringBuffer s(TEXT": Consistency check failure, id'd tree not found: "); \
  2697. s.append(PATH).append(", id=").append(ID); \
  2698. LOG(MCoperatorWarning, unknownJob, s.str()); \
  2699. } \
  2700. else if (!LOCALTREE) \
  2701. { \
  2702. StringBuffer s(TEXT": Consistency check failure, positional property specification: "); \
  2703. s.append(PATH).append(", in client update not found in parent tree: ").append(PARENTNAME); \
  2704. LOG(MCoperatorWarning, unknownJob, s.str()); \
  2705. } \
  2706. else if (IDTREE != LOCALTREE) \
  2707. { \
  2708. StringBuffer s(TEXT": Consistency check failure, positional property specification does not match id'd tree, prop=");\
  2709. s.append(PATH); \
  2710. LOG(MCoperatorWarning, unknownJob, s.str()); \
  2711. }
  2712. PDState CServerRemoteTree::checkChange(IPropertyTree &changeTree, CBranchChange &parentBranchChange)
  2713. {
  2714. PDState res = PDS_None;
  2715. bool checkExternal = false;
  2716. Owned<IPropertyTreeIterator> iter = changeTree.getElements("*");
  2717. ICopyArrayOf<IPropertyTree> toremove;
  2718. ForEach(*iter)
  2719. {
  2720. IPropertyTree &e = iter->query();
  2721. const char *name = e.queryName();
  2722. switch (name[0])
  2723. {
  2724. case 'R':
  2725. {
  2726. IPropertyTree *idTree = SDSManager->queryRegisteredTree(e.getPropInt64("@id"));
  2727. if (!idTree)
  2728. throw MakeSDSException(SDSExcpt_OrphanedNode, "renaming %s to %s", e.queryProp("@from"), e.queryProp("@to"));
  2729. #ifdef SIBLING_MOVEMENT_CHECK
  2730. StringBuffer localTreePath(e.queryProp("@from"));
  2731. localTreePath.append('[').append(e.getPropInt("@pos")).append(']');
  2732. IPropertyTree *localTree = queryPropTree(localTreePath.str());
  2733. consistencyCheck("RENAME", idTree, localTree, localTreePath.str(), queryName(), e.getPropInt64("@id"));
  2734. #endif
  2735. int pos = findChild(idTree);
  2736. if (renameTree(idTree, e.queryProp("@to")))
  2737. {
  2738. e.setPropInt("@pos", pos+1);
  2739. mergePDState(res, PDS_Structure);
  2740. PDState _res = res;
  2741. mergePDState(_res, PDS_Renames);
  2742. Owned<CBranchChange> childChange = new CBranchChange(*(CRemoteTreeBase *)idTree);
  2743. childChange->noteChange(_res, _res);
  2744. childChange->Link();
  2745. parentBranchChange.addChildBranch(*childChange);
  2746. }
  2747. else
  2748. {
  2749. toremove.append(e);
  2750. continue;
  2751. }
  2752. break;
  2753. }
  2754. case 'D':
  2755. {
  2756. IPropertyTree *idTree = SDSManager->queryRegisteredTree(e.getPropInt64("@id"));
  2757. if (!idTree)
  2758. {
  2759. toremove.append(e);
  2760. continue;
  2761. }
  2762. #ifdef SIBLING_MOVEMENT_CHECK
  2763. StringBuffer localTreePath(e.queryProp("@name"));
  2764. localTreePath.append('[').append(e.getPropInt("@pos")).append(']');
  2765. IPropertyTree *localTree = queryPropTree(localTreePath.str());
  2766. consistencyCheck("DELETE", idTree, localTree, localTreePath.str(), queryName(), e.getPropInt64("@id"));
  2767. #endif
  2768. int pos = findChild(idTree);
  2769. if (NotFound == pos)
  2770. {
  2771. toremove.append(e);
  2772. continue;
  2773. }
  2774. e.setPropInt("@pos", pos+1);
  2775. Owned<CBranchChange> childChange = new CBranchChange(*(CRemoteTreeBase *)idTree);
  2776. if (!removeTree(idTree))
  2777. throw MakeSDSException(-1, "::checkChange - Failed to remove child(%s) from parent(%s) at %s(%d)", idTree->queryName(), queryName(), __FILE__, __LINE__);
  2778. mergePDState(res, PDS_Structure);
  2779. PDState _res = res;
  2780. mergePDState(_res, PDS_Deleted);
  2781. childChange->noteChange(_res, _res);
  2782. childChange->Link();
  2783. parentBranchChange.addChildBranch(*childChange);
  2784. break;
  2785. }
  2786. case 'A':
  2787. {
  2788. switch (name[1])
  2789. {
  2790. case 'D':
  2791. {
  2792. Owned<IAttributeIterator> iter = e.getAttributes();
  2793. ForEach(*iter)
  2794. {
  2795. if (removeAttribute(iter->queryName()))
  2796. mergePDState(res, PDS_Data);
  2797. }
  2798. break;
  2799. }
  2800. case 'C':
  2801. {
  2802. Owned<IAttributeIterator> iter = e.getAttributes();
  2803. ForEach(*iter)
  2804. setProp(iter->queryName(), iter->queryValue());
  2805. mergePDState(res, PDS_Data);
  2806. break;
  2807. }
  2808. default:
  2809. throwUnexpected();
  2810. }
  2811. break;
  2812. }
  2813. case 'T':
  2814. break;
  2815. }
  2816. }
  2817. ForEachItemIn(tr, toremove)
  2818. changeTree.removeTree(&toremove.item(tr));
  2819. if (changeTree.getPropBool("@localValue"))
  2820. {
  2821. bool binary=changeTree.isBinary(NULL);
  2822. IPTArrayValue *v = ((PTree &)changeTree).queryValue();
  2823. setValue(v?new CPTValue(v->queryValueRawSize(), v->queryValueRaw(), binary, true, v->isCompressed()):NULL, binary);
  2824. if (changeTree.getPropBool("@new"))
  2825. mergePDState(res, PDS_New);
  2826. mergePDState(res, PDS_Data);
  2827. checkExternal = true;
  2828. }
  2829. else if (changeTree.getPropBool("@appendValue"))
  2830. {
  2831. bool binary=changeTree.isBinary(NULL);
  2832. if (binary != isBinary(NULL))
  2833. throw MakeSDSException(-1, "Error attempting to append binary and non-binary data together, in node: %s", queryName());
  2834. __int64 index = getPropInt64(EXT_ATTR);
  2835. MemoryBuffer mb;
  2836. if (index)
  2837. {
  2838. MemoryBuffer mbv;
  2839. SDSManager->getExternalValue(index, mbv);
  2840. CPTValue v(mbv);
  2841. v.getValue(mb, binary);
  2842. }
  2843. else
  2844. getPropBin(NULL, mb);
  2845. changeTree.getPropBin(NULL, mb);
  2846. if (binary)
  2847. setPropBin(NULL, mb.length(), mb.toByteArray());
  2848. else
  2849. {
  2850. mb.append('\0');
  2851. setProp(NULL, (const char *)mb.toByteArray());
  2852. }
  2853. mergePDState(res, PDS_Data);
  2854. checkExternal = true;
  2855. }
  2856. if (checkExternal)
  2857. {
  2858. __int64 index = getPropInt64(EXT_ATTR);
  2859. if (index)
  2860. {
  2861. bool r = false;
  2862. if (!testExternalCandidate())
  2863. {
  2864. SDSManager->deleteExternal(index); // i.e. no longer e.g. now less than threshold.
  2865. r = removeProp(EXT_ATTR);
  2866. }
  2867. else
  2868. SDSManager->writeExternal(*this, false, index);
  2869. if (r)
  2870. {
  2871. IPropertyTree *t = changeTree.queryPropTree(ATTRDELETE_TAG);
  2872. if (!t)
  2873. t = changeTree.addPropTree(ATTRDELETE_TAG, createPTree());
  2874. t->addProp(EXT_ATTR, "");
  2875. }
  2876. else
  2877. changeTree.setProp(NULL, (const char *)NULL);
  2878. }
  2879. else if (testExternalCandidate())
  2880. {
  2881. try
  2882. {
  2883. SDSManager->writeExternal(*this);
  2884. IPropertyTree *t = changeTree.queryPropTree(ATTRCHANGE_TAG);
  2885. if (!t)
  2886. t = changeTree.addPropTree(ATTRCHANGE_TAG, createPTree());
  2887. changeTree.setProp(NULL, (const char *)NULL);
  2888. t->setProp(EXT_ATTR, queryProp(EXT_ATTR));
  2889. }
  2890. catch (IException *)
  2891. {
  2892. setProp(NULL, NULL); // in the event of failure during externalization, lose the value
  2893. throw;
  2894. }
  2895. }
  2896. }
  2897. return res;
  2898. }
  2899. PDState CServerRemoteTree::processData(CServerConnection &connection, IPropertyTree &changeTree, MemoryBuffer &newIds)
  2900. {
  2901. Owned<CBranchChange> top;
  2902. PDState res = processData(changeTree, top, newIds);
  2903. changeTree.removeProp("@name");
  2904. if (res)
  2905. SDSManager->writeTransactions++;
  2906. // return asap from here, don't even wait for pool threads to queue, can take time.
  2907. if (res && !RTM_MODE(connection.queryMode(), RTM_INTERNAL))
  2908. {
  2909. CPTStack stack = connection.queryPTreePath();
  2910. if (connection.queryRoot() == (IPropertyTree *)SDSManager->queryRoot())
  2911. stack.pop();
  2912. connection.notify();
  2913. SDSManager->startNotification(changeTree, stack, *top);
  2914. }
  2915. return res;
  2916. }
  2917. PDState CServerRemoteTree::processData(IPropertyTree &changeTree, Owned<CBranchChange> &parentBranchChange, MemoryBuffer &newIds)
  2918. {
  2919. Owned<CBranchChange> branchChange = new CBranchChange(*this);
  2920. PDState localChange, res;
  2921. localChange = res = checkChange(changeTree, *branchChange);
  2922. Owned<IPropertyTreeIterator> iter = changeTree.getElements(RESERVED_CHANGE_NODE);
  2923. if (iter->first())
  2924. {
  2925. bool levelNotified = false;
  2926. do
  2927. {
  2928. IPropertyTree &childChanges = iter->query();
  2929. CServerRemoteTree *child;
  2930. StringAttr childName;
  2931. if (childChanges.getPropBool("@new"))
  2932. {
  2933. child = (CServerRemoteTree *)createChild(childChanges.getPropInt("@pos", -1), childChanges.queryProp("@name"));
  2934. newIds.append(child->queryServerId());
  2935. mergePDState(localChange, PDS_Added);
  2936. mergePDState(res, PDS_Added);
  2937. }
  2938. else
  2939. {
  2940. child = (CServerRemoteTree *) SDSManager->queryRegisteredTree(childChanges.getPropInt64("@id"));
  2941. #ifdef SIBLING_MOVEMENT_CHECK
  2942. StringBuffer localTreePath(childChanges.queryProp("@name"));
  2943. localTreePath.append('[').append(childChanges.getPropInt("@pos")).append(']');
  2944. CRemoteTreeBase *localTree = (CRemoteTreeBase *) queryPropTree(localTreePath.str());
  2945. consistencyCheck("PROP UPDATE", child, localTree, localTreePath.str(), queryName(), childChanges.getPropInt64("@id"))
  2946. #endif
  2947. if (NULL == child)
  2948. throw MakeSDSException(SDSExcpt_ClientCacheDirty, "child(%s) not found in parent(%s) at %s(%d)", childChanges.queryProp("@name"), queryName(), __FILE__, __LINE__);
  2949. int pos = findChild(child);
  2950. if (NotFound == pos)
  2951. throw MakeSDSException(SDSExcpt_ClientCacheDirty, "child(%s) not found in parent(%s) at %s(%d)", child->queryName(), queryName(), __FILE__, __LINE__);
  2952. childChanges.setPropInt("@pos", pos+1);
  2953. }
  2954. if (!levelNotified)
  2955. {
  2956. levelNotified = true;
  2957. branchChange->noteChange(localChange, res);
  2958. }
  2959. mergePDState(res, child->processData(childChanges, branchChange, newIds));
  2960. }
  2961. while (iter->next());
  2962. }
  2963. else
  2964. branchChange->noteChange(localChange, res);
  2965. if ((localChange != PDS_None) && isSubscribed())
  2966. SDSManager->notifyNode(*this, localChange);
  2967. if (!parentBranchChange.get())
  2968. parentBranchChange.setown(branchChange.getClear());
  2969. else
  2970. parentBranchChange->addChildBranch(*branchChange.getClear());
  2971. return res;
  2972. }
  2973. /////////////////
  2974. class CPendingLockBlock
  2975. {
  2976. CLock &lock;
  2977. public:
  2978. CPendingLockBlock(CLock &_lock);
  2979. ~CPendingLockBlock();
  2980. };
  2981. typedef Int64Array IdPath;
  2982. #define LOCKSESSCHECK (1000*60*5)
  2983. class CLock : implements IInterface, public CInterface
  2984. {
  2985. DECL_NAMEDCOUNT;
  2986. CLockTable &table;
  2987. unsigned sub, readLocks, holdLocks, pending, waiting;
  2988. IdPath idPath;
  2989. ConnectionInfoMap connectionInfo;
  2990. mutable CheckedCriticalSection crit;
  2991. Semaphore sem;
  2992. StringAttr xpath;
  2993. __int64 treeId;
  2994. bool exclusive;
  2995. Linked<CServerRemoteTree> parent, child;
  2996. #ifdef _DEBUG
  2997. DebugInfo debugInfo;
  2998. #endif
  2999. bool validateConnectionSessions()
  3000. {
  3001. PROGLOG("validateConnectionSessions");
  3002. bool ret = false;
  3003. try
  3004. {
  3005. IArrayOf<IMapping> entries;
  3006. {
  3007. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  3008. HashIterator iter(connectionInfo);
  3009. ForEach (iter)
  3010. entries.append(*LINK(&iter.query()));
  3011. }
  3012. ForEachItemIn(e, entries)
  3013. {
  3014. IMapping &imap = entries.item(e);
  3015. LockData *lD = connectionInfo.mapToValue(&imap);
  3016. Owned<INode> node = querySessionManager().getProcessSessionNode(lD->sessId);
  3017. if (node)
  3018. {
  3019. SessionId nodeSessId = querySessionManager().lookupProcessSession(node);
  3020. if (nodeSessId)
  3021. {
  3022. if (lD->sessId != nodeSessId)
  3023. {
  3024. StringBuffer out("Removing stale connection session [");
  3025. out.appendf("%" I64F "x], connectionId [%" I64F "x]", lD->sessId, * ((ConnectionId *) imap.getKey()));
  3026. out.append(" xpath [").append(xpath).append("]");
  3027. PROGLOG("%s", out.str());
  3028. querySessionManager().stopSession(lD->sessId, true);
  3029. ret = true;
  3030. }
  3031. else
  3032. {
  3033. StringBuffer nodeStr;
  3034. node->endpoint().getUrlStr(nodeStr);
  3035. PROGLOG("Validating connection to %s", nodeStr.str());
  3036. if (!queryWorldCommunicator().verifyConnection(node, LOCKSESSCHECK))
  3037. {
  3038. StringBuffer out("Terminating connection session to ");
  3039. out.append(nodeStr);
  3040. out.append(" [");
  3041. out.appendf("%" I64F "x], connectionId [%" I64F "x]", lD->sessId, * ((ConnectionId *) imap.getKey()));
  3042. out.append(" xpath [").append(xpath).append("]");
  3043. PROGLOG("%s", out.str());
  3044. queryCoven().disconnect(node);
  3045. ret = true;
  3046. }
  3047. }
  3048. }
  3049. }
  3050. }
  3051. }
  3052. catch (IException *e)
  3053. {
  3054. EXCLOG(e, "validateConnectionSessions");
  3055. e->Release();
  3056. }
  3057. PROGLOG("validateConnectionSessions done");
  3058. return ret;
  3059. }
  3060. LockStatus doLock(unsigned mode, unsigned timeout, ConnectionId id, SessionId sessionId, IUnlockCallback &callback, bool change=false)
  3061. {
  3062. if (INFINITE == timeout)
  3063. {
  3064. for (;;)
  3065. {
  3066. if (!SDSManager->queryConnection(id))
  3067. return LockFailed;
  3068. LockStatus lockStatus = tryLock(mode, id, sessionId, change);
  3069. if (lockStatus == LockSucceeded || lockStatus == LockHeld)
  3070. return lockStatus;
  3071. else
  3072. {
  3073. bool timedout = false;
  3074. waiting++;
  3075. {
  3076. CHECKEDCRITICALUNBLOCK(crit, fakeCritTimeout);
  3077. callback.unblock();
  3078. timedout = !sem.wait(LOCKSESSCHECK);
  3079. callback.block();
  3080. }
  3081. if (timedout)
  3082. {
  3083. if (!sem.wait(0))
  3084. {
  3085. waiting--;
  3086. StringBuffer s("Infinite timeout lock still waiting: ");
  3087. toString(s);
  3088. PROGLOG("%s", s.str());
  3089. }
  3090. {
  3091. CHECKEDCRITICALUNBLOCK(crit, fakeCritTimeout);
  3092. callback.unblock();
  3093. validateConnectionSessions();
  3094. callback.block();
  3095. }
  3096. }
  3097. }
  3098. }
  3099. }
  3100. else
  3101. {
  3102. CTimeMon tm(timeout);
  3103. for (;;)
  3104. {
  3105. if (!SDSManager->queryConnection(id))
  3106. return LockFailed;
  3107. LockStatus lockStatus = tryLock(mode, id, sessionId, change);
  3108. if (lockStatus == LockSucceeded || lockStatus == LockHeld)
  3109. return lockStatus;
  3110. else
  3111. {
  3112. bool timedout = false;
  3113. waiting++;
  3114. {
  3115. CHECKEDCRITICALUNBLOCK(crit, fakeCritTimeout);
  3116. callback.unblock();
  3117. unsigned remaining;
  3118. if (tm.timedout(&remaining) || !sem.wait(remaining>LOCKSESSCHECK?LOCKSESSCHECK:remaining))
  3119. timedout = true;
  3120. callback.block();
  3121. }
  3122. if (timedout) {
  3123. if (!sem.wait(0))
  3124. waiting--; //// only dec waiting if waiting wasn't signalled.
  3125. bool disconnects;
  3126. {
  3127. CHECKEDCRITICALUNBLOCK(crit, fakeCritTimeout);
  3128. callback.unblock();
  3129. disconnects = validateConnectionSessions();
  3130. callback.block();
  3131. }
  3132. if (tm.timedout())
  3133. {
  3134. if (disconnects) // if some sessions disconnected, one final try
  3135. {
  3136. if (!SDSManager->queryConnection(id))
  3137. return LockFailed;
  3138. lockStatus = tryLock(mode, id, sessionId, change);
  3139. if (lockStatus == LockSucceeded || lockStatus == LockHeld)
  3140. return lockStatus;
  3141. }
  3142. break;
  3143. }
  3144. }
  3145. // have to very careful here, have regained crit locks but have since timed out
  3146. // therefore before gaining crits after signal (this lock was unlocked)
  3147. // other thread can grab lock at this time, but this thread can't cause others to increase 'waiting' at this time.
  3148. // and not after crit locks regained.
  3149. if (tm.timedout())
  3150. break;
  3151. }
  3152. }
  3153. }
  3154. return LockTimedOut;
  3155. }
  3156. LockStatus tryLock(unsigned mode, ConnectionId id, SessionId sessId, bool changingMode=false)
  3157. {
  3158. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  3159. LockData *existingLd = NULL;
  3160. bool hadReadLock = false;
  3161. bool hadHoldLock = false;
  3162. if (changingMode)
  3163. {
  3164. existingLd = connectionInfo.getValue(id);
  3165. if (existingLd)
  3166. {
  3167. if ((existingLd->mode & RTM_LOCKBASIC_MASK) == (mode & RTM_LOCKBASIC_MASK))
  3168. return LockSucceeded; // nothing to do
  3169. // record and unlock existing state
  3170. switch (existingLd->mode & RTM_LOCKBASIC_MASK)
  3171. {
  3172. case (RTM_LOCK_HOLD+RTM_LOCK_READ):
  3173. holdLocks--;
  3174. hadHoldLock = true;
  3175. // fall into...
  3176. case RTM_LOCK_READ:
  3177. readLocks--;
  3178. hadReadLock = true;
  3179. break;
  3180. case (RTM_LOCK_WRITE+RTM_LOCK_READ):
  3181. case RTM_LOCK_WRITE:
  3182. exclusive = false;
  3183. // change will succeed
  3184. break;
  3185. case 0: // no locking
  3186. break;
  3187. default:
  3188. throwUnexpected();
  3189. }
  3190. }
  3191. else
  3192. changingMode = false; // nothing to restore in event of no change
  3193. }
  3194. switch (mode & RTM_LOCKBASIC_MASK)
  3195. {
  3196. case 0:
  3197. {
  3198. if (changingMode)
  3199. break;
  3200. return LockSucceeded;
  3201. }
  3202. case (RTM_LOCK_READ+RTM_LOCK_HOLD):
  3203. case RTM_LOCK_READ: // cannot fail if changingMode=true (exclusive will have been unlocked)
  3204. if (exclusive)
  3205. return LockFailed;
  3206. readLocks++;
  3207. if (mode & RTM_LOCK_HOLD)
  3208. holdLocks++;
  3209. break;
  3210. case (RTM_LOCK_WRITE+RTM_LOCK_READ):
  3211. case RTM_LOCK_WRITE:
  3212. if (exclusive || readLocks || holdLocks)
  3213. {
  3214. if (changingMode)
  3215. {
  3216. // only an unlocked read lock can fail and need restore here.
  3217. if (hadReadLock)
  3218. readLocks++;
  3219. if (hadHoldLock)
  3220. holdLocks++;
  3221. }
  3222. return holdLocks ? LockHeld : LockFailed;
  3223. }
  3224. exclusive = true;
  3225. #ifdef _DEBUG
  3226. debugInfo.ExclOwningThread = GetCurrentThreadId();
  3227. debugInfo.ExclOwningConnection = id;
  3228. debugInfo.ExclOwningSession = sessId;
  3229. #endif
  3230. break;
  3231. default:
  3232. if (changingMode)
  3233. {
  3234. // only an unlocked read lock can fail and need restore here.
  3235. if (hadReadLock)
  3236. readLocks++;
  3237. if (hadHoldLock)
  3238. holdLocks++;
  3239. }
  3240. throwUnexpected();
  3241. }
  3242. if (changingMode)
  3243. {
  3244. existingLd->mode = mode;
  3245. wakeWaiting();
  3246. }
  3247. else
  3248. {
  3249. if (RTM_LOCK_SUB & mode)
  3250. sub++;
  3251. LockData ld(mode, sessId, msTick());
  3252. connectionInfo.setValue(id, ld);
  3253. }
  3254. return LockSucceeded;
  3255. }
  3256. inline void wakeWaiting()
  3257. {
  3258. if (waiting)
  3259. {
  3260. sem.signal(waiting); // get blocked locks to recheck.
  3261. waiting=0;
  3262. }
  3263. }
  3264. public:
  3265. IMPLEMENT_IINTERFACE;
  3266. CLock(CLockTable &_table, __int64 _treeId, IdPath &_idPath, const char *_xpath, unsigned mode, ConnectionId id, SessionId sessId)
  3267. : table(_table), treeId(_treeId), xpath(_xpath), exclusive(false), sub(0), readLocks(0), holdLocks(0), waiting(0), pending(0)
  3268. {
  3269. INIT_NAMEDCOUNT;
  3270. verifyex(tryLock(mode, id, sessId)==LockSucceeded);
  3271. ForEachItemIn(i, _idPath)
  3272. idPath.append(_idPath.item(i));
  3273. }
  3274. ~CLock()
  3275. {
  3276. if (parent)
  3277. clearLastRef();
  3278. }
  3279. inline void clearLastRef();
  3280. bool querySub() { return (0 != sub); }
  3281. const void *queryFindParam() const
  3282. {
  3283. return (const void *) &treeId;
  3284. }
  3285. bool matchHead(IdPath &_idPath)
  3286. {
  3287. unsigned o = idPath.ordinality();
  3288. ForEachItemIn(i, _idPath)
  3289. {
  3290. if (i>=o) return false;
  3291. else if (idPath.item(i) != _idPath.item(i))
  3292. return false;
  3293. }
  3294. return true;
  3295. }
  3296. bool unlock(ConnectionId id, bool delayDelete=false)
  3297. {
  3298. bool ret = false;
  3299. CPendingLockBlock b(*this); // carefully placed, removePending can destroy this, therefore must be destroyed last
  3300. {
  3301. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  3302. LockData *ld = connectionInfo.getValue(id);
  3303. if (ld) // not necessarily any lock info for this connection
  3304. {
  3305. switch (ld->mode & RTM_LOCKBASIC_MASK)
  3306. {
  3307. case RTM_LOCK_READ+RTM_LOCK_HOLD:
  3308. assertex(holdLocks);
  3309. holdLocks--;
  3310. // fall into...
  3311. case RTM_LOCK_READ:
  3312. assertex(readLocks);
  3313. readLocks--;
  3314. break;
  3315. case (RTM_LOCK_WRITE+RTM_LOCK_READ):
  3316. case RTM_LOCK_WRITE:
  3317. assertex(exclusive && 0 == readLocks && 0 == holdLocks);
  3318. exclusive = false;
  3319. #ifdef _DEBUG
  3320. debugInfo.clearExclusive();
  3321. #endif
  3322. break;
  3323. case 0: // no locking
  3324. break;
  3325. default:
  3326. throwUnexpected();
  3327. }
  3328. if (RTM_LOCK_SUB & ld->mode)
  3329. sub--;
  3330. connectionInfo.remove(id);
  3331. if (parent && 0 == connectionInfo.count())
  3332. {
  3333. if (delayDelete)
  3334. {
  3335. parent.clear();
  3336. child.clear();
  3337. }
  3338. else
  3339. clearLastRef();
  3340. ret = true;
  3341. }
  3342. }
  3343. wakeWaiting();
  3344. }
  3345. return ret;
  3346. }
  3347. void unlockAll()
  3348. {
  3349. CPendingLockBlock b(*this); // carefully placed, removePending can destroy this.
  3350. { CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  3351. HashIterator iter(connectionInfo);
  3352. while (iter.first())
  3353. {
  3354. IMapping &map = iter.query();
  3355. ConnectionId id = *(ConnectionId *)map.getKey();
  3356. unlock(id);
  3357. }
  3358. }
  3359. }
  3360. inline void addPending()
  3361. {
  3362. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  3363. pending++;
  3364. }
  3365. inline void removePending()
  3366. {
  3367. Linked<CLock> destroy;
  3368. {
  3369. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  3370. pending--;
  3371. if (0 == (lockCount()+pending))
  3372. {
  3373. destroy.set(this);
  3374. table.removeExact(this);
  3375. }
  3376. }
  3377. }
  3378. LockStatus lock(unsigned mode, unsigned timeout, ConnectionId id, SessionId sessionId, IUnlockCallback &callback)
  3379. {
  3380. bool ret = false;
  3381. CPendingLockBlock b(*this); // carefully placed, removePending can destroy this, therefore must be destroyed last
  3382. {
  3383. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  3384. return doLock(mode, timeout, id, sessionId, callback);
  3385. }
  3386. return LockFailed;
  3387. }
  3388. void changeMode(ConnectionId id, SessionId sessionId, unsigned newMode, unsigned timeout, IUnlockCallback &callback)
  3389. {
  3390. CPendingLockBlock b(*this); // carefully placed, removePending can destroy this.
  3391. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  3392. LockStatus result = doLock(newMode, timeout, id, sessionId, callback, true);
  3393. if (result != LockSucceeded)
  3394. {
  3395. StringBuffer s;
  3396. switch (result)
  3397. {
  3398. case LockFailed:
  3399. throw MakeSDSException(SDSExcpt_ConnectionAbsent, "Lost connection performing changeMode on connection to : %s", xpath.get());
  3400. case LockTimedOut:
  3401. throw MakeSDSException(SDSExcpt_LockTimeout, "Lock timeout performing changeMode on connection to : %s, existing lock info: %s", xpath.get(), toString(s).str());
  3402. case LockHeld:
  3403. throw MakeSDSException(SDSExcpt_LockHeld, "Lock is held performing changeMode on connection to : %s, existing lock info: %s", xpath.get(), toString(s).str());
  3404. }
  3405. }
  3406. }
  3407. unsigned lockCount()
  3408. {
  3409. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  3410. return connectionInfo.count();
  3411. }
  3412. bool associated(ConnectionId connectionId)
  3413. {
  3414. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  3415. return NULL!=connectionInfo.getValue(connectionId);
  3416. }
  3417. const char *queryXPath() const { return xpath; }
  3418. ILockInfo *getLockInfo() const
  3419. {
  3420. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  3421. return createLockInfo(xpath, connectionInfo); // NB: doesn't resolve sessionId to Endpoint string at this point
  3422. }
  3423. void setDROLR(CServerRemoteTree *_parent, CServerRemoteTree *_child)
  3424. {
  3425. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  3426. if (parent)
  3427. {
  3428. assertex(parent.get() == _parent);
  3429. assertex(child.get() == _child);
  3430. return;
  3431. }
  3432. parent.set(_parent);
  3433. child.set(_child);
  3434. }
  3435. StringBuffer &toString(StringBuffer &out) const
  3436. {
  3437. Owned<ILockInfo> lockInfo = getLockInfo();
  3438. return lockInfo->toString(out, 0, true);
  3439. }
  3440. };
  3441. CPendingLockBlock::CPendingLockBlock(CLock &_lock) : lock(_lock)
  3442. {
  3443. lock.addPending();
  3444. }
  3445. CPendingLockBlock::~CPendingLockBlock()
  3446. {
  3447. lock.removePending();
  3448. }
  3449. template <> void CLockTable::onRemove(void *et)
  3450. {
  3451. ((CLock*)et)->Release();
  3452. }
  3453. ///////////////
  3454. CSDSTransactionServer::CSDSTransactionServer(CCovenSDSManager &_manager)
  3455. : Thread("SDS Manager, CSDSTransactionServer"), manager(_manager), CTransactionLogTracker(DAMP_SDSCMD_MAX)
  3456. {
  3457. stopped = true;
  3458. }
  3459. int CSDSTransactionServer::run()
  3460. {
  3461. ICoven &coven=queryCoven();
  3462. CMessageHandler<CSDSTransactionServer> handler("CSDSTransactionServer",this,&CSDSTransactionServer::processMessage, &manager, 100, TIMEOUT_ON_CLOSEDOWN);
  3463. stopped = false;
  3464. CMessageBuffer mb;
  3465. while (!stopped)
  3466. {
  3467. try
  3468. {
  3469. #ifdef TRACE_QWAITING
  3470. unsigned waiting = coven.probe(RANK_ALL,MPTAG_DALI_SDS_REQUEST,NULL);
  3471. static unsigned lastwaiting = 0;
  3472. static unsigned lasttick = 0;
  3473. if ((waiting>lastwaiting+25)||(waiting<lastwaiting/2)||
  3474. ((waiting>100)&&(msTick()-lasttick>1000))) {
  3475. DBGLOG("QPROBE: MPTAG_DALI_SDS_REQUEST has %d waiting",waiting);
  3476. lastwaiting = waiting;
  3477. lasttick = msTick();
  3478. }
  3479. #endif
  3480. mb.clear();
  3481. if (coven.recv(mb, RANK_ALL, MPTAG_DALI_SDS_REQUEST, NULL))
  3482. {
  3483. msgCount++;
  3484. try
  3485. {
  3486. SdsCommand action;
  3487. mb.read((int &)action);
  3488. action = (SdsCommand) (((unsigned)action) & ~DAMP_SDSCMD_LAZYEXT);
  3489. switch (action)
  3490. {
  3491. case DAMP_SDSCMD_CONNECT:
  3492. case DAMP_SDSCMD_MCONNECT:
  3493. case DAMP_SDSCMD_GETCHILDREN:
  3494. case DAMP_SDSCMD_GETCHILDREN2:
  3495. case DAMP_SDSCMD_GET:
  3496. case DAMP_SDSCMD_GET2:
  3497. case DAMP_SDSCMD_GETELEMENTS:
  3498. case DAMP_SDSCMD_DATA:
  3499. case DAMP_SDSCMD_CHANGEMODE:
  3500. case DAMP_SDSCMD_GETXPATHS:
  3501. case DAMP_SDSCMD_GETXPATHSPLUSIDS:
  3502. case DAMP_SDSCMD_GETEXTVALUE:
  3503. case DAMP_SDSCMD_GETELEMENTSRAW:
  3504. case DAMP_SDSCMD_GETCOUNT:
  3505. {
  3506. mb.reset();
  3507. handler.handleMessage(mb);
  3508. mb.clear(); // ^ has copied mb
  3509. break;
  3510. }
  3511. case DAMP_SDSCMD_GETSTORE:
  3512. {
  3513. TimingBlock xactTimingBlock(xactTimingStats);
  3514. CServerRemoteTree *root = manager.queryRoot();
  3515. mb.clear();
  3516. if (root)
  3517. {
  3518. mb.append(DAMP_SDSREPLY_OK);
  3519. root->serializeCutOffRT(mb);
  3520. }
  3521. else
  3522. mb.append(DAMP_SDSREPLY_EMPTY);
  3523. break;
  3524. }
  3525. #ifdef LEGACY_CLIENT_RESPONSE
  3526. // give a re
  3527. case DAMP_SDSCMD_VERSION:
  3528. {
  3529. TimingBlock xactTimingBlock(xactTimingStats);
  3530. mb.clear().append(DAMP_SDSREPLY_ERROR);
  3531. throw MakeStringException(-1, "Client too old to communicate with this dali");
  3532. }
  3533. #endif
  3534. case DAMP_SDSCMD_DIAGNOSTIC:
  3535. {
  3536. TimingBlock xactTimingBlock(xactTimingStats);
  3537. SdsDiagCommand cmd;
  3538. mb.read((int &)cmd);
  3539. switch (cmd)
  3540. {
  3541. case DIAG_CMD_LOCKINFO:
  3542. {
  3543. StringAttr ipPattern, xpathPattern;
  3544. mb.read(ipPattern).read(xpathPattern);
  3545. mb.clear().append(DAMP_SDSREPLY_OK);
  3546. Owned<ILockInfoCollection> lockInfoCollection = SDSManager->getLocks(ipPattern, xpathPattern);
  3547. lockInfoCollection->serialize(mb);
  3548. break;
  3549. }
  3550. case DIAG_CMD_STATS:
  3551. {
  3552. mb.clear().append(DAMP_SDSREPLY_OK);
  3553. SDSManager->collectUsageStats(mb);
  3554. break;
  3555. }
  3556. case DIAG_CMD_CONNECTIONS:
  3557. {
  3558. mb.clear().append(DAMP_SDSREPLY_OK);
  3559. SDSManager->collectConnections(mb);
  3560. break;
  3561. }
  3562. case DIAG_CMD_SUBSCRIBERS:
  3563. {
  3564. mb.clear().append(DAMP_SDSREPLY_OK);
  3565. SDSManager->collectSubscribers(mb);
  3566. break;
  3567. }
  3568. default:
  3569. assertex(false);
  3570. }
  3571. break;
  3572. }
  3573. case DAMP_SDSCMD_GETPROPS:
  3574. {
  3575. mb.clear().append(DAMP_SDSREPLY_OK);
  3576. manager.queryProperties().serialize(mb);
  3577. break;
  3578. }
  3579. case DAMP_SDSCMD_UPDTENV:
  3580. {
  3581. Owned<IPropertyTree> newEnv = createPTree(mb);
  3582. bool forceGroupUpdate;
  3583. mb.read(forceGroupUpdate);
  3584. StringBuffer response;
  3585. bool result = manager.updateEnvironment(newEnv, forceGroupUpdate, response);
  3586. mb.clear().append(DAMP_SDSREPLY_OK).append(result).append(response);
  3587. break;
  3588. }
  3589. default:
  3590. throw MakeSDSException(SDSExcpt_UnrecognisedCommand, "%d", action);
  3591. }
  3592. }
  3593. catch (IException *e)
  3594. {
  3595. mb.clear();
  3596. mb.append((int) DAMP_SDSREPLY_ERROR);
  3597. mb.append(e->errorCode());
  3598. StringBuffer s;
  3599. mb.append(e->errorMessage(s).str());
  3600. StringBuffer clientUrl("EXCEPTION in reply to client ");
  3601. mb.getSender().getUrlStr(clientUrl);
  3602. EXCLOG(e, clientUrl.str(), MSGCLS_warning);
  3603. e->Release();
  3604. }
  3605. if (mb.length())
  3606. {
  3607. try { coven.reply(mb); }
  3608. catch (IJSOCK_Exception *e)
  3609. {
  3610. LOG(MCwarning, unknownJob, e, "Failed to reply to client (CSDSTransactionServer thread)");
  3611. e->Release();
  3612. }
  3613. catch (IMP_Exception *e)
  3614. {
  3615. LOG(MCwarning, unknownJob, e, "Failed to reply to client (CSDSTransactionServer thread)");
  3616. e->Release();
  3617. }
  3618. }
  3619. }
  3620. else
  3621. stopped = true;
  3622. }
  3623. catch (IException *e)
  3624. {
  3625. StringBuffer s("Failure receiving message from client ");
  3626. mb.getSender().getUrlStr(s);
  3627. LOG(MCwarning, unknownJob, e, s.str());
  3628. e->Release();
  3629. }
  3630. }
  3631. return 0;
  3632. }
  3633. // backward compat.
  3634. bool checkOldFormat(CServerRemoteTree *parentServerTree, IPropertyTree *tree, MemoryBuffer &mb)
  3635. {
  3636. CPState state;
  3637. mb.read((int &)state);
  3638. bool change = false;
  3639. if (state)
  3640. {
  3641. if (CPS_Renames & state)
  3642. {
  3643. for (;;)
  3644. {
  3645. __int64 id;
  3646. mb.read(id);
  3647. if (0 == id)
  3648. break;
  3649. StringAttr newName;
  3650. mb.read(newName);
  3651. IPropertyTree *child = SDSManager->queryRegisteredTree(id);
  3652. if (child)
  3653. {
  3654. assertex(parentServerTree);
  3655. int pos = parentServerTree->findChild(child);
  3656. if (NotFound == pos)
  3657. throw MakeSDSException(SDSExcpt_ClientCacheDirty, "::checkChange - child(%s) not found in parent(%s) at %s(%d)", child->queryName(), parentServerTree->queryName(), __FILE__, __LINE__);
  3658. IPropertyTree *t = createPTree();
  3659. t->setProp("@from", child->queryName());
  3660. t->setProp("@to", newName);
  3661. t->setPropInt64("@id", id);
  3662. #ifdef SIBLING_MOVEMENT_CHECK
  3663. t->setProp("@pos", pos);
  3664. #endif
  3665. tree->addPropTree(RENAME_TAG, t);
  3666. change = true;
  3667. }
  3668. }
  3669. }
  3670. if (CPS_Deletions & state)
  3671. {
  3672. for (;;)
  3673. {
  3674. __int64 id;
  3675. mb.read(id);
  3676. if (0 == id)
  3677. break;
  3678. IPropertyTree *child = SDSManager->queryRegisteredTree(id);
  3679. if (child)
  3680. {
  3681. assertex(parentServerTree);
  3682. int pos = parentServerTree->findChild(child);
  3683. if (NotFound == pos)
  3684. continue;
  3685. IPropertyTree *t = createPTree();
  3686. t->setProp("@name", child->queryName());
  3687. t->setPropInt64("@id", id);
  3688. #ifdef SIBLING_MOVEMENT_CHECK
  3689. t->setPropInt("@pos", pos+1);
  3690. #endif
  3691. tree->addPropTree(DELETE_TAG, t);
  3692. change = true;
  3693. }
  3694. }
  3695. }
  3696. if (CPS_AttrDeletions & state)
  3697. {
  3698. unsigned count, c;
  3699. mb.read(count);
  3700. if (count)
  3701. {
  3702. IPropertyTree *ct = tree->queryPropTree(ATTRCHANGE_TAG);
  3703. IPropertyTree *t = tree->addPropTree(ATTRDELETE_TAG, createPTree());
  3704. for (c=0; c<count; c++)
  3705. {
  3706. StringAttr attr;
  3707. mb.read(attr);
  3708. if (ct) ct->removeProp(attr);
  3709. t->addProp(attr, "");
  3710. }
  3711. change = true;
  3712. }
  3713. }
  3714. if (CPS_Changed & state)
  3715. {
  3716. Owned<PTree> clientTree = new LocalPTree();
  3717. clientTree->deserializeSelf(mb);
  3718. __int64 serverId;
  3719. mb.read(serverId);
  3720. byte STIInfo;
  3721. mb.read(STIInfo);
  3722. tree->setPropBool("@localValue", true);
  3723. if (clientTree->queryValue())
  3724. {
  3725. bool binary = clientTree->isBinary(NULL);
  3726. IPTArrayValue *v = ((PTree *)clientTree)->detachValue();
  3727. ((PTree *)tree)->setValue(v, binary);
  3728. }
  3729. else
  3730. ((PTree *)tree)->setValue(new CPTValue(0, NULL, false, true, false), false);
  3731. Owned<IAttributeIterator> attrs = clientTree->getAttributes();
  3732. if (attrs->first())
  3733. {
  3734. IPropertyTree *t = createPTree();
  3735. do
  3736. {
  3737. t->setProp(attrs->queryName(), clientTree->queryProp(attrs->queryName()));
  3738. }
  3739. while (attrs->next());
  3740. tree->addPropTree(ATTRCHANGE_TAG, t);
  3741. }
  3742. change = true;
  3743. }
  3744. }
  3745. return change;
  3746. }
  3747. bool translateOldFormat(CServerRemoteTree *parentServerTree, IPropertyTree *parentTree, MemoryBuffer &mb)
  3748. {
  3749. bool change = checkOldFormat(parentServerTree, parentTree, mb);
  3750. bool hasChildren;
  3751. mb.read(hasChildren);
  3752. if (hasChildren)
  3753. {
  3754. for (;;)
  3755. {
  3756. __int64 id;
  3757. int pos = -1;
  3758. mb.read(id);
  3759. if (NoMoreChildrenMarker == id)
  3760. break;
  3761. mb.read(pos);
  3762. CServerRemoteTree *serverTree = NULL;
  3763. Owned<IPropertyTree> tree = createPTree(RESERVED_CHANGE_NODE);
  3764. if (0 == id)
  3765. {
  3766. StringAttr childName;
  3767. mb.read(childName);
  3768. tree->setPropBool("@new", true);
  3769. tree->setProp("@name", childName);
  3770. if (-1 != pos)
  3771. tree->setPropInt("@pos", pos+1);
  3772. }
  3773. else
  3774. {
  3775. assertex(parentServerTree);
  3776. serverTree = (CServerRemoteTree *) SDSManager->queryRegisteredTree(id);
  3777. assertex(serverTree);
  3778. pos = parentServerTree->findChild(serverTree);
  3779. if (NotFound == pos)
  3780. throw MakeSDSException(SDSExcpt_ClientCacheDirty, "child(%s) not found in parent(%s) at %s(%d)", serverTree->queryName(), parentServerTree->queryName(), __FILE__, __LINE__);
  3781. tree->setProp("@name", serverTree->queryName());
  3782. tree->setPropInt64("@id", id);
  3783. tree->setPropInt("@pos", pos+1);
  3784. }
  3785. if (translateOldFormat(serverTree, tree, mb))
  3786. {
  3787. parentTree->addPropTree(tree->queryName(), LINK(tree));
  3788. change = true;
  3789. }
  3790. }
  3791. }
  3792. return change;
  3793. }
  3794. ///
  3795. void CSDSTransactionServer::processMessage(CMessageBuffer &mb)
  3796. {
  3797. TimingBlock xactTimingBlock(xactTimingStats);
  3798. ICoven &coven = queryCoven();
  3799. StringAttr xpath;
  3800. ConnectionId connectionId;
  3801. SessionId id;
  3802. unsigned mode;
  3803. unsigned timeout;
  3804. SdsCommand action = (SdsCommand)-1;
  3805. try
  3806. {
  3807. mb.read((int &)action);
  3808. bool getExt = 0 == (action & DAMP_SDSCMD_LAZYEXT);
  3809. action = (SdsCommand) (((unsigned)action) & ~DAMP_SDSCMD_LAZYEXT);
  3810. TransactionLog transactionLog(*this, action, mb.getSender()); // only active if queryTransactionLogging()==true
  3811. switch (action)
  3812. {
  3813. case DAMP_SDSCMD_CONNECT:
  3814. {
  3815. TimingBlock connectTimingBlock(connectTimingStats);
  3816. Owned<CLCLockBlock> lockBlock;
  3817. unsigned startPos = mb.getPos();
  3818. mb.read(id);
  3819. mb.read(mode);
  3820. mb.read(timeout);
  3821. mb.read(xpath);
  3822. if (queryTransactionLogging())
  3823. transactionLog.log("xpath='%s' mode=%d", xpath.get(), (unsigned)mode);
  3824. Owned<LinkingCriticalBlock> connectCritBlock = new LinkingCriticalBlock(manager.connectCrit, __FILE__, __LINE__);
  3825. if (RTM_CREATE == (mode & RTM_CREATE_MASK) || RTM_CREATE_QUERY == (mode & RTM_CREATE_MASK))
  3826. lockBlock.setown(new CLCWriteLockBlock(manager.dataRWLock, readWriteTimeout, __FILE__, __LINE__));
  3827. else
  3828. lockBlock.setown(new CLCReadLockBlock(manager.dataRWLock, readWriteTimeout, __FILE__, __LINE__));
  3829. if (queryTransactionLogging())
  3830. transactionLog.markExtra();
  3831. connectionId = 0;
  3832. CServerRemoteTree *_tree;
  3833. Owned<CServerRemoteTree> tree;
  3834. manager.createConnection(id, mode, timeout, xpath, _tree, connectionId, true, connectCritBlock);
  3835. if (connectionId)
  3836. tree.setown(_tree);
  3837. connectCritBlock.clear();
  3838. if (connectionId)
  3839. {
  3840. if (0 == id)
  3841. {
  3842. StringBuffer str("Dali client passing sessionid=0 to connect (xpath=");
  3843. str.append(xpath).append(", mode=").append(mode).append(", connectionId=").appendf("%" I64F "x", connectionId).append(")");
  3844. WARNLOG("%s", str.str());
  3845. }
  3846. mb.clear();
  3847. mb.append((int)DAMP_SDSREPLY_OK);
  3848. mb.append(connectionId);
  3849. tree->serializeCutOffRT(mb, RTM_SUB & mode?FETCH_ENTIRE:tree->getPropBool("@fetchEntire")?FETCH_ENTIRE_COND : 0, 0, getExt);
  3850. }
  3851. else
  3852. {
  3853. mb.clear();
  3854. mb.append((int)DAMP_SDSREPLY_EMPTY);
  3855. }
  3856. break;
  3857. }
  3858. case DAMP_SDSCMD_MCONNECT:
  3859. {
  3860. TimingBlock connectTimingBlock(connectTimingStats);
  3861. Owned<CLCLockBlock> lockBlock;
  3862. if (queryTransactionLogging())
  3863. transactionLog.log();
  3864. unsigned startPos = mb.getPos();
  3865. mb.read(id);
  3866. mb.read(timeout);
  3867. Owned<IMultipleConnector> mConnect = deserializeIMultipleConnector(mb);
  3868. mb.clear();
  3869. lockBlock.setown(new CLCReadLockBlock(manager.dataRWLock, readWriteTimeout, __FILE__, __LINE__));
  3870. try
  3871. {
  3872. Owned<CRemoteConnections> remoteConnections = new CRemoteConnections;
  3873. unsigned c;
  3874. for (c=0; c<mConnect->queryConnections(); c++)
  3875. {
  3876. StringAttr xpath;
  3877. unsigned mode;
  3878. mConnect->getConnectionDetails(c, xpath, mode);
  3879. if (queryTransactionLogging())
  3880. transactionLog.extra(", xpath='%s', mode=%d", xpath.get(), mode);
  3881. connectionId = 0;
  3882. CServerRemoteTree *_tree;
  3883. Owned<CServerRemoteTree> tree;
  3884. Owned<LinkingCriticalBlock> connectCritBlock = new LinkingCriticalBlock(manager.connectCrit, __FILE__, __LINE__);
  3885. manager.createConnection(id, mode, timeout, xpath, _tree, connectionId, true, connectCritBlock);
  3886. if (connectionId)
  3887. tree.setown(_tree);
  3888. connectCritBlock.clear();
  3889. if (connectionId)
  3890. {
  3891. if (0 == id)
  3892. {
  3893. StringBuffer str("Dali client passing sessionid=0 to multi connect (xpath=");
  3894. str.append(xpath).append(", mode=").append(mode).append(", connectionId=").appendf("%" I64F "x", connectionId).append(")");
  3895. WARNLOG("%s", str.str());
  3896. }
  3897. CRemoteConnection *conn = new CRemoteConnection(*SDSManager, connectionId, xpath, id, mode, timeout);
  3898. assertex(conn);
  3899. remoteConnections->add(conn);
  3900. mb.append((int)DAMP_SDSREPLY_OK);
  3901. mb.append(connectionId);
  3902. tree->serializeCutOffRT(mb, RTM_SUB & mode?FETCH_ENTIRE:tree->getPropBool("@fetchEntire")?FETCH_ENTIRE_COND : 0, 0, getExt);
  3903. }
  3904. else
  3905. {
  3906. mb.append((int)DAMP_SDSREPLY_EMPTY);
  3907. }
  3908. }
  3909. // success detach establish connections from holder (which would otherwise disconnect them)
  3910. remoteConnections->detachConnections();
  3911. }
  3912. catch (IException *e)
  3913. {
  3914. StringBuffer s("Failed to establish locks to multiple paths: ");
  3915. getMConnectString(mConnect, s);
  3916. LOG(MCwarning, unknownJob, e, s.str());
  3917. throw;
  3918. }
  3919. catch (DALI_CATCHALL)
  3920. {
  3921. StringBuffer s("(Unknown exception); Failed to establish locks to multiple paths: ");
  3922. getMConnectString(mConnect, s);
  3923. throw;
  3924. }
  3925. break;
  3926. }
  3927. case DAMP_SDSCMD_GET:
  3928. case DAMP_SDSCMD_GET2:
  3929. {
  3930. mb.read(connectionId);
  3931. if (queryTransactionLogging())
  3932. transactionLog.log();
  3933. __int64 serverId;
  3934. mb.read(serverId);
  3935. CHECKEDDALIREADLOCKBLOCK(manager.dataRWLock, readWriteTimeout);
  3936. CHECKEDCRITICALBLOCK(SDSManager->treeRegCrit, fakeCritTimeout);
  3937. Owned<CServerRemoteTree> tree = manager.getRegisteredTree(serverId);
  3938. if (queryTransactionLogging())
  3939. {
  3940. CServerConnection *conn = manager.queryConnection(connectionId);
  3941. transactionLog.extra(", xpath='%s', node=%s", conn?conn->queryXPath():"???", tree?tree->queryName():"???");
  3942. }
  3943. mb.clear();
  3944. if (!tree)
  3945. {
  3946. if (DAMP_SDSCMD_GET2 == action)
  3947. mb.append((int)DAMP_SDSREPLY_EMPTY);
  3948. else
  3949. {
  3950. CServerConnection *connection = manager.queryConnection(connectionId);
  3951. StringBuffer s;
  3952. if (connection)
  3953. {
  3954. s.append("path=").append(connection->queryXPath());
  3955. s.append(", mode=").append(connection->queryMode());
  3956. }
  3957. else
  3958. s.append("Missing connection!");
  3959. throw MakeSDSException(SDSExcpt_UnknownTreeId, "get: treeId = (%d), connection={ %s }", (unsigned)serverId, s.str());
  3960. }
  3961. }
  3962. else
  3963. {
  3964. mb.append((int)DAMP_SDSREPLY_OK);
  3965. tree->serializeCutOffRT(mb, tree->getPropBool("@fetchEntire")?-1 : 0, 0, getExt);
  3966. }
  3967. break;
  3968. }
  3969. case DAMP_SDSCMD_GETCHILDREN:
  3970. case DAMP_SDSCMD_GETCHILDREN2:
  3971. {
  3972. mb.read(connectionId);
  3973. if (queryTransactionLogging())
  3974. {
  3975. CServerConnection *conn = manager.queryConnection(connectionId);
  3976. transactionLog.log("%s",conn?conn->queryXPath():"???");
  3977. }
  3978. __int64 serverId;
  3979. CHECKEDDALIREADLOCKBLOCK(manager.dataRWLock, readWriteTimeout);
  3980. CHECKEDCRITICALBLOCK(SDSManager->treeRegCrit, fakeCritTimeout);
  3981. CMessageBuffer replyMb;
  3982. replyMb.init(mb.getSender(), mb.getTag(), mb.getReplyTag());
  3983. replyMb.append((int)DAMP_SDSREPLY_OK);
  3984. bool first = true, empty = false;
  3985. for (;;)
  3986. {
  3987. mb.read(serverId);
  3988. if (!serverId) break;
  3989. if (!first && empty) replyMb.clear();
  3990. unsigned levels;
  3991. mb.read(levels);
  3992. Owned<CServerRemoteTree> parent = manager.getRegisteredTree(serverId);
  3993. if (!parent)
  3994. {
  3995. if (DAMP_SDSCMD_GETCHILDREN2 == action)
  3996. replyMb.append(false);
  3997. else
  3998. {
  3999. if (first) // if only one, can acheive without serialization change.
  4000. {
  4001. empty = true;
  4002. replyMb.clear().append((int)DAMP_SDSREPLY_EMPTY);
  4003. }
  4004. else
  4005. {
  4006. CServerConnection *connection = manager.queryConnection(connectionId);
  4007. StringBuffer s;
  4008. if (connection)
  4009. {
  4010. s.append("path=").append(connection->queryXPath());
  4011. s.append(", mode=").append(connection->queryMode());
  4012. }
  4013. else
  4014. s.append("Missing connection!");
  4015. throw MakeSDSException(SDSExcpt_UnknownTreeId, "GETCHILDREN: Failed to locate parent (%d), connection={ %s }", (unsigned)serverId, s.str());
  4016. }
  4017. }
  4018. }
  4019. else
  4020. {
  4021. if (DAMP_SDSCMD_GETCHILDREN2 == action)
  4022. replyMb.append(true);
  4023. parent->serializeCutOffChildrenRT(replyMb, 0==levels ? (unsigned)-1 : levels, 0, getExt);
  4024. if (queryTransactionLogging())
  4025. transactionLog.extra(", node=%s",parent->queryName());
  4026. }
  4027. first = false;
  4028. }
  4029. mb.clear();
  4030. mb.transferFrom(replyMb);
  4031. break;
  4032. }
  4033. case DAMP_SDSCMD_GETELEMENTS:
  4034. {
  4035. mb.read(connectionId);
  4036. if (queryTransactionLogging())
  4037. {
  4038. CServerConnection *conn = manager.queryConnection(connectionId);
  4039. transactionLog.log("%s",conn?conn->queryXPath():"???");
  4040. }
  4041. CHECKEDDALIREADLOCKBLOCK(manager.dataRWLock, readWriteTimeout);
  4042. CHECKEDCRITICALBLOCK(SDSManager->treeRegCrit, fakeCritTimeout);
  4043. CServerConnection *connection = manager.queryConnection(connectionId);
  4044. if (!connection)
  4045. throw MakeSDSException(SDSExcpt_ConnectionAbsent, " [getElements]");
  4046. StringAttr xpath;
  4047. mb.read(xpath);
  4048. if (queryTransactionLogging())
  4049. transactionLog.extra(", xpath='%s'", xpath.get());
  4050. Owned<IPropertyTreeIterator> iter = connection->queryRoot()->getElements(xpath);
  4051. ICopyArrayOf<CServerRemoteTree> arr;
  4052. ForEach (*iter) arr.append((CServerRemoteTree &)iter->query());
  4053. CMessageBuffer replyMb;
  4054. replyMb.init(mb.getSender(), mb.getTag(), mb.getReplyTag());
  4055. replyMb.append((int)DAMP_SDSREPLY_OK);
  4056. replyMb.append(arr.ordinality());
  4057. ForEachItemIn(i, arr)
  4058. arr.item(i).serializeSelfRT(replyMb, getExt);
  4059. mb.clear();
  4060. mb.transferFrom(replyMb);
  4061. break;
  4062. }
  4063. case DAMP_SDSCMD_DATA:
  4064. {
  4065. TimingSizeBlock commitTimingBlock(commitTimingStats);
  4066. CheckTime block0("DAMP_SDSCMD_DATA total");
  4067. unsigned inputStart = mb.getPos();
  4068. mb.read(connectionId);
  4069. byte disconnect; // kludge, high bit to indicate new client format. (for backward compat.)
  4070. bool deleteRoot;
  4071. mb.read(disconnect);
  4072. bool oldFormat = (0 == (0x80 & disconnect));
  4073. disconnect &= ~0x80;
  4074. if (1 == disconnect)
  4075. mb.read(deleteRoot);
  4076. bool data = mb.length() != mb.getPos();
  4077. if (queryTransactionLogging())
  4078. {
  4079. CServerConnection *conn = manager.queryConnection(connectionId);
  4080. transactionLog.log("disconnect=%s, data=%s, deleteRoot=%s", disconnect?"true":"false", data?"true":"false", deleteRoot?"true":"false");
  4081. }
  4082. Owned<CLCLockBlock> lockBlock;
  4083. {
  4084. CheckTime block1("DAMP_SDSCMD_DATA.1");
  4085. if (data || deleteRoot)
  4086. lockBlock.setown(new CLCWriteLockBlock(manager.dataRWLock, readWriteTimeout, __FILE__, __LINE__));
  4087. else
  4088. lockBlock.setown(new CLCReadLockBlock(manager.dataRWLock, readWriteTimeout, __FILE__, __LINE__));
  4089. }
  4090. unsigned dataStart = mb.getPos();
  4091. commitTimingBlock.recordSize(mb.length() - dataStart);
  4092. CServerConnection *connection = manager.queryConnection(connectionId);
  4093. if (!connection)
  4094. throw MakeSDSException(SDSExcpt_ConnectionAbsent, " [commit]");
  4095. try
  4096. {
  4097. if (queryTransactionLogging())
  4098. transactionLog.extra(", xpath='%s'", connection->queryXPath());
  4099. CServerRemoteTree *tree = data ? (CServerRemoteTree *)connection->queryRoot() : (CServerRemoteTree *)connection->queryRootUnvalidated();
  4100. MemoryBuffer newIds;
  4101. Owned<IPropertyTree> changeTree;
  4102. if (data)
  4103. {
  4104. if (oldFormat)
  4105. {
  4106. Owned<IPropertyTree> t = createPTree(RESERVED_CHANGE_NODE);
  4107. t->setProp("@name", tree->queryName());
  4108. if (translateOldFormat(tree, t, mb))
  4109. changeTree.setown(LINK(t));
  4110. }
  4111. else
  4112. changeTree.setown(createPTree(mb));
  4113. }
  4114. if (changeTree && tree->processData(*connection, *changeTree, newIds))
  4115. { // something commited, if RTM_Create was used need to remember this.
  4116. CheckTime block6("DAMP_SDSCMD_DATA.6");
  4117. StringBuffer path;
  4118. connection->queryPTreePath().getAbsolutePath(path);
  4119. manager.saveDelta(path.str(), *changeTree);
  4120. }
  4121. mb.clear();
  4122. mb.append((int)DAMP_SDSREPLY_OK);
  4123. mb.append(newIds); // JCSMORE not particularly efficient change later
  4124. if (block0.slow())
  4125. {
  4126. block0.appendMsg(", xpath=").append(connection->queryXPath());
  4127. block0.appendMsg(", block size = ").append(mb.length());
  4128. }
  4129. }
  4130. catch (IException *)
  4131. {
  4132. if (disconnect)
  4133. manager.disconnect(connectionId, deleteRoot, (data || deleteRoot)?NULL:&lockBlock);
  4134. throw;
  4135. }
  4136. if (disconnect)
  4137. manager.disconnect(connectionId, deleteRoot, (data || deleteRoot)?NULL:&lockBlock);
  4138. break;
  4139. }
  4140. case DAMP_SDSCMD_CHANGEMODE:
  4141. {
  4142. mb.read(connectionId);
  4143. if (queryTransactionLogging())
  4144. transactionLog.log();
  4145. CHECKEDDALIWRITELOCKBLOCK(manager.dataRWLock, readWriteTimeout);
  4146. Linked<CServerConnection> connection = manager.queryConnection(connectionId);
  4147. if (!connection)
  4148. throw MakeSDSException(SDSExcpt_ConnectionAbsent, " [changeMode]");
  4149. CServerRemoteTree *tree = (CServerRemoteTree *) connection->queryRoot();
  4150. assertex(tree);
  4151. if (queryTransactionLogging())
  4152. transactionLog.extra(", xpath='%s'", connection->queryXPath());
  4153. unsigned newMode;
  4154. unsigned timeout;
  4155. mb.read(newMode);
  4156. mb.read(timeout);
  4157. mb.clear();
  4158. manager.changeLockMode(*connection, newMode, timeout);
  4159. if (!manager.queryConnection(connectionId))
  4160. {
  4161. manager.unlock(tree->queryServerId(), connectionId);
  4162. throw MakeSDSException(SDSExcpt_AbortDuringConnection, " during changeMode");
  4163. }
  4164. mb.append((int) DAMP_SDSREPLY_OK);
  4165. break;
  4166. }
  4167. case DAMP_SDSCMD_GETXPATHS:
  4168. case DAMP_SDSCMD_GETXPATHSPLUSIDS:
  4169. {
  4170. __int64 serverId;
  4171. mb.read(serverId);
  4172. mb.read(xpath);
  4173. if (queryTransactionLogging())
  4174. transactionLog.log("xpath='%s'", xpath.get());
  4175. mb.clear();
  4176. CHECKEDDALIREADLOCKBLOCK(manager.dataRWLock, readWriteTimeout);
  4177. Owned<IPropertyTree> matchTree = SDSManager->getXPaths(serverId, xpath, DAMP_SDSCMD_GETXPATHSPLUSIDS==action);
  4178. if (matchTree)
  4179. {
  4180. mb.append((int) DAMP_SDSREPLY_OK);
  4181. matchTree->serialize(mb);
  4182. }
  4183. else
  4184. mb.append((int) DAMP_SDSREPLY_EMPTY);
  4185. break;
  4186. }
  4187. case DAMP_SDSCMD_GETXPATHSCRITERIA:
  4188. {
  4189. StringAttr matchXPath, sortBy;
  4190. bool caseinsensitive, ascending;
  4191. unsigned from, limit;
  4192. mb.read(xpath);
  4193. mb.read(matchXPath);
  4194. mb.read(sortBy);
  4195. mb.read(caseinsensitive);
  4196. mb.read(ascending);
  4197. mb.read(from);
  4198. mb.read(limit);
  4199. if (queryTransactionLogging())
  4200. {
  4201. transactionLog.log("xpath='%s',matchXPath='%s',sortBy='%s',acscending=%s,from=%d,limit=%d",
  4202. xpath.get(), matchXPath.get(), sortBy.get(),
  4203. ascending?"true":"false", from, limit);
  4204. }
  4205. mb.clear();
  4206. CHECKEDDALIREADLOCKBLOCK(manager.dataRWLock, readWriteTimeout);
  4207. Owned<IPropertyTree> matchTree = SDSManager->getXPathsSortLimitMatchTree(xpath, matchXPath, sortBy, caseinsensitive, ascending, from, limit);
  4208. if (matchTree)
  4209. {
  4210. mb.append((int) DAMP_SDSREPLY_OK);
  4211. matchTree->serialize(mb);
  4212. }
  4213. else
  4214. mb.append((int) DAMP_SDSREPLY_EMPTY);
  4215. break;
  4216. }
  4217. case DAMP_SDSCMD_GETEXTVALUE:
  4218. {
  4219. __int64 serverId;
  4220. mb.read(serverId);
  4221. mb.clear().append((int) DAMP_SDSREPLY_OK);
  4222. if (queryTransactionLogging())
  4223. {
  4224. Owned<CServerRemoteTree> idTree = (CServerRemoteTree *) SDSManager->getRegisteredTree(serverId);
  4225. transactionLog.log("%s", idTree?idTree->queryName():"???");
  4226. }
  4227. SDSManager->getExternalValueFromServerId(serverId, mb);
  4228. break;
  4229. }
  4230. case DAMP_SDSCMD_GETELEMENTSRAW:
  4231. {
  4232. CHECKEDDALIREADLOCKBLOCK(manager.dataRWLock, readWriteTimeout);
  4233. StringAttr _xpath;
  4234. mb.read(_xpath);
  4235. if (queryTransactionLogging())
  4236. transactionLog.log("%s", xpath.get());
  4237. CMessageBuffer replyMb;
  4238. replyMb.init(mb.getSender(), mb.getTag(), mb.getReplyTag());
  4239. replyMb.append((int)DAMP_SDSREPLY_OK);
  4240. unsigned pos = replyMb.length();
  4241. unsigned count = 0;
  4242. replyMb.append(count);
  4243. const char *xpath = _xpath.get();
  4244. if ('/' == *xpath) ++xpath;
  4245. Owned<IPropertyTreeIterator> iter = manager.queryRoot()->getElements(xpath);
  4246. ForEach (*iter)
  4247. {
  4248. ++count;
  4249. IPropertyTree &e = iter->query();
  4250. e.serialize(replyMb);
  4251. }
  4252. replyMb.writeDirect(pos,sizeof(count),&count);
  4253. mb.clear();
  4254. mb.transferFrom(replyMb);
  4255. break;
  4256. }
  4257. case DAMP_SDSCMD_GETCOUNT:
  4258. {
  4259. mb.read(xpath);
  4260. if (queryTransactionLogging())
  4261. transactionLog.log("xpath='%s'", xpath.get());
  4262. CHECKEDDALIREADLOCKBLOCK(manager.dataRWLock, readWriteTimeout);
  4263. mb.clear();
  4264. mb.append((int)DAMP_SDSREPLY_OK);
  4265. mb.append(manager.queryCount(xpath));
  4266. break;
  4267. }
  4268. default:
  4269. throwUnexpected();
  4270. }
  4271. }
  4272. catch (IException *e)
  4273. {
  4274. mb.clear();
  4275. mb.append((int) DAMP_SDSREPLY_ERROR);
  4276. StringBuffer s;
  4277. e->errorMessage(s);
  4278. // NB: wanted to do this in a catch (IPTreeException *) block, but did catch,
  4279. // in spite of being able to query with dynamic cast
  4280. // something to do with rethrow, if I changed to catch early as IPT then it would catch here correctly.
  4281. if (QUERYINTERFACE(e, IPTreeException))
  4282. {
  4283. s.append(" in xpath '").append(xpath).append("'");
  4284. e->Release();
  4285. e = MakeSDSException(SDSExcpt_IPTError, "%s", s.str());
  4286. }
  4287. mb.append(e->errorCode());
  4288. mb.append(e->errorMessage(s.clear()));
  4289. StringBuffer clientUrl("EXCEPTION in reply to client ");
  4290. mb.getSender().getUrlStr(clientUrl);
  4291. EXCLOG(e, clientUrl.str(), MSGCLS_warning);
  4292. e->Release();
  4293. }
  4294. catch (DALI_CATCHALL)
  4295. {
  4296. Owned<IException> e = MakeSDSException(-1, "Unknown server exception processing client action: %d", action);
  4297. mb.clear();
  4298. mb.append((int) DAMP_SDSREPLY_ERROR);
  4299. StringBuffer s;
  4300. mb.append(e->errorCode());
  4301. mb.append(e->errorMessage(s).str());
  4302. StringBuffer clientUrl("EXCEPTION in reply to client ");
  4303. mb.getSender().getUrlStr(clientUrl);
  4304. LOG(MCoperatorError, unknownJob, e);
  4305. }
  4306. try {
  4307. CheckTime block10("DAMP_REQUEST reply");
  4308. coven.reply(mb);
  4309. }
  4310. catch (IMP_Exception *e)
  4311. {
  4312. LOG(MCwarning, unknownJob, e, "Failed to reply to client (processMessage)");
  4313. e->Release();
  4314. }
  4315. catch (IException *e)
  4316. {
  4317. // attempt error reply, on failed initial reply, reply error *might* have been message sensitive, e.g. OOM.
  4318. mb.clear();
  4319. mb.append((int) DAMP_SDSREPLY_ERROR);
  4320. mb.append(e->errorCode());
  4321. StringBuffer s;
  4322. mb.append(e->errorMessage(s).str());
  4323. StringBuffer clientUrl("EXCEPTION in reply to client ");
  4324. mb.getSender().getUrlStr(clientUrl);
  4325. EXCLOG(e, clientUrl.str(), MSGCLS_warning);
  4326. e->Release();
  4327. try
  4328. {
  4329. coven.reply(mb);
  4330. LOG(MCdebugInfo(100), unknownJob, "Failed to reply, but succeeded sending initial reply error to client");
  4331. }
  4332. catch (IException *e)
  4333. {
  4334. LOG(MCwarning, unknownJob, e, "Failed to reply and failed to send reply error to client");
  4335. e->Release();
  4336. }
  4337. }
  4338. }
  4339. void CSDSTransactionServer::stop()
  4340. {
  4341. if (!stopped) {
  4342. stopped = true;
  4343. queryCoven().cancel(RANK_ALL, MPTAG_DALI_SDS_REQUEST);
  4344. }
  4345. PROGLOG("clearing remaining sds locks");
  4346. manager.clearSDSLocks();
  4347. PROGLOG("waiting for transaction server to stop");
  4348. join();
  4349. }
  4350. /////////////////
  4351. // CServerConnection impl.
  4352. IPropertyTree *CServerConnection::queryRoot()
  4353. {
  4354. if (((CServerRemoteTree *)root.get())->isOrphaned())
  4355. throw MakeSDSException(SDSExcpt_OrphanedNode, "%s", queryXPath());
  4356. return root;
  4357. }
  4358. ////////////////
  4359. void CLock::clearLastRef()
  4360. {
  4361. if (parent)
  4362. {
  4363. CPendingLockBlock b(*this); // carefully placed, removePending can destroy this.
  4364. parent->removeTree(child);
  4365. parent.clear();
  4366. child.clear();
  4367. }
  4368. }
  4369. ////////////////
  4370. class ConnectionIdHashTable : public SuperHashTableOf<ConnectionId, ConnectionId>
  4371. {
  4372. public:
  4373. ~ConnectionIdHashTable() { _releaseAll(); }
  4374. IMPLEMENT_SUPERHASHTABLEOF_REF_FIND(ConnectionId, ConnectionId);
  4375. virtual void onAdd(void *et) { }
  4376. virtual void onRemove(void *et) { delete (ConnectionId *)et; }
  4377. virtual unsigned getHashFromElement(const void *et) const
  4378. {
  4379. return hashc((const unsigned char *) et, sizeof(ConnectionId), 0);
  4380. }
  4381. virtual unsigned getHashFromFindParam(const void *fp) const
  4382. {
  4383. return hashc((const unsigned char *) fp, sizeof(ConnectionId), 0);
  4384. }
  4385. virtual const void *getFindParam(const void *et) const
  4386. {
  4387. return et;
  4388. }
  4389. virtual bool matchesFindParam(const void *et, const void *fp, unsigned) const
  4390. {
  4391. return *(ConnectionId *)et == *(ConnectionId *)fp;
  4392. }
  4393. };
  4394. static bool retryRename(const char *from, const char *to, unsigned maxAttempts, unsigned delay)
  4395. {
  4396. unsigned attempts=maxAttempts;
  4397. for (;;)
  4398. {
  4399. OwnedIFile iFile = createIFile(from);
  4400. try
  4401. {
  4402. iFile->rename(to);
  4403. break;
  4404. }
  4405. catch (IException *e)
  4406. {
  4407. StringBuffer errTxt("Failed to rename: ");
  4408. EXCLOG(e, errTxt.append(from).append(" to ").append(to).append(", retrying...").str());
  4409. e->Release();
  4410. }
  4411. if (attempts && 0 == --attempts)
  4412. break;
  4413. MilliSleep(delay);
  4414. }
  4415. return (attempts>0);
  4416. }
  4417. #ifdef NODELETE
  4418. static bool retryCopy(const char *from, const char *to, unsigned maxAttempts, unsigned delay)
  4419. {
  4420. unsigned attempts=maxAttempts;
  4421. for (;;)
  4422. {
  4423. StringBuffer _from;
  4424. StringBuffer fname;
  4425. splitFilename(from, &_from, &_from, &fname, &fname);
  4426. _from.append('_').append(fname);
  4427. OwnedIFile iFile = createIFile(from);
  4428. try
  4429. {
  4430. iFile->rename(_from.str());
  4431. copyFile(to, _from.str());
  4432. break;
  4433. }
  4434. catch (IException *e)
  4435. {
  4436. EXCLOG(e, NULL);
  4437. e->Release();
  4438. }
  4439. if (attempts && 0 == --attempts)
  4440. break;
  4441. DBGLOG("Failed to copy: %s to %s, retrying...", from, to);
  4442. MilliSleep(delay);
  4443. }
  4444. return (attempts>0);
  4445. }
  4446. #endif
  4447. inline unsigned nextEditionN(unsigned e, unsigned i=1)
  4448. {
  4449. return e+i;
  4450. }
  4451. inline unsigned prevEditionN(unsigned e, unsigned i=1)
  4452. {
  4453. return e-i;
  4454. }
  4455. void removeDaliFile(const char *path, const char *base, unsigned e)
  4456. {
  4457. StringBuffer filename(path);
  4458. constructStoreName(base, e, filename);
  4459. OwnedIFile iFile = createIFile(filename.str());
  4460. try
  4461. {
  4462. iFile->remove();
  4463. }
  4464. catch (IException *e)
  4465. {
  4466. EXCLOG(e, NULL);
  4467. e->Release();
  4468. }
  4469. }
  4470. // Ensure internally used branches are present
  4471. void initializeInternals(IPropertyTree *root)
  4472. {
  4473. ensurePTree(root, "Files");
  4474. ensurePTree(root, "Queues");
  4475. ensurePTree(root, "Groups");
  4476. ensurePTree(root, "Status");
  4477. ensurePTree(root, "WorkUnits");
  4478. ensurePTree(root, "JobQueues");
  4479. ensurePTree(root, "Environment");
  4480. ensurePTree(root, "Locks");
  4481. ensurePTree(root, "DFU");
  4482. ensurePTree(root, "DFU/RECOVERY");
  4483. ensurePTree(root, "DFU/WorkUnits");
  4484. ensurePTree(root, "Files/Relationships");
  4485. root->removeProp("Status/Servers");
  4486. root->addPropTree("Status/Servers",createPTree());
  4487. }
  4488. IPropertyTree *loadStore(const char *storeFilename, IPTreeMaker *iMaker, unsigned crcValidation, bool logErrorsOnly=false, const bool *abort=NULL)
  4489. {
  4490. CHECKEDCRITICALBLOCK(loadStoreCrit, fakeCritTimeout);
  4491. CHECKEDCRITICALBLOCK(saveStoreCrit, fakeCritTimeout);
  4492. Owned<IPropertyTree> root;
  4493. try
  4494. {
  4495. OwnedIFile iFileStore = createIFile(storeFilename);
  4496. OwnedIFileIO iFileIOStore = iFileStore->open(IFOread);
  4497. if (!iFileIOStore)
  4498. throw MakeSDSException(SDSExcpt_OpenStoreFailed, "%s", storeFilename);
  4499. Owned<IFileIOStream> fstream = createIOStream(iFileIOStore);
  4500. Owned<ICrcIOStream> crcPipeStream = createCrcPipeStream(fstream);
  4501. Owned<IIOStream> ios = createBufferedIOStream(crcPipeStream);
  4502. root.setown((CServerRemoteTree *) createPTree(*ios, ipt_none, ptr_ignoreWhiteSpace, iMaker));
  4503. ios.clear();
  4504. unsigned crc = crcPipeStream->queryCrc();
  4505. if (crcValidation && crc != crcValidation)
  4506. LOG(MCoperatorWarning, unknownJob, "Error processing store %s - CRC ERROR (file size=%" I64F "d, validation crc=%x, calculated crc=%x)", storeFilename, iFileIOStore->size(), crcValidation, crc); // not fatal yet (maybe later)
  4507. }
  4508. catch (IException *e)
  4509. {
  4510. if (!abort)
  4511. {
  4512. StringBuffer s("Exception - loading store file : ");
  4513. s.appendf("%s", storeFilename);
  4514. LOG(MCoperatorError, unknownJob, e, s.str());
  4515. }
  4516. if (SDSExcpt_OpenStoreFailed != e->errorCode())
  4517. if (!logErrorsOnly)
  4518. throw;
  4519. e->Release();
  4520. }
  4521. catch (DALI_CATCHALL)
  4522. {
  4523. IException *e = MakeStringException(0, "Unknown exception - loading store file : %s", storeFilename);
  4524. LOG(MCdisaster, unknownJob, e, "");
  4525. if (!logErrorsOnly)
  4526. throw;
  4527. e->Release();
  4528. }
  4529. return LINK(root);
  4530. }
  4531. // Not really coalescing, blocking transations and saving store (which will delete pending transactions).
  4532. class CLightCoalesceThread : implements ICoalesce, public CInterface
  4533. {
  4534. bool stopped, within24;
  4535. Semaphore sem;
  4536. unsigned writeTransactionsNow, lastSaveWriteTransactions, lastWarning;
  4537. unsigned idlePeriod, minimumTimeBetweenSaves, idleRate;
  4538. Linked<IPropertyTree> config;
  4539. Owned<IJlibDateTime> quietStartTime, quietEndTime;
  4540. CheckedCriticalSection crit;
  4541. IStoreHelper *iStoreHelper;
  4542. class CThreaded : public Thread
  4543. {
  4544. CLightCoalesceThread *coalesce;
  4545. public:
  4546. CThreaded() : Thread("CLightCoalesceThread") { coalesce = NULL; }
  4547. void init(CLightCoalesceThread *_coalesce) { coalesce = _coalesce; start(); }
  4548. virtual int run() { coalesce->main(); return 1; }
  4549. } threaded;
  4550. public:
  4551. IMPLEMENT_IINTERFACE;
  4552. CLightCoalesceThread(IPropertyTree &_config, IStoreHelper *_iStoreHelper) : config(&_config), iStoreHelper(_iStoreHelper)
  4553. {
  4554. stopped = false;
  4555. idlePeriod = config->getPropInt("@lCIdlePeriod", DEFAULT_LCIDLE_PERIOD)*1000;
  4556. minimumTimeBetweenSaves = config->getPropInt("@lCMinTime", DEFAULT_LCMIN_TIME)*1000;
  4557. idleRate = config->getPropInt("@lCIdleRate", DEFAULT_LCIDLE_RATE);
  4558. char const *quietStartTimeStr = config->queryProp("@lCQuietStartTime");
  4559. if (quietStartTimeStr)
  4560. {
  4561. if (*quietStartTimeStr)
  4562. {
  4563. quietStartTime.setown(createDateTime());
  4564. quietStartTime->setLocalTimeString(quietStartTimeStr);
  4565. quietStartTime->setGmtDate(1970, 1, 1);
  4566. }
  4567. else
  4568. quietStartTimeStr = NULL;
  4569. }
  4570. char const *quietEndTimeStr = config->queryProp("@lCQuietEndTime");
  4571. if (quietStartTimeStr && !quietEndTimeStr)
  4572. {
  4573. WARNLOG("Start time for quiet period specified without end time, ignoring times");
  4574. quietStartTime.clear();
  4575. }
  4576. else if (quietEndTimeStr && *quietEndTimeStr)
  4577. {
  4578. if (!quietStartTimeStr)
  4579. WARNLOG("End time for quiet period specified without start time, ignoring times");
  4580. else
  4581. {
  4582. quietEndTime.setown(createDateTime());
  4583. quietEndTime->setLocalTimeString(quietEndTimeStr);
  4584. quietEndTime->setGmtDate(1970, 1, 1);
  4585. within24 = quietStartTime->compare(*quietEndTime) <= 0;
  4586. }
  4587. }
  4588. }
  4589. ~CLightCoalesceThread()
  4590. {
  4591. stop();
  4592. }
  4593. void main()
  4594. {
  4595. unsigned t = 0;
  4596. lastSaveWriteTransactions = SDSManager->writeTransactions;
  4597. lastWarning = 0;
  4598. unsigned lastEdition = iStoreHelper->queryCurrentEdition();
  4599. while (!stopped)
  4600. {
  4601. unsigned writeTransactionsNow = SDSManager->writeTransactions;
  4602. if (!sem.wait(idlePeriod))
  4603. {
  4604. if (writeTransactionsNow != lastSaveWriteTransactions)
  4605. {
  4606. if (quietStartTime)
  4607. {
  4608. Owned<IJlibDateTime> nowTime = createDateTimeNow();
  4609. nowTime->setGmtDate(1970, 1, 1);
  4610. if (within24)
  4611. {
  4612. if (!(nowTime->compare(*quietStartTime) >= 0 && nowTime->compare(*quietEndTime) <= 0))
  4613. continue; // if outside quiet period within 0-24
  4614. }
  4615. else if (nowTime->compare(*quietEndTime) > 0 && nowTime->compare(*quietStartTime) < 0)
  4616. continue; // if inside period excluded by quiet period
  4617. }
  4618. if (lastEdition == iStoreHelper->queryCurrentEdition()) // if not then something else has saved (e.g. probably sasha)
  4619. {
  4620. unsigned transactions = SDSManager->writeTransactions-writeTransactionsNow; // don't care about rollover.
  4621. if (0 == transactions ||
  4622. (0 != idleRate && idlePeriod>=60000 && (transactions/(idlePeriod/60000))<=idleRate))
  4623. {
  4624. StringBuffer filename;
  4625. iStoreHelper->getPrimaryLocation(filename);
  4626. iStoreHelper->getCurrentStoreFilename(filename);
  4627. OwnedIFile iFile = createIFile(filename);
  4628. CDateTime createTime, nowTime;
  4629. nowTime.setNow();
  4630. int diff = 0;
  4631. try
  4632. {
  4633. if (iFile->getTime(&createTime, NULL, NULL))
  4634. diff = ((int)nowTime.getSimple()-(int)createTime.getSimple())*1000;
  4635. }
  4636. catch (IException *e)
  4637. {
  4638. StringBuffer errMsg("failed to get createtime for : ");
  4639. errMsg.append(filename);
  4640. EXCLOG(e, errMsg.str());
  4641. e->Release();
  4642. }
  4643. int period;
  4644. if (diff<=0 || diff > (int)minimumTimeBetweenSaves) // <0 - createTime>nowTime, assume time skew and allow save.
  4645. {
  4646. period = minimumTimeBetweenSaves-idlePeriod;
  4647. if (0 > period) period = 0;
  4648. {
  4649. CHECKEDCRITICALBLOCK(saveStoreCrit, fakeCritTimeout);
  4650. SDSManager->blockingSave(&lastSaveWriteTransactions);
  4651. lastEdition = iStoreHelper->queryCurrentEdition();
  4652. }
  4653. t = lastWarning = 0;
  4654. }
  4655. else
  4656. period = minimumTimeBetweenSaves-diff;
  4657. sem.wait(period);
  4658. }
  4659. else
  4660. {
  4661. t += idlePeriod/1000;
  4662. if (t/3600 >= STORENOTSAVE_WARNING_PERIOD && ((t-lastWarning)/3600>(STORENOTSAVE_WARNING_PERIOD/2)))
  4663. {
  4664. WARNLOG("Store has not been saved for %d hours", t/3600);
  4665. lastWarning = t;
  4666. }
  4667. }
  4668. }
  4669. else
  4670. {
  4671. t = lastWarning = 0;
  4672. lastEdition = iStoreHelper->queryCurrentEdition();
  4673. }
  4674. }
  4675. }
  4676. }
  4677. }
  4678. // implements ICoalsce
  4679. virtual void start()
  4680. {
  4681. threaded.init(this);
  4682. }
  4683. virtual void stop()
  4684. {
  4685. if (!stopped)
  4686. {
  4687. stopped = true;
  4688. sem.signal();
  4689. threaded.join();
  4690. }
  4691. }
  4692. };
  4693. /////////////////
  4694. class CUnlockCallback : implements IUnlockCallback
  4695. { // NB: unblock() always called 1st, then block()
  4696. StringAttr xpath;
  4697. ConnectionId connectionId;
  4698. CServerRemoteTree &tree;
  4699. bool lockedForWrite, unlocked;
  4700. public:
  4701. CUnlockCallback(const char *_xpath, ConnectionId _connectionId, CServerRemoteTree &_tree) : xpath(_xpath), connectionId(_connectionId), tree(_tree), lockedForWrite(false), unlocked(false) { }
  4702. void block()
  4703. {
  4704. assertex(unlocked);
  4705. unsigned got = msTick();
  4706. if (lockedForWrite)
  4707. CHECKEDWRITELOCKENTER(SDSManager->dataRWLock, readWriteTimeout);
  4708. else
  4709. CHECKEDREADLOCKENTER(SDSManager->dataRWLock, readWriteTimeout);
  4710. CHECKEDCRITENTER(SDSManager->lockCrit, fakeCritTimeout);
  4711. unlocked = false;
  4712. unsigned e=msTick()-got;
  4713. if (e>readWriteSlowTracing)
  4714. {
  4715. StringBuffer s("TIME: CUnlockCallback(write=");
  4716. s.append(lockedForWrite).append(",xpath=").append(xpath).append(", connectionId=").appendf("%" I64F "x", connectionId).append(") took ").append(e);
  4717. DBGLOG("%s", s.str());
  4718. if (readWriteStackTracing)
  4719. PrintStackReport();
  4720. }
  4721. if (tree.isOrphaned())
  4722. throw MakeSDSException(SDSExcpt_OrphanedNode, "Whilst completing lock to %s", xpath.get());
  4723. }
  4724. void unblock()
  4725. {
  4726. unlocked = true;
  4727. lockedForWrite = SDSManager->dataRWLock.queryWriteLocked();
  4728. CHECKEDCRITLEAVE(SDSManager->lockCrit);
  4729. if (lockedForWrite)
  4730. SDSManager->dataRWLock.unlockWrite();
  4731. else
  4732. SDSManager->dataRWLock.unlockRead();
  4733. }
  4734. };
  4735. class CStoreHelper : implements IStoreHelper, public CInterface
  4736. {
  4737. StringAttr storeName, location, remoteBackupLocation;
  4738. CStoreInfo storeInfo, deltaInfo;
  4739. unsigned configFlags;
  4740. const bool *abort;
  4741. unsigned delay, keepStores;
  4742. SessionId mySessId;
  4743. void clearStoreInfo(const char *base, const char *location, unsigned edition, CStoreInfo *storeInfo=NULL)
  4744. {
  4745. StringBuffer wcard;
  4746. wcard.append(base).append(".*");
  4747. StringBuffer path, filename;
  4748. filename.append(base).append('.').append(edition);
  4749. if (location) path.append(location);
  4750. path.append(filename);
  4751. if (!storeInfo || !storeInfo->cache)
  4752. {
  4753. Owned<IDirectoryIterator> dIter = createDirectoryIterator(location, wcard.str());
  4754. ForEach (*dIter)
  4755. {
  4756. for (;;)
  4757. {
  4758. try { dIter->query().remove(); break; }
  4759. catch (IException *e)
  4760. {
  4761. e->Release();
  4762. if (abort && *abort)
  4763. return;
  4764. MilliSleep(delay);
  4765. }
  4766. }
  4767. }
  4768. }
  4769. else
  4770. {
  4771. if (0 != stricmp(filename.str(), storeInfo->cache))
  4772. {
  4773. StringBuffer path(location);
  4774. path.append(storeInfo->cache);
  4775. OwnedIFile iFile = createIFile(path.str());
  4776. for (;;)
  4777. {
  4778. try { iFile->remove(); break; }
  4779. catch (IException *e)
  4780. {
  4781. e->Release();
  4782. if (abort && *abort)
  4783. return;
  4784. MilliSleep(delay);
  4785. }
  4786. }
  4787. }
  4788. storeInfo->cache.clear();
  4789. }
  4790. }
  4791. void writeStoreInfo(const char *base, const char *location, unsigned edition, unsigned *crc, CStoreInfo *storeInfo=NULL)
  4792. {
  4793. StringBuffer path, filename;
  4794. filename.append(base).append('.').append(edition);
  4795. if (location) path.append(location);
  4796. path.append(filename);
  4797. OwnedIFile iFile = createIFile(path.str());
  4798. OwnedIFileIO iFileIO = iFile->open(IFOcreate);
  4799. if (crc)
  4800. iFileIO->write(0, sizeof(unsigned), crc);
  4801. if (storeInfo)
  4802. storeInfo->cache.set(filename.str());
  4803. }
  4804. void updateStoreInfo(const char *base, const char *location, unsigned edition, unsigned *crc, CStoreInfo *storeInfo=NULL)
  4805. {
  4806. clearStoreInfo(base, location, edition, storeInfo);
  4807. writeStoreInfo(base, location, edition, crc, storeInfo);
  4808. }
  4809. void refreshInfo(CStoreInfo &info, const char *base)
  4810. {
  4811. OwnedIFile found;
  4812. OwnedIFileIO iFileIO;
  4813. if (info.cache.length()) // avoid directory lookup if poss.
  4814. {
  4815. StringBuffer path(location);
  4816. OwnedIFile iFile = createIFile(path.append(info.cache).str());
  4817. if (iFile->exists())
  4818. {
  4819. found.set(iFile);
  4820. try { iFileIO.setown(found->open(IFOread)); }
  4821. catch (IException *e)
  4822. {
  4823. e->Release();
  4824. }
  4825. }
  4826. }
  4827. if (!iFileIO)
  4828. {
  4829. StringBuffer wcard;
  4830. wcard.append(base).append(".*");
  4831. Owned<IDirectoryIterator> dIter = createDirectoryIterator(location, wcard.str());
  4832. unsigned totalDelays = 0;
  4833. for (;;)
  4834. {
  4835. if (dIter->first())
  4836. {
  4837. const char *name = dIter->query().queryFilename();
  4838. StringBuffer base, ext, fname;
  4839. splitFilename(name, NULL, NULL, &base, &ext);
  4840. fname.append(base).append(ext);
  4841. info.cache.set(fname.str());
  4842. found.set(&dIter->query());
  4843. try { iFileIO.setown(found->open(IFOread)); }
  4844. catch (IException *e)
  4845. {
  4846. e->Release();
  4847. }
  4848. if (iFileIO)
  4849. break;
  4850. }
  4851. totalDelays++;
  4852. if (totalDelays >= MAXDELAYS)
  4853. throw MakeSDSException(SDSExcpt_StoreInfoMissing, "store.<edition> file appears to be missing");
  4854. if (abort && *abort)
  4855. return;
  4856. MilliSleep(delay);
  4857. }
  4858. }
  4859. assertex(iFileIO);
  4860. StringBuffer tail, ext, fname;
  4861. splitFilename(found->queryFilename(), NULL, NULL, &tail, &ext);
  4862. fname.append(tail).append(ext);
  4863. const char *name = fname.str();
  4864. const char *editionBegin = name+strlen(base)+1;
  4865. info.edition = atoi(editionBegin);
  4866. if (iFileIO->size())
  4867. iFileIO->read(0, sizeof(unsigned), &info.crc);
  4868. else
  4869. info.crc = 0;
  4870. }
  4871. void refreshStoreInfo() { refreshInfo(storeInfo, "store"); }
  4872. void refreshDeltaInfo() { refreshInfo(deltaInfo, "store"); }
  4873. void checkInfo(const char *base, CStoreInfo &info)
  4874. {
  4875. StringBuffer wcard;
  4876. wcard.append(base).append(".*");
  4877. Owned<IDirectoryIterator> dIter = createDirectoryIterator(location, wcard.str());
  4878. if (!dIter->first())
  4879. updateStoreInfo(base, location, 0, NULL, &info);
  4880. else if (dIter->next())
  4881. throw MakeStringException(0, "Multiple store.X files - only one corresponding to latest dalisds<X>.xml should exist");
  4882. }
  4883. void renameDelta(unsigned oldEdition, unsigned newEdition, const char *path)
  4884. {
  4885. StringBuffer deltaName(path);
  4886. constructStoreName(DELTANAME, oldEdition, deltaName);
  4887. OwnedIFile oldDelta = createIFile(deltaName.str());
  4888. if (oldDelta->exists())
  4889. {
  4890. deltaName.clear();
  4891. constructStoreName(DELTANAME, newEdition, deltaName);
  4892. oldDelta->rename(deltaName.str());
  4893. }
  4894. }
  4895. struct CheckDeltaBlock
  4896. {
  4897. CheckDeltaBlock(CStoreHelper &_storeHelper) : storeHelper(_storeHelper)
  4898. {
  4899. bool created = false;
  4900. StringBuffer deltaIPStr(storeHelper.location);
  4901. OwnedIFile deltaIPIFile = createIFile(deltaIPStr.append(DELTAINPROGRESS).str());
  4902. activeDetachIPStr.append(DETACHINPROGRESS);
  4903. inactiveDetachIPStr.append(storeHelper.location).append('_').append(DETACHINPROGRESS);
  4904. detachIPIFile.setown(createIFile(inactiveDetachIPStr.str()));
  4905. OwnedIFileIO detachIPIO = detachIPIFile->open(IFOcreate);
  4906. detachIPIO->write(0, sizeof(storeHelper.mySessId), &storeHelper.mySessId);
  4907. detachIPIO.clear();
  4908. detachIPIFile->rename(activeDetachIPStr.str());
  4909. // check often do not wait any longer than necessary
  4910. unsigned d=0;
  4911. while (deltaIPIFile->exists())
  4912. {
  4913. if (0 == d++ % 50)
  4914. PROGLOG("Waiting for a saveDelta in progress");
  4915. MilliSleep(100);
  4916. }
  4917. }
  4918. ~CheckDeltaBlock() noexcept(false)
  4919. {
  4920. if (detachIPIFile)
  4921. {
  4922. unsigned a=0;
  4923. for (;;)
  4924. {
  4925. try { detachIPIFile->remove(); break; }
  4926. catch (IException *e)
  4927. {
  4928. EXCLOG(e, "removing detach file marker");
  4929. if (a++ > 10) throw;
  4930. e->Release();
  4931. }
  4932. MilliSleep(500);
  4933. }
  4934. }
  4935. }
  4936. private:
  4937. CStoreHelper &storeHelper;
  4938. OwnedIFile detachIPIFile;
  4939. StringBuffer activeDetachIPStr, inactiveDetachIPStr;
  4940. };
  4941. public:
  4942. IMPLEMENT_IINTERFACE;
  4943. CStoreHelper(const char *_storeName, const char *_location, const char *_remoteBackupLocation, unsigned _configFlags, unsigned _keepStores, unsigned _delay, const bool *_abort) : storeName(_storeName), location(_location), remoteBackupLocation(_remoteBackupLocation), configFlags(_configFlags), keepStores(_keepStores), delay(_delay), abort(_abort)
  4944. {
  4945. mySessId = daliClientActive()?myProcessSession():0;
  4946. if (!keepStores) keepStores = DEFAULT_KEEP_LASTN_STORES;
  4947. checkInfo("store", storeInfo);
  4948. checkInfo("store", deltaInfo);
  4949. if (0 == (SH_External & configFlags))
  4950. {
  4951. refreshStoreInfo();
  4952. unsigned edition = storeInfo.edition;
  4953. Owned<IDirectoryIterator> di = createDirectoryIterator(location, "dali*.xml");
  4954. ForEach (*di)
  4955. {
  4956. StringBuffer fname;
  4957. di->getName(fname);
  4958. if ('_' != fname.charAt(7)) // Unhelpful naming convention to differentiate store files from externals!
  4959. {
  4960. if (0 == memicmp("inc", fname.str()+4, 3) || 0 == memicmp("sds", fname.str()+4, 3))
  4961. {
  4962. const char *num = fname.str()+7;
  4963. const char *dot = (const char *)strchr(num, '.');
  4964. unsigned fileEdition = atoi_l(num, dot-num);
  4965. int d = (int)fileEdition-(int)edition;
  4966. if (edition != fileEdition && (d>=1 || d<(-(int)keepStores)))
  4967. {
  4968. IFile &file = di->query();
  4969. CDateTime dt;
  4970. dt.setNow();
  4971. StringBuffer newName(file.queryFilename());
  4972. newName.append('.');
  4973. unsigned i=newName.length();
  4974. dt.getString(newName); // base on date, incase any old copies.
  4975. for (;i<newName.length();i++)
  4976. if (newName.charAt(i)==':')
  4977. newName.setCharAt(i,'_');
  4978. newName.append(".unused");
  4979. PROGLOG("Detected spurious data file : '%s' - renaming to %s", file.queryFilename(), newName.str());
  4980. try
  4981. {
  4982. file.rename(newName.str());
  4983. }
  4984. catch (IException *e) // safe to ignore these (should e.g. files be in use).
  4985. {
  4986. EXCLOG(e, NULL);
  4987. e->Release();
  4988. }
  4989. }
  4990. }
  4991. }
  4992. }
  4993. StringBuffer dst(location);
  4994. addPathSepChar(dst);
  4995. dst.append(DEBUG_DIR);
  4996. addPathSepChar(dst);
  4997. OwnedIFile dFile = createIFile(dst.str());
  4998. Owned<IDirectoryIterator> dIter = dFile->directoryFiles();
  4999. ForEach(*dIter)
  5000. dIter->query().remove();
  5001. }
  5002. }
  5003. virtual StringBuffer &getDetachedDeltaName(StringBuffer &detachName)
  5004. {
  5005. refreshDeltaInfo();
  5006. constructStoreName(DELTADETACHED, deltaInfo.edition, detachName);
  5007. return detachName;
  5008. }
  5009. virtual bool loadDelta(const char *filename, IFile *iFile, IPropertyTree *root)
  5010. {
  5011. Owned<IFileIO> iFileIO = iFile->open(IFOread);
  5012. if (!iFileIO) // no delta to load
  5013. return true;
  5014. MemoryBuffer tmp;
  5015. char *ptr = (char *) tmp.reserveTruncate(strlen(deltaHeader));
  5016. unsigned embeddedCrc = 0;
  5017. offset_t pos = 0;
  5018. bool hasCrcHeader = false; // check really only needed for deltas proceeding CRC header
  5019. if (strlen(deltaHeader) == iFileIO->read(0, strlen(deltaHeader), ptr))
  5020. {
  5021. if (0 == memicmp(deltaHeader, ptr, 5))
  5022. {
  5023. pos = deltaHeaderSizeStart;
  5024. hasCrcHeader = true;
  5025. embeddedCrc = (unsigned)atoi64_l(ptr+deltaHeaderCrcOff, 10);
  5026. if (0 == memicmp(deltaHeader+deltaHeaderSizeStart, ptr+deltaHeaderSizeStart, 6)) // has <SIZE> too
  5027. {
  5028. pos = strlen(deltaHeader);
  5029. offset_t lastGood;
  5030. if (sscanf(ptr+deltaHeaderSizeOff, "%" I64F "X", &lastGood))
  5031. {
  5032. offset_t fSize = iFileIO->size();
  5033. if (fSize > lastGood)
  5034. {
  5035. offset_t diff = fSize - lastGood;
  5036. LOG(MCoperatorError, unknownJob, "Delta file '%s', has %" I64F "d bytes of trailing data (possible power loss during save?), file size: %" I64F "d, last committed size: %" I64F "d", filename, diff, fSize, lastGood);
  5037. LOG(MCoperatorError, unknownJob, "Resetting delta file '%s' to size: %" I64F "d", filename, lastGood);
  5038. iFileIO->close();
  5039. backup(filename);
  5040. iFileIO.setown(iFile->open(IFOreadwrite));
  5041. iFileIO->setSize(lastGood);
  5042. iFileIO->close();
  5043. iFileIO.setown(iFile->open(IFOread));
  5044. }
  5045. }
  5046. }
  5047. }
  5048. }
  5049. OwnedIFileIOStream iFileIOStream = createIOStream(iFileIO);
  5050. iFileIOStream->seek(pos, IFSbegin);
  5051. Owned<ICrcIOStream> crcPipeStream = createCrcPipeStream(iFileIOStream); // crc *rest* of stream
  5052. Owned<IIOStream> ios = createBufferedIOStream(crcPipeStream);
  5053. bool noErrors;
  5054. Owned<IException> deltaE;
  5055. noErrors = applyXmlDeltas(*root, *ios, 0 == (SH_RecoverFromIncErrors & configFlags));
  5056. if (noErrors && hasCrcHeader)
  5057. {
  5058. unsigned crc = crcPipeStream->queryCrc();
  5059. if (embeddedCrc != crc)
  5060. {
  5061. noErrors = false;
  5062. StringBuffer s;
  5063. LOG(MCoperatorWarning, unknownJob, "%s", s.append("Delta '").append(filename).append("' crc mismatch").str());
  5064. }
  5065. }
  5066. return noErrors;
  5067. }
  5068. virtual bool loadDeltas(IPropertyTree *root, bool *errors)
  5069. {
  5070. bool res = false;
  5071. if (errors) *errors = false;
  5072. StringBuffer deltaFilename(location);
  5073. constructStoreName(DELTANAME, storeInfo.edition, deltaFilename);
  5074. StringBuffer detachPath(location);
  5075. OwnedIFile detachedDeltaIFile = createIFile(getDetachedDeltaName(detachPath).str());
  5076. bool detached = detachedDeltaIFile->exists();
  5077. OwnedIFile deltaIFile = createIFile(deltaFilename.str());
  5078. for (;;)
  5079. {
  5080. StringAttr filename;
  5081. IFile *iFile;
  5082. if (detached)
  5083. {
  5084. iFile = detachedDeltaIFile;
  5085. filename.set(iFile->queryFilename());
  5086. }
  5087. else
  5088. {
  5089. iFile = deltaIFile;
  5090. filename.set(iFile->queryFilename());
  5091. if (!iFile->exists())
  5092. break;
  5093. }
  5094. PROGLOG("Loading delta: %s", filename.get());
  5095. bool noError;
  5096. Owned<IException> deltaE;
  5097. try { noError = loadDelta(filename, iFile, root); }
  5098. catch (IException *e) { deltaE.setown(e); noError = false; }
  5099. if (!noError)
  5100. {
  5101. backup(filename);
  5102. if (errors) *errors = true;
  5103. if (deltaE)
  5104. throw LINK(deltaE);
  5105. }
  5106. res = true;
  5107. if (detached)
  5108. detached = false;
  5109. else
  5110. break;
  5111. }
  5112. return res;
  5113. }
  5114. virtual bool detachCurrentDelta()
  5115. {
  5116. StringBuffer deltaFilename(location);
  5117. getCurrentDeltaFilename(deltaFilename);
  5118. refreshStoreInfo();
  5119. bool res = false;
  5120. try
  5121. {
  5122. CheckDeltaBlock cD(*this);
  5123. OwnedIFile deltaIFile = createIFile(deltaFilename.str());
  5124. if (deltaIFile->exists())
  5125. {
  5126. #ifdef NODELETE
  5127. StringBuffer detachPath(location);
  5128. getDetachedDeltaName(detachPath);
  5129. if (retryCopy(deltaFilename.str(), detachPath.str(), 5, delay))
  5130. #else
  5131. StringBuffer detachName;
  5132. getDetachedDeltaName(detachName);
  5133. if (retryRename(deltaFilename.str(), detachName.str(), 5, delay))
  5134. #endif
  5135. res = true;
  5136. }
  5137. if (remoteBackupLocation.length())
  5138. {
  5139. deltaFilename.clear().append(remoteBackupLocation);
  5140. getCurrentDeltaFilename(deltaFilename);
  5141. OwnedIFile iFile = createIFile(deltaFilename);
  5142. if (iFile->exists())
  5143. {
  5144. StringBuffer detachName;
  5145. getDetachedDeltaName(detachName);
  5146. iFile->rename(detachName.str());
  5147. }
  5148. }
  5149. }
  5150. catch (IException *e)
  5151. {
  5152. LOG(MCoperatorError, unknownJob, e, "detachCurrentDelta");
  5153. e->Release();
  5154. }
  5155. return res;
  5156. }
  5157. virtual void saveStore(IPropertyTree *root, unsigned *_newEdition, bool currentEdition=false)
  5158. {
  5159. LOG(MCdebugInfo(100), unknownJob, "Saving store");
  5160. refreshStoreInfo();
  5161. unsigned edition = storeInfo.edition;
  5162. unsigned newEdition = currentEdition?edition:nextEditionN(edition);
  5163. bool done = false;
  5164. try
  5165. {
  5166. unsigned crc = 0;
  5167. StringBuffer tmpStoreName;
  5168. OwnedIFileIO iFileIOTmpStore = createUniqueFile(location, TMPSAVENAME, NULL, tmpStoreName);
  5169. OwnedIFile iFileTmpStore = createIFile(tmpStoreName);
  5170. try
  5171. {
  5172. OwnedIFileIOStream fstream = createIOStream(iFileIOTmpStore);
  5173. Owned<ICrcIOStream> crcPipeStream = createCrcPipeStream(fstream);
  5174. Owned<IIOStream> ios = createBufferedIOStream(crcPipeStream);
  5175. #ifdef _DEBUG
  5176. toXML(root, *ios); // formatted (default)
  5177. #else
  5178. toXML(root, *ios, 0, 0);
  5179. #endif
  5180. ios.clear();
  5181. fstream.clear();
  5182. crc = crcPipeStream->queryCrc();
  5183. crcPipeStream.clear();
  5184. iFileIOTmpStore.clear();
  5185. }
  5186. catch (IException *e)
  5187. {
  5188. LOG(MCoperatorError, unknownJob, e, "Exception(1) - Error saving store file");
  5189. iFileIOTmpStore.clear();
  5190. iFileTmpStore->remove();
  5191. throw;
  5192. }
  5193. StringBuffer newStoreName;
  5194. constructStoreName(storeName, newEdition, newStoreName);
  5195. StringBuffer newStoreNamePath(location);
  5196. newStoreNamePath.append(newStoreName);
  5197. refreshStoreInfo();
  5198. if (storeInfo.edition != edition)
  5199. {
  5200. WARNLOG("Another process has updated the edition whilst saving the store: %s", newStoreNamePath.str());
  5201. iFileTmpStore->remove();
  5202. return;
  5203. }
  5204. try
  5205. {
  5206. OwnedIFile newStoreIFile = createIFile(newStoreNamePath.str());
  5207. newStoreIFile->remove();
  5208. iFileTmpStore->rename(newStoreName.str());
  5209. }
  5210. catch (IException *e)
  5211. {
  5212. StringBuffer errMsg;
  5213. EXCLOG(e, errMsg.append("Failed to rename new store to : ").append(newStoreNamePath).append(". Has already been created by another process?").str());
  5214. e->Release();
  5215. iFileTmpStore->remove();
  5216. return;
  5217. }
  5218. if (0 != (SH_CheckNewDelta & configFlags))
  5219. {
  5220. CheckDeltaBlock cD(*this);
  5221. try { renameDelta(edition, newEdition, location); }
  5222. catch (IException *e)
  5223. {
  5224. StringBuffer s("Exception(2) - Error saving store file");
  5225. LOG(MCoperatorError, unknownJob, e, s.str());
  5226. e->Release();
  5227. return;
  5228. }
  5229. if (remoteBackupLocation.length())
  5230. {
  5231. try { renameDelta(edition, newEdition, remoteBackupLocation); }
  5232. catch (IException *e)
  5233. {
  5234. LOG(MCoperatorError, unknownJob, e, "Failure handling backup");
  5235. e->Release();
  5236. }
  5237. }
  5238. clearStoreInfo("store", location, 0, NULL);
  5239. writeStoreInfo("store", location, newEdition, &crc, &storeInfo);
  5240. }
  5241. else
  5242. {
  5243. clearStoreInfo("store", location, 0, NULL);
  5244. writeStoreInfo("store", location, newEdition, &crc, &storeInfo);
  5245. }
  5246. try
  5247. {
  5248. if (remoteBackupLocation.length())
  5249. {
  5250. PROGLOG("Copying store to backup location");
  5251. StringBuffer rL(remoteBackupLocation);
  5252. constructStoreName(storeName, newEdition, rL);
  5253. copyFile(rL.str(), newStoreNamePath.str());
  5254. clearStoreInfo("store", remoteBackupLocation, 0, NULL);
  5255. writeStoreInfo("store", remoteBackupLocation, newEdition, &crc, &storeInfo);
  5256. PROGLOG("Copy done");
  5257. }
  5258. }
  5259. catch (IException *e)
  5260. {
  5261. StringBuffer s;
  5262. LOG(MCoperatorError, unknownJob, e, s.append("Failure to backup dali to remote location: ").append(remoteBackupLocation));
  5263. e->Release();
  5264. }
  5265. if (_newEdition)
  5266. *_newEdition = newEdition;
  5267. done = true;
  5268. LOG(MCdebugInfo(100), unknownJob, "Store saved");
  5269. }
  5270. catch (IException *e)
  5271. {
  5272. StringBuffer s("Exception(3) - Error saving store file");
  5273. LOG(MCoperatorError, unknownJob, e, s.str());
  5274. e->Release();
  5275. }
  5276. if (done)
  5277. {
  5278. #ifndef NODELETE
  5279. unsigned toDeleteEdition = prevEditionN(edition, keepStores+(currentEdition?1:0));
  5280. StringBuffer filename(location);
  5281. constructStoreName(storeName, toDeleteEdition, filename);
  5282. OwnedIFile iFile = createIFile(filename.str());
  5283. if (iFile->exists())
  5284. PROGLOG("Deleting old store: %s", filename.str());
  5285. removeDaliFile(location, storeName, toDeleteEdition);
  5286. removeDaliFile(location, DELTANAME, toDeleteEdition);
  5287. removeDaliFile(location, DELTADETACHED, toDeleteEdition);
  5288. if (remoteBackupLocation)
  5289. {
  5290. removeDaliFile(remoteBackupLocation, storeName, toDeleteEdition);
  5291. removeDaliFile(remoteBackupLocation, DELTANAME, toDeleteEdition);
  5292. removeDaliFile(remoteBackupLocation, DELTADETACHED, toDeleteEdition);
  5293. }
  5294. #endif
  5295. }
  5296. }
  5297. virtual unsigned queryCurrentEdition()
  5298. {
  5299. refreshStoreInfo();
  5300. return storeInfo.edition;
  5301. }
  5302. virtual StringBuffer &getCurrentStoreFilename(StringBuffer &res, unsigned *crc=NULL)
  5303. {
  5304. refreshStoreInfo();
  5305. constructStoreName(storeName, storeInfo.edition, res);
  5306. if (crc)
  5307. * crc = storeInfo.crc;
  5308. return res;
  5309. }
  5310. virtual StringBuffer &getCurrentDeltaFilename(StringBuffer &res, unsigned *crc=NULL)
  5311. {
  5312. refreshDeltaInfo();
  5313. constructStoreName(DELTANAME, deltaInfo.edition, res);
  5314. if (crc)
  5315. * crc = deltaInfo.crc; // TBD, combine into store.<edition>.<store_crc>.<delta_crc>
  5316. return res;
  5317. }
  5318. virtual StringBuffer &getCurrentStoreInfoFilename(StringBuffer &res)
  5319. {
  5320. refreshStoreInfo();
  5321. res.append(storeInfo.cache);
  5322. return res;
  5323. }
  5324. virtual void backup(const char *filename)
  5325. {
  5326. try
  5327. {
  5328. unsigned crc = getFileCRC(filename);
  5329. StringBuffer dst(location);
  5330. if (dst.length())
  5331. addPathSepChar(dst);
  5332. dst.append(DEBUG_DIR);
  5333. addPathSepChar(dst);
  5334. recursiveCreateDirectoryForFile(dst.str());
  5335. OwnedIFile dFile = createIFile(dst.str());
  5336. Owned<IDirectoryIterator> dIter = dFile->directoryFiles();
  5337. unsigned debugFiles = 0;
  5338. ForEach (*dIter) debugFiles++;
  5339. if (debugFiles >= 10) return;
  5340. StringBuffer fname(filename);
  5341. getFileNameOnly(fname);
  5342. dst.append(fname.str()).append('.').append(crc);
  5343. OwnedIFile backupIFile = createIFile(dst.str());
  5344. if (!backupIFile->exists()) // a copy could already have been made
  5345. {
  5346. PROGLOG("Backing up: %s", filename);
  5347. OwnedIFile iFile = createIFile(filename);
  5348. copyFile(backupIFile, iFile);
  5349. PROGLOG("Backup made: %s", dst.str());
  5350. }
  5351. }
  5352. catch (IException *e)
  5353. {
  5354. StringBuffer tmp;
  5355. EXCLOG(e, tmp.append("Failed to take backup of: ").append(filename).str());
  5356. e->Release();
  5357. }
  5358. }
  5359. virtual StringBuffer &getPrimaryLocation(StringBuffer &_location)
  5360. {
  5361. _location.append(location);
  5362. return _location;
  5363. }
  5364. virtual StringBuffer &getBackupLocation(StringBuffer &backupLocation)
  5365. {
  5366. backupLocation.append(remoteBackupLocation);
  5367. return backupLocation;
  5368. }
  5369. friend struct CheckDeltaBlock;
  5370. };
  5371. IStoreHelper *createStoreHelper(const char *storeName, const char *location, const char *remoteBackupLocation, unsigned configFlags, unsigned keepStores, unsigned delay, const bool *abort)
  5372. {
  5373. if (!storeName) storeName = "dalisds";
  5374. return new CStoreHelper(storeName, location, remoteBackupLocation, configFlags, keepStores, delay, abort);
  5375. }
  5376. ///////////////
  5377. #ifdef _MSC_VER
  5378. #pragma warning (push)
  5379. #pragma warning (disable : 4355) // 'this' : used in base member initializer list
  5380. #endif
  5381. CCovenSDSManager::CCovenSDSManager(ICoven &_coven, IPropertyTree &_config, const char *_dataPath)
  5382. : coven(_coven), config(_config), server(*this), dataPath(_dataPath), backupHandler(_config)
  5383. {
  5384. config.Link();
  5385. restartOnError = config.getPropBool("@restartOnUnhandled");
  5386. root = NULL;
  5387. writeTransactions=0;
  5388. externalEnvironment = false;
  5389. ignoreExternals=false;
  5390. unsigned initNodeTableSize = queryCoven().getInitSDSNodes();
  5391. allNodes.ensure(initNodeTableSize?initNodeTableSize:INIT_NODETABLE_SIZE);
  5392. externalSizeThreshold = config.getPropInt("@externalSizeThreshold", DEFAULT_EXTERNAL_SIZE_THRESHOLD);
  5393. remoteBackupLocation.set(config.queryProp("@remoteBackupLocation"));
  5394. nextExternal = 1;
  5395. if (0 == coven.getServerRank())
  5396. {
  5397. if (coven.size() > 1)
  5398. {
  5399. unsigned s;
  5400. for (s=1; s<coven.size(); s++)
  5401. {
  5402. CMessageBuffer mb;
  5403. __int64 dummy=0;
  5404. mb.append(dummy);
  5405. coven.sendRecv(mb, s, MPTAG_DALI_SDS_REQUEST);
  5406. bool success;
  5407. mb.read(success);
  5408. assertex(success);
  5409. }
  5410. }
  5411. }
  5412. else
  5413. {
  5414. CMessageBuffer mb;
  5415. #ifdef TRACE_QWAITING
  5416. unsigned waiting = coven.probe(0,MPTAG_DALI_SDS_REQUEST,NULL);
  5417. if ((waiting!=0)&&(waiting%10==0))
  5418. DBGLOG("QPROBE: MPTAG_DALI_SDS_REQUEST.2 has %d waiting",waiting);
  5419. #endif
  5420. if (coven.recv(mb, 0, MPTAG_DALI_SDS_REQUEST, NULL))
  5421. {
  5422. __int64 dummy;
  5423. mb.read(dummy);
  5424. mb.clear().append(true); // denote success
  5425. coven.reply(mb);
  5426. }
  5427. else
  5428. assertex(false);
  5429. }
  5430. registerSubscriptionManager(SDS_PUBLISHER, this);
  5431. connectionSubscriptionManager.setown(new CConnectionSubscriptionManager());
  5432. nodeSubscriptionManager.setown(new CNodeSubscriptionManager(*this));
  5433. registerSubscriptionManager(SDSCONN_PUBLISHER, connectionSubscriptionManager.get());
  5434. registerSubscriptionManager(SDSNODE_PUBLISHER, nodeSubscriptionManager);
  5435. // add external handlers
  5436. Owned<CXMLFileExternal> xmlExternalHandler = new CXMLFileExternal(dataPath, backupHandler);
  5437. externalHandlers.replace(* new CExternalHandlerMapping(EF_XML, *xmlExternalHandler));
  5438. Owned<CLegacyBinaryFileExternal> legacyBinaryExternalHandler = new CLegacyBinaryFileExternal(dataPath, backupHandler);
  5439. externalHandlers.replace(* new CExternalHandlerMapping(EF_LegacyBinaryValue, *legacyBinaryExternalHandler));
  5440. Owned<CBinaryFileExternal> binaryExternalHandler = new CBinaryFileExternal(dataPath, backupHandler);
  5441. externalHandlers.replace(* new CExternalHandlerMapping(EF_BinaryValue, *binaryExternalHandler));
  5442. properties.setown(createPTree("Properties"));
  5443. IPropertyTree *clientProps = properties->setPropTree("Client", config.hasProp("Client") ? config.getPropTree("Client") : createPTree());
  5444. clientProps->setPropBool("@serverIterAvailable", true);
  5445. clientProps->setPropBool("@useAppendOpt", true);
  5446. clientProps->setPropBool("@serverGetIdsAvailable", true);
  5447. IPropertyTree *throttle = clientProps->setPropTree("Throttle", createPTree());
  5448. throttle->setPropInt("@limit", CLIENT_THROTTLE_LIMIT);
  5449. throttle->setPropInt("@delay", CLIENT_THROTTLE_DELAY);
  5450. // NB: dataPath is assumed to be local
  5451. RemoteFilename rfn;
  5452. if (dataPath.length())
  5453. rfn.setLocalPath(dataPath);
  5454. else
  5455. {
  5456. char cwd[1024];
  5457. if (!GetCurrentDirectory(1024, cwd)) {
  5458. ERRLOG("CCovenSDSManager: Current directory path too big, setting local path to null");
  5459. cwd[0] = 0;
  5460. }
  5461. rfn.setLocalPath(cwd);
  5462. }
  5463. unsigned keepLastN = config.getPropInt("@keepStores", DEFAULT_KEEP_LASTN_STORES);
  5464. StringBuffer path;
  5465. rfn.getRemotePath(path);
  5466. properties->setProp("@dataPathUrl", path.str());
  5467. properties->setPropInt("@keepStores", keepLastN);
  5468. if (remoteBackupLocation.length())
  5469. {
  5470. properties->setProp("@backupPathUrl", remoteBackupLocation.get());
  5471. backupHandler.init(remoteBackupLocation, config.getPropBool("@asyncBackup", true));
  5472. }
  5473. const char *storeName = config.queryProp("@store");
  5474. if (!storeName) storeName = "dalisds";
  5475. #if 1 // legacy
  5476. StringBuffer tail, ext;
  5477. splitFilename(storeName, NULL, NULL, &tail, &ext);
  5478. if (0 == stricmp(".xml", ext.str()))
  5479. {
  5480. config.setProp("@store", tail.str());
  5481. storeName = tail.str();
  5482. }
  5483. #endif
  5484. StringBuffer tmp(dataPath);
  5485. OwnedIFile inProgressIFile = createIFile(tmp.append(DELTAINPROGRESS).str());
  5486. inProgressIFile->remove();
  5487. OwnedIFile detachIPIFile = createIFile(tmp.append(DETACHINPROGRESS).str());
  5488. detachIPIFile->remove();
  5489. unsigned configFlags = config.getPropBool("@recoverFromIncErrors", true) ? SH_RecoverFromIncErrors : 0;
  5490. configFlags |= config.getPropBool("@backupErrorFiles", true) ? SH_BackupErrorFiles : 0;
  5491. iStoreHelper = createStoreHelper(storeName, dataPath, remoteBackupLocation, configFlags, keepLastN, 100, &server.queryStopped());
  5492. doTimeComparison = false;
  5493. if (config.getPropBool("@lightweightCoalesce", true))
  5494. coalesce.setown(new CLightCoalesceThread(config, iStoreHelper));
  5495. }
  5496. #ifdef _MSC_VER
  5497. #pragma warning (pop)
  5498. #endif
  5499. CCovenSDSManager::~CCovenSDSManager()
  5500. {
  5501. backupHandler.stop();
  5502. if (unhandledThread) unhandledThread->join();
  5503. if (coalesce) coalesce->stop();
  5504. scanNotifyPool.clear();
  5505. notifyPool.clear();
  5506. connections.kill();
  5507. ::Release(iStoreHelper);
  5508. if (!config.getPropBool("@leakStore", true)) // intentional default leak of time consuming deconstruction of tree
  5509. ::Release(root);
  5510. else
  5511. enableMemLeakChecking(false);
  5512. config.Release();
  5513. }
  5514. bool compareFiles(IFile *file1, IFile *file2, bool compareTimes=true)
  5515. {
  5516. if (file1->exists())
  5517. {
  5518. if (file2->exists())
  5519. {
  5520. if (file1->size() == file2->size())
  5521. {
  5522. if (!compareTimes) return true;
  5523. CDateTime modifiedTimeBackup;
  5524. file1->getTime(NULL, &modifiedTimeBackup, NULL);
  5525. CDateTime modifiedTime;
  5526. file2->getTime(NULL, &modifiedTime, NULL);
  5527. if (0 == modifiedTimeBackup.compare(modifiedTime, false))
  5528. return true;
  5529. }
  5530. }
  5531. }
  5532. else
  5533. return !file2->exists();
  5534. return false;
  5535. }
  5536. void CCovenSDSManager::validateDeltaBackup()
  5537. {
  5538. // check consistency of delta
  5539. StringBuffer deltaFilename(dataPath);
  5540. iStoreHelper->getCurrentDeltaFilename(deltaFilename);
  5541. OwnedIFile iFileDelta = createIFile(deltaFilename.str());
  5542. deltaFilename.clear().append(remoteBackupLocation);
  5543. iStoreHelper->getCurrentDeltaFilename(deltaFilename);
  5544. OwnedIFile iFileDeltaBackup = createIFile(deltaFilename.str());
  5545. if (!compareFiles(iFileDeltaBackup, iFileDelta, false))
  5546. {
  5547. WARNLOG("Delta file backup doesn't exist or differs, filename=%s", deltaFilename.str());
  5548. copyFile(iFileDeltaBackup, iFileDelta);
  5549. }
  5550. }
  5551. void CCovenSDSManager::validateBackup()
  5552. {
  5553. // check consistency of store info file.
  5554. StringBuffer storeInfoFilename(dataPath);
  5555. iStoreHelper->getCurrentStoreInfoFilename(storeInfoFilename);
  5556. OwnedIFile infoIFile = createIFile(storeInfoFilename.str());
  5557. if (infoIFile->exists())
  5558. {
  5559. StringBuffer rL(remoteBackupLocation);
  5560. iStoreHelper->getCurrentStoreInfoFilename(rL);
  5561. copyFile(rL.str(), storeInfoFilename.str());
  5562. }
  5563. // check consistency of delta
  5564. validateDeltaBackup();
  5565. // ensure there's a copy of the primary store present at startup.
  5566. StringBuffer storeFilename(dataPath);
  5567. iStoreHelper->getCurrentStoreFilename(storeFilename);
  5568. OwnedIFile iFileStore = createIFile(storeFilename.str());
  5569. storeFilename.clear().append(remoteBackupLocation);
  5570. iStoreHelper->getCurrentStoreFilename(storeFilename);
  5571. OwnedIFile iFileBackupStore = createIFile(storeFilename.str());
  5572. if (!compareFiles(iFileBackupStore, iFileStore))
  5573. {
  5574. WARNLOG("Store backup file doesn't exist or differs, filename=%s", storeFilename.str());
  5575. copyFile(iFileBackupStore, iFileStore);
  5576. }
  5577. }
  5578. static int uint64compare(unsigned __int64 const *i1, unsigned __int64 const *i2)
  5579. {
  5580. if (*i1==*i2) return 0;
  5581. if (*i1<*i2) return -1;
  5582. return 1;
  5583. }
  5584. class CLegacyFmtItem : public CInterface
  5585. {
  5586. public:
  5587. CLegacyFmtItem(const char *_name, const char *_ext, unsigned __int64 _num) : name(_name), ext(_ext), num(_num) { }
  5588. StringAttr name, ext;
  5589. unsigned __int64 num;
  5590. };
  5591. static int extNcompareFunc(CInterface * const *_itm1, CInterface * const *_itm2)
  5592. {
  5593. CLegacyFmtItem *itm1 = (CLegacyFmtItem *)*_itm1;
  5594. CLegacyFmtItem *itm2 = (CLegacyFmtItem *)*_itm2;
  5595. if (itm1->num==itm2->num) return 0;
  5596. if (itm1->num<itm2->num) return -1;
  5597. return 1;
  5598. }
  5599. void CCovenSDSManager::loadStore(const char *storeName, const bool *abort)
  5600. {
  5601. if (root) root->Release();
  5602. class CNodeCreate : implements IPTreeNodeCreator, public CInterface
  5603. {
  5604. public:
  5605. IMPLEMENT_IINTERFACE;
  5606. virtual IPropertyTree *create(const char *tag) { return createServerTree(tag); }
  5607. } nodeCreator;
  5608. class CSDSTreeMaker : public CPTreeMaker
  5609. {
  5610. public:
  5611. CSDSTreeMaker(IPTreeNodeCreator *nodeCreator) : CPTreeMaker(ipt_none, nodeCreator) { }
  5612. virtual void endNode(const char *tag, unsigned length, const void *value, bool binary, offset_t endOffset)
  5613. {
  5614. IPropertyTree *node = queryCurrentNode();
  5615. CPTreeMaker::endNode(tag, length, value, binary, endOffset);
  5616. if (((CServerRemoteTree *)node)->testExternalCandidate())
  5617. convertQueue.append(*(CServerRemoteTree *)node);
  5618. }
  5619. ICopyArrayOf<CServerRemoteTree> convertQueue;
  5620. } treeMaker(&nodeCreator);
  5621. Owned<IPropertyTree> oldEnvironment;
  5622. try
  5623. {
  5624. bool saveNeeded = false;
  5625. if (!storeName)
  5626. storeName = config.queryProp("@store");
  5627. unsigned crc = 0;
  5628. StringBuffer storeFilename(dataPath);
  5629. iStoreHelper->getCurrentStoreFilename(storeFilename, &crc);
  5630. LOG(MCdebugInfo(100), unknownJob, "loading store %d, storedCrc=%x", iStoreHelper->queryCurrentEdition(), crc);
  5631. root = (CServerRemoteTree *)::loadStore(storeFilename.str(), &treeMaker, crc, false, abort);
  5632. if (!root)
  5633. {
  5634. StringBuffer s(storeName);
  5635. LOG(MCdebugInfo(100), unknownJob, "Store %d does not exist, creating new store", iStoreHelper->queryCurrentEdition());
  5636. root = new CServerRemoteTree("SDS");
  5637. }
  5638. bool errors;
  5639. Owned<IException> deltaE;
  5640. try { iStoreHelper->loadDeltas(root, &errors); }
  5641. catch (IException *e) { deltaE.setown(e); errors = true; }
  5642. if (errors && config.getPropBool("@backupErrorFiles", true))
  5643. {
  5644. iStoreHelper->backup(storeFilename.str());
  5645. if (deltaE.get())
  5646. throw LINK(deltaE);
  5647. }
  5648. LOG(MCdebugInfo(100), unknownJob, "store loaded");
  5649. const char *environment = config.queryProp("@environment");
  5650. if (environment && *environment)
  5651. {
  5652. LOG(MCdebugInfo(100), unknownJob, "loading external Environment from: %s", environment);
  5653. Owned<IFile> envFile = createIFile(environment);
  5654. if (!envFile->exists())
  5655. throw MakeStringException(0, "'%s' does not exist", environment);
  5656. OwnedIFileIO iFileIO = envFile->open(IFOread);
  5657. if (!iFileIO)
  5658. throw MakeStringException(0, "Failed to open '%s'", environment);
  5659. Owned<IPropertyTree> envTree = createPTreeFromXMLFile(environment);
  5660. if (0 != stricmp("Environment", envTree->queryName()))
  5661. throw MakeStringException(0, "External environment file '%s', has '%s' as root, expecting a 'Environment' xml node.", environment, envTree->queryName());
  5662. Owned <IMPServer> thisDali = getMPServer();
  5663. assertex(thisDali);
  5664. IPropertyTree *thisDaliInfo = findDaliProcess(envTree, thisDali->queryMyNode()->endpoint());
  5665. assertex(thisDaliInfo);
  5666. const char *daliName = thisDaliInfo->queryProp("@name");
  5667. if (daliName)
  5668. {
  5669. VStringBuffer xpath("Software/DaliServerPlugin[@daliServers='%s']", daliName);
  5670. Owned<IPropertyTreeIterator> plugins = envTree->getElements(xpath);
  5671. ForEach(*plugins)
  5672. {
  5673. Owned<IPluggableFactory> factory = loadPlugin(&plugins->query());
  5674. assertex (factory);
  5675. if (!factory->initializeStore())
  5676. throw MakeStringException(0, "Failed to initialize plugin store '%s'", plugins->query().queryProp("@type"));
  5677. }
  5678. }
  5679. oldEnvironment.setown(root->getPropTree("Environment"));
  5680. root->removeTree(oldEnvironment);
  5681. root->addPropTree("Environment", envTree.getClear());
  5682. externalEnvironment = true;
  5683. }
  5684. UInt64Array refExts;
  5685. PROGLOG("Scanning store for external references");
  5686. Owned<IPropertyTreeIterator> rootIter = root->getElements("//*");
  5687. ForEach (*rootIter)
  5688. {
  5689. IPropertyTree &tree = rootIter->query();
  5690. __int64 index = tree.getPropInt64(EXT_ATTR);
  5691. if (index)
  5692. refExts.append(index);
  5693. }
  5694. PROGLOG("External reference count = %d", refExts.ordinality());
  5695. refExts.sort(uint64compare);
  5696. if (refExts.ordinality())
  5697. nextExternal = refExts.tos()+1; // JCSMORE could keep array and fill gaps
  5698. // build list of primary, backup and legacy type external files
  5699. CIArrayOf<CLegacyFmtItem> legacyFmts;
  5700. UInt64Array primaryExts, backupExts;
  5701. unsigned l = strlen(EXTERNAL_NAME_PREFIX);
  5702. bool primary = true;
  5703. Owned<IDirectoryIterator> di = createDirectoryIterator(dataPath);
  5704. for (;;)
  5705. {
  5706. try
  5707. {
  5708. ForEach(*di)
  5709. {
  5710. StringBuffer fname;
  5711. di->getName(fname);
  5712. if (fname.length() > l && 0 == memicmp(EXTERNAL_NAME_PREFIX, fname.str(), l))
  5713. {
  5714. StringBuffer name, ext;
  5715. splitFilename(fname, NULL, NULL, &name, &ext);
  5716. if (ext.length())
  5717. {
  5718. ext.remove(0, 1); // delete '.'
  5719. if (0 != stricmp(EF_BinaryValue, ext.str()))
  5720. {
  5721. unsigned __int64 num = atoi64(name.str()+l);
  5722. legacyFmts.append(* new CLegacyFmtItem(name.str(), ext.str(), num));
  5723. }
  5724. #ifndef _WIN32 // win32 files are case insensitive, so no point in this step.
  5725. StringBuffer lfName(fname);
  5726. lfName.toLowerCase();
  5727. if (0 != strcmp(fname.str(), lfName.str()))
  5728. di->query().rename(lfName.str());
  5729. #endif
  5730. unsigned __int64 extN = atoi64(name.str()+l);
  5731. if (primary)
  5732. primaryExts.append(extN);
  5733. else
  5734. backupExts.append(extN);
  5735. }
  5736. }
  5737. }
  5738. }
  5739. catch (IException *e)
  5740. {
  5741. if (primary)
  5742. throw;
  5743. EXCLOG(e, NULL);
  5744. e->Release();
  5745. }
  5746. if (!primary || 0 == remoteBackupLocation.length())
  5747. break;
  5748. di.setown(createDirectoryIterator(remoteBackupLocation));
  5749. primary = false;
  5750. }
  5751. primaryExts.sort(uint64compare);
  5752. backupExts.sort(uint64compare);
  5753. // Compare list of primary/backup externals against referenced externals and issue warnings
  5754. // Copy over any if reference and present either only in primary or backup
  5755. IExternalHandler *extHandler = queryExternalHandler(EF_BinaryValue);
  5756. primary = true;
  5757. UInt64Array missingPrimarys;
  5758. for (;;)
  5759. {
  5760. UInt64Array &fileExts = primary ? primaryExts : backupExts;
  5761. unsigned __int64 refN = refExts.ordinality() ? refExts.item(0) : (unsigned __int64)-1;
  5762. unsigned __int64 fileN = fileExts.ordinality() ? fileExts.item(0) : (unsigned __int64)-1;
  5763. unsigned fileP=0;
  5764. unsigned refP=0;
  5765. unsigned missP=0;
  5766. while (fileN != ((unsigned __int64)-1) || refN != ((unsigned __int64)-1))
  5767. {
  5768. while (fileN == refN)
  5769. {
  5770. if (!primary)
  5771. {
  5772. bool found = false;
  5773. while (missP < missingPrimarys.ordinality())
  5774. {
  5775. unsigned __int64 missN = missingPrimarys.item(missP);
  5776. if (refN == missN)
  5777. {
  5778. ++missP;
  5779. found = true;
  5780. break;
  5781. }
  5782. if (missN > refN)
  5783. break;
  5784. ++missP;
  5785. }
  5786. // i.e. found in backup, but was listed missing in primary
  5787. if (found)
  5788. {
  5789. StringBuffer fname;
  5790. StringBuffer name(EXTERNAL_NAME_PREFIX);
  5791. name.append(refN);
  5792. extHandler->getFilename(fname, name.str());
  5793. PROGLOG("Copying missing primary external from backup: %s", fname.str());
  5794. StringBuffer backupFname(remoteBackupLocation);
  5795. extHandler->getName(backupFname, name.str());
  5796. try
  5797. {
  5798. copyFile(fname.str(), backupFname.str());
  5799. }
  5800. catch (IException *e)
  5801. {
  5802. EXCLOG(e, NULL);
  5803. e->Release();
  5804. }
  5805. }
  5806. }
  5807. ++refP;
  5808. ++fileP;
  5809. refN = (refP == refExts.ordinality()) ? (unsigned __int64)-1 : refExts.item(refP);
  5810. fileN = (fileP == fileExts.ordinality()) ? (unsigned __int64)-1 : fileExts.item(fileP);
  5811. if (fileN == ((unsigned __int64)-1) || refN == ((unsigned __int64)-1))
  5812. break;
  5813. }
  5814. while (fileN < refN)
  5815. {
  5816. StringBuffer fname;
  5817. StringBuffer name(EXTERNAL_NAME_PREFIX);
  5818. name.append(fileN);
  5819. extHandler->getName(fname, name.str());
  5820. WARNLOG("Unreferenced %s external %s file", primary?"primary":"backup", fname.str());
  5821. ++fileP;
  5822. if (fileP == fileExts.ordinality())
  5823. {
  5824. fileN = (unsigned __int64)-1;
  5825. break;
  5826. }
  5827. fileN = fileExts.item(fileP);
  5828. }
  5829. while (refN < fileN)
  5830. {
  5831. StringBuffer fname;
  5832. StringBuffer name(EXTERNAL_NAME_PREFIX);
  5833. name.append(refN);
  5834. extHandler->getName(fname, name.str());
  5835. WARNLOG("External %s file reference %s missing", primary?"primary":"backup", fname.str());
  5836. if (primary)
  5837. missingPrimarys.append(refN);
  5838. else
  5839. {
  5840. bool found = false;
  5841. while (missP < missingPrimarys.ordinality())
  5842. {
  5843. unsigned __int64 missN = missingPrimarys.item(missP);
  5844. if (refN == missN)
  5845. {
  5846. ++missP;
  5847. found = true;
  5848. break;
  5849. }
  5850. if (missN > refN)
  5851. break;
  5852. ++missP;
  5853. }
  5854. if (!found)
  5855. {
  5856. // i.e. not missing in primary, but missing in backup
  5857. StringBuffer fname;
  5858. StringBuffer name(EXTERNAL_NAME_PREFIX);
  5859. name.append(refN);
  5860. extHandler->getFilename(fname, name.str());
  5861. PROGLOG("Copying missing backup external from primary: %s", fname.str());
  5862. StringBuffer backupFname(remoteBackupLocation);
  5863. extHandler->getName(backupFname, name.str());
  5864. try
  5865. {
  5866. copyFile(backupFname.str(), fname.str());
  5867. }
  5868. catch (IException *e)
  5869. {
  5870. EXCLOG(e, NULL);
  5871. e->Release();
  5872. }
  5873. }
  5874. }
  5875. ++refP;
  5876. if (refP == refExts.ordinality())
  5877. {
  5878. refN = (unsigned __int64)-1;
  5879. break;
  5880. }
  5881. refN = refExts.item(refP);
  5882. }
  5883. }
  5884. if (!primary || 0 == remoteBackupLocation.length())
  5885. break;
  5886. if (missingPrimarys.ordinality())
  5887. missingPrimarys.sort(uint64compare);
  5888. primary = false;
  5889. }
  5890. // check any marked with legacy formats
  5891. if (legacyFmts.ordinality() && refExts.ordinality())
  5892. {
  5893. legacyFmts.sort(extNcompareFunc);
  5894. unsigned refP = 0;
  5895. unsigned __int64 refN = refExts.item(refP++);
  5896. bool done = false;
  5897. ForEachItemIn(l, legacyFmts)
  5898. {
  5899. CLegacyFmtItem &itm = legacyFmts.item(l);
  5900. while (refN<itm.num)
  5901. {
  5902. if (refP == refExts.ordinality())
  5903. {
  5904. done = true;
  5905. break;
  5906. }
  5907. refN = refExts.item(refP++);
  5908. }
  5909. if (done)
  5910. break;
  5911. if (refN == itm.num)
  5912. {
  5913. IExternalHandler *extHandler = queryExternalHandler(itm.ext);
  5914. if (!extHandler)
  5915. {
  5916. WARNLOG("Unknown external extension, external=%s, extension=%s", itm.name.get(), itm.ext.get());
  5917. continue;
  5918. }
  5919. StringBuffer fname;
  5920. extHandler->getFilename(fname, itm.name);
  5921. PROGLOG("Converting legacy external type(%s) to new, file=%s", itm.ext.get(), fname.str());
  5922. MemoryBuffer mb;
  5923. Owned<IPropertyTree> tree = createPTree("tmp");
  5924. extHandler->read(itm.name, *tree, mb, true);
  5925. mb.append(""); // no children
  5926. tree.setown(createPTree(mb));
  5927. IExternalHandler *extHandlerNew = queryExternalHandler(EF_BinaryValue);
  5928. extHandlerNew->write(itm.name, *tree);
  5929. extHandler->remove(itm.name);
  5930. }
  5931. }
  5932. }
  5933. root->removeProp("External"); // remove legacy /External if present
  5934. unsigned items = treeMaker.convertQueue.ordinality();
  5935. if (items)
  5936. {
  5937. LOG(MCdebugInfo(100), unknownJob, "Converting %d items larger than threshold size %d, to external definitions", items, externalSizeThreshold);
  5938. ForEachItemIn(i, treeMaker.convertQueue)
  5939. SDSManager->writeExternal(treeMaker.convertQueue.item(i), true);
  5940. saveNeeded = true;
  5941. }
  5942. if (saveNeeded)
  5943. {
  5944. LOG(MCdebugInfo(100), unknownJob, "Saving converted store");
  5945. SDSManager->saveStore();
  5946. }
  5947. }
  5948. catch (IException *e)
  5949. {
  5950. LOG(MCoperatorError, unknownJob, e, "Exception - Failed to load main store");
  5951. throw;
  5952. }
  5953. catch (DALI_CATCHALL)
  5954. {
  5955. LOG(MCoperatorError, unknownJob, "Unknown exception - Failed to load main store");
  5956. throw;
  5957. }
  5958. if (!root)
  5959. {
  5960. root = (CServerRemoteTree *) createServerTree();
  5961. root->setName("root");
  5962. }
  5963. if (remoteBackupLocation.length())
  5964. {
  5965. try { validateBackup(); }
  5966. catch (IException *e) { LOG(MCoperatorError, unknownJob, e, "Validating backup"); e->Release(); }
  5967. StringBuffer deltaFilename(dataPath);
  5968. iStoreHelper->getCurrentDeltaFilename(deltaFilename);
  5969. OwnedIFile iFileDelta = createIFile(deltaFilename.str());
  5970. deltaFilename.clear().append(remoteBackupLocation);
  5971. iStoreHelper->getCurrentDeltaFilename(deltaFilename);
  5972. OwnedIFile iFileDeltaBackup = createIFile(deltaFilename.str());
  5973. CDateTime modifiedTime;
  5974. if (!iFileDelta->exists() && !iFileDeltaBackup->exists())
  5975. doTimeComparison = true;
  5976. else if (iFileDelta->getTime(NULL, &modifiedTime, NULL))
  5977. {
  5978. if (iFileDeltaBackup->setTime(NULL, &modifiedTime, NULL))
  5979. doTimeComparison = true;
  5980. }
  5981. if (!doTimeComparison)
  5982. LOG(MCoperatorWarning, unknownJob, "Unable to use time comparison when comparing delta backup file");
  5983. }
  5984. Owned<IRemoteConnection> conn = connect("/", 0, RTM_INTERNAL, INFINITE);
  5985. initializeInternals(conn->queryRoot());
  5986. conn.clear();
  5987. bool forceGroupUpdate = config.getPropBool("DFS/@forceGroupUpdate");
  5988. StringBuffer response;
  5989. initClusterGroups(forceGroupUpdate, response, oldEnvironment);
  5990. if (response.length())
  5991. PROGLOG("DFS group initialization : %s", response.str()); // should this be a syslog?
  5992. }
  5993. void CCovenSDSManager::saveStore(const char *storeName, bool currentEdition)
  5994. {
  5995. struct CIgnore
  5996. {
  5997. CIgnore() { SDSManager->ignoreExternals=true; }
  5998. ~CIgnore() { SDSManager->ignoreExternals=false; }
  5999. } ignore;
  6000. iStoreHelper->saveStore(root, NULL, currentEdition);
  6001. unsigned initNodeTableSize = allNodes.maxElements()+OVERFLOWSIZE;
  6002. queryCoven().setInitSDSNodes(initNodeTableSize>INIT_NODETABLE_SIZE?initNodeTableSize:INIT_NODETABLE_SIZE);
  6003. }
  6004. CServerRemoteTree *CCovenSDSManager::queryRegisteredTree(__int64 uniqId)
  6005. {
  6006. CHECKEDCRITICALBLOCK(treeRegCrit, fakeCritTimeout);
  6007. return (CServerRemoteTree *)allNodes.queryElem(uniqId);
  6008. }
  6009. CServerRemoteTree *CCovenSDSManager::getRegisteredTree(__int64 uniqId)
  6010. {
  6011. CHECKEDCRITICALBLOCK(treeRegCrit, fakeCritTimeout);
  6012. return (CServerRemoteTree *)allNodes.getElem(uniqId);
  6013. }
  6014. CServerRemoteTree *CCovenSDSManager::queryRoot()
  6015. {
  6016. return root;
  6017. }
  6018. StringBuffer &transformToAbsolute(StringBuffer &result, const char *xpath, unsigned index)
  6019. {
  6020. const char *end = xpath+strlen(xpath);
  6021. const char *p = end;
  6022. const char *q = NULL;
  6023. for (;;)
  6024. {
  6025. if (p == xpath)
  6026. {
  6027. p = end;
  6028. break;
  6029. }
  6030. --p;
  6031. if ('/' == *p)
  6032. {
  6033. p = end;
  6034. break;
  6035. }
  6036. else if ('[' == *p)
  6037. {
  6038. q = p;
  6039. break;
  6040. }
  6041. }
  6042. if (q)
  6043. result.append(p-xpath, xpath);
  6044. else
  6045. result.append(xpath);
  6046. result.append('[');
  6047. result.append(index);
  6048. result.append(']');
  6049. return result;
  6050. }
  6051. void cleanChangeTree(IPropertyTree &tree)
  6052. {
  6053. tree.removeProp("@id");
  6054. Owned<IPropertyTreeIterator> iter = tree.getElements(RENAME_TAG);
  6055. ForEach (*iter)
  6056. iter->query().removeProp("@id");
  6057. iter.setown(tree.getElements(DELETE_TAG));
  6058. ForEach (*iter)
  6059. iter->query().removeProp("@id");
  6060. iter.setown(tree.getElements(RESERVED_CHANGE_NODE));
  6061. ForEach (*iter)
  6062. cleanChangeTree(iter->query());
  6063. }
  6064. void CCovenSDSManager::saveDelta(const char *path, IPropertyTree &changeTree)
  6065. {
  6066. CHECKEDCRITICALBLOCK(saveIncCrit, fakeCritTimeout);
  6067. // translate changeTree to inc format (e.g. remove id's)
  6068. if (externalEnvironment)
  6069. {
  6070. // don't save any changed to /Environment if external
  6071. if (0 == strncmp("/Environment", path, strlen("/Environment")))
  6072. {
  6073. WARNLOG("Attempt to change read-only Dali environment, path = %s", path);
  6074. return;
  6075. }
  6076. if (0 == strcmp("/", path) && changeTree.hasProp("*[@name=\"Environment\"]"))
  6077. {
  6078. WARNLOG("Attempt to change read-only Dali environment, path = %s", path);
  6079. return;
  6080. }
  6081. }
  6082. cleanChangeTree(changeTree);
  6083. // write out with header details (e.g. path)
  6084. Owned<IPropertyTree> header = createPTree("Header");
  6085. header->setProp("@path", path);
  6086. IPropertyTree *delta = header->addPropTree("Delta", createPTree());
  6087. delta->addPropTree(changeTree.queryName(), LINK(&changeTree));
  6088. StringBuffer fname(dataPath);
  6089. OwnedIFile deltaIPIFile = createIFile(fname.append(DELTAINPROGRESS).str());
  6090. OwnedIFileIO deltaIPIFileIO = deltaIPIFile->open(IFOcreate);
  6091. deltaIPIFileIO.clear();
  6092. struct RemoveDIPBlock
  6093. {
  6094. IFile &iFile;
  6095. bool done;
  6096. void doit() { done = true; iFile.remove(); }
  6097. RemoveDIPBlock(IFile &_iFile) : iFile(_iFile), done(false) { }
  6098. ~RemoveDIPBlock () { if (!done) doit(); }
  6099. } removeDIP(*deltaIPIFile);
  6100. StringBuffer detachIPStr(dataPath);
  6101. OwnedIFile detachIPIFile = createIFile(detachIPStr.append(DETACHINPROGRESS).str());
  6102. if (detachIPIFile->exists()) // very small window where this can happen.
  6103. {
  6104. // implies other operation about to access current delta
  6105. // CHECK session is really alive, otherwise it has been orphaned, so remove it.
  6106. try
  6107. {
  6108. SessionId sessId = 0;
  6109. OwnedIFileIO detachIPIO = detachIPIFile->open(IFOread);
  6110. if (detachIPIO)
  6111. {
  6112. size_t s = detachIPIO->read(0, sizeof(sessId), &sessId);
  6113. detachIPIO.clear();
  6114. if (sizeof(sessId) == s)
  6115. {
  6116. // double check session is really alive
  6117. if (querySessionManager().sessionStopped(sessId, 0))
  6118. detachIPIFile->remove();
  6119. else
  6120. {
  6121. // *cannot block* because other op (sasha) accessing remote dali files, can access dali.
  6122. removeDIP.doit();
  6123. PROGLOG("blocked");
  6124. toXML(header, blockedDelta);
  6125. return;
  6126. }
  6127. }
  6128. }
  6129. }
  6130. catch (IException *e) { EXCLOG(e, NULL); e->Release(); }
  6131. }
  6132. bool first = false;
  6133. try
  6134. {
  6135. StringBuffer deltaFilename(dataPath);
  6136. iStoreHelper->getCurrentDeltaFilename(deltaFilename);
  6137. toXML(header, blockedDelta);
  6138. OwnedIFile iFile = createIFile(deltaFilename.str());
  6139. first = !iFile->exists() || 0 == iFile->size();
  6140. writeDelta(blockedDelta, *iFile);
  6141. }
  6142. catch (IException *e)
  6143. {
  6144. // NB: writeDelta retries a few times before giving up.
  6145. VStringBuffer errMsg("saveDelta: failed to save delta data, blockedDelta size=%d", blockedDelta.length());
  6146. LOG(MCoperatorError, unknownJob, e, errMsg.str());
  6147. e->Release();
  6148. return;
  6149. }
  6150. if (remoteBackupLocation.length())
  6151. {
  6152. try
  6153. {
  6154. if (backupOutOfSync) // true if there was previously an exception during synchronously writing delta to backup.
  6155. {
  6156. LOG(MCoperatorError, unknownJob, "Backup delta is out of sync due to a prior backup write error, attempting to resync");
  6157. // catchup - check and copy primary delta to backup
  6158. validateDeltaBackup();
  6159. backupOutOfSync = false;
  6160. LOG(MCoperatorError, unknownJob, "Backup delta resynchronized");
  6161. }
  6162. else
  6163. backupHandler.addDelta(blockedDelta, iStoreHelper->queryCurrentEdition(), first);
  6164. }
  6165. catch (IException *e)
  6166. {
  6167. LOG(MCoperatorError, unknownJob, e, "saveDelta: failed to save backup delta data");
  6168. e->Release();
  6169. backupOutOfSync = true;
  6170. }
  6171. }
  6172. if (blockedDelta.length() > 0x100000)
  6173. blockedDelta.kill();
  6174. else
  6175. blockedDelta.clear();
  6176. }
  6177. CSubscriberContainerList *CCovenSDSManager::getSubscribers(const char *xpath, CPTStack &stack)
  6178. {
  6179. return subscribers.getQualifiedList(xpath, stack);
  6180. }
  6181. inline void serverToClientTree(CServerRemoteTree &src, CClientRemoteTree &dst)
  6182. {
  6183. if (src.getPropInt64(EXT_ATTR))
  6184. {
  6185. MemoryBuffer mb;
  6186. src.serializeSelfRT(mb, false);
  6187. dst.deserializeSelfRT(mb);
  6188. }
  6189. else
  6190. dst.clone(src, true, false);
  6191. dst.setServerId(src.queryServerId());
  6192. if (src.hasChildren()) dst.addServerTreeInfo(STI_HaveChildren);
  6193. }
  6194. class CMultipleConnector : implements IMultipleConnector, public CInterface
  6195. {
  6196. StringArray xpaths;
  6197. UnsignedArray modes;
  6198. public:
  6199. IMPLEMENT_IINTERFACE;
  6200. CMultipleConnector() { }
  6201. CMultipleConnector(MemoryBuffer &src)
  6202. {
  6203. unsigned c;
  6204. src.read(c);
  6205. xpaths.ensure(c);
  6206. modes.ensure(c);
  6207. while (c--)
  6208. {
  6209. StringAttr xpath;
  6210. unsigned mode;
  6211. src.read(xpath);
  6212. src.read(mode);
  6213. xpaths.append(xpath);
  6214. modes.append(mode);
  6215. }
  6216. }
  6217. // IMultipleConnector impl.
  6218. virtual void addConnection(const char *xpath, unsigned mode)
  6219. {
  6220. if (RTM_CREATE == (mode & RTM_CREATE_MASK) || RTM_CREATE_QUERY == (mode & RTM_CREATE_MASK))
  6221. throw MakeSDSException(SDSExcpt_BadMode, "multiple connections do not support creation modes");
  6222. xpaths.append(xpath);
  6223. modes.append(mode);
  6224. }
  6225. virtual unsigned queryConnections() { return xpaths.ordinality(); }
  6226. virtual void getConnectionDetails(unsigned which, StringAttr &xpath, unsigned &mode)
  6227. {
  6228. xpath.set(xpaths.item(which));
  6229. mode = modes.item(which);
  6230. }
  6231. virtual void serialize(MemoryBuffer &dst)
  6232. {
  6233. unsigned c=xpaths.ordinality();
  6234. dst.append(c);
  6235. while (c--)
  6236. dst.append(xpaths.item(c)).append(modes.item(c));
  6237. }
  6238. };
  6239. IMultipleConnector *deserializeIMultipleConnector(MemoryBuffer &src)
  6240. {
  6241. return new CMultipleConnector(src);
  6242. }
  6243. IMultipleConnector *createIMultipleConnector()
  6244. {
  6245. return new CMultipleConnector();
  6246. }
  6247. StringBuffer &getMConnectString(IMultipleConnector *mConnect, StringBuffer &s)
  6248. {
  6249. unsigned c;
  6250. for (c=0; c<mConnect->queryConnections(); c++)
  6251. {
  6252. StringAttr xpath;
  6253. unsigned mode;
  6254. mConnect->getConnectionDetails(c, xpath, mode);
  6255. s.append("xpath=\"").append(xpath).append("\" [mode=").append(mode).append("]");
  6256. if (c != mConnect->queryConnections()-1)
  6257. s.append(", ");
  6258. }
  6259. return s;
  6260. }
  6261. // ISDSManager impl.
  6262. IRemoteConnections *CCovenSDSManager::connect(IMultipleConnector *mConnect, SessionId id, unsigned timeout)
  6263. {
  6264. Owned<CLCLockBlock> lockBlock;
  6265. lockBlock.setown(new CLCReadLockBlock(dataRWLock, readWriteTimeout, __FILE__, __LINE__));
  6266. Owned<CRemoteConnections> remoteConnections = new CRemoteConnections;
  6267. unsigned c;
  6268. for (c=0; c<mConnect->queryConnections(); c++)
  6269. {
  6270. StringAttr xpath;
  6271. unsigned mode;
  6272. mConnect->getConnectionDetails(c, xpath, mode);
  6273. // connect can return NULL.
  6274. remoteConnections->add(connect(xpath, id, mode, timeout));
  6275. }
  6276. return LINK(remoteConnections);
  6277. }
  6278. IRemoteConnection *CCovenSDSManager::connect(const char *xpath, SessionId id, unsigned mode, unsigned timeout)
  6279. {
  6280. Owned<CLCLockBlock> lockBlock;
  6281. Owned<LinkingCriticalBlock> connectCritBlock;
  6282. if (!RTM_MODE(mode, RTM_INTERNAL))
  6283. {
  6284. connectCritBlock.setown(new LinkingCriticalBlock(connectCrit, __FILE__, __LINE__));
  6285. if (RTM_CREATE == (mode & RTM_CREATE_MASK) || RTM_CREATE_QUERY == (mode & RTM_CREATE_MASK))
  6286. lockBlock.setown(new CLCWriteLockBlock(dataRWLock, readWriteTimeout, __FILE__, __LINE__));
  6287. else
  6288. lockBlock.setown(new CLCReadLockBlock(dataRWLock, readWriteTimeout, __FILE__, __LINE__));
  6289. }
  6290. CServerRemoteTree *_tree;
  6291. Owned<CServerRemoteTree> tree;
  6292. ConnectionId connectionId = 0;
  6293. createConnection(id, mode, timeout, xpath, _tree, connectionId, true, connectCritBlock);
  6294. if (connectionId)
  6295. tree.setown(_tree);
  6296. connectCritBlock.clear();
  6297. if (connectionId)
  6298. {
  6299. CRemoteConnection *conn = new CRemoteConnection(*this, connectionId, xpath, id, mode, timeout);
  6300. assertex(conn);
  6301. CDisableFetchChangeBlock block(*conn);
  6302. CClientRemoteTree *clientTree = new CClientRemoteTree(*conn);
  6303. assertex(clientTree);
  6304. serverToClientTree(*tree, *clientTree);
  6305. conn->setRoot(clientTree);
  6306. return conn;
  6307. }
  6308. return NULL;
  6309. }
  6310. SubscriptionId CCovenSDSManager::subscribe(const char *xpath, ISDSSubscription &notify, bool sub, bool sendValue)
  6311. {
  6312. assertex(xpath);
  6313. if (sub && sendValue)
  6314. throw MakeSDSException(SDSExcpt_Unsupported, "Subscription to sub elements, with sendValue option unsupported");
  6315. StringBuffer s;
  6316. if ('/' != *xpath)
  6317. {
  6318. s.append('/').append(xpath);
  6319. xpath = s.str();
  6320. }
  6321. CSDSSubscriberProxy *subscriber = new CSDSSubscriberProxy(xpath, sub, sendValue, notify);
  6322. querySubscriptionManager(SDS_PUBLISHER)->add(subscriber, subscriber->getId());
  6323. return subscriber->getId();
  6324. }
  6325. SubscriptionId CCovenSDSManager::subscribeExact(const char *xpath, ISDSNodeSubscription &notify, bool sendValue)
  6326. {
  6327. assertex(xpath);
  6328. StringBuffer s;
  6329. if ('/' != *xpath)
  6330. {
  6331. s.append('/').append(xpath);
  6332. xpath = s.str();
  6333. }
  6334. CSDSNodeSubscriberProxy *subscriber = new CSDSNodeSubscriberProxy(xpath, sendValue, notify);
  6335. querySubscriptionManager(SDSNODE_PUBLISHER)->add(subscriber, subscriber->getId());
  6336. return subscriber->getId();
  6337. }
  6338. void CCovenSDSManager::unsubscribe(SubscriptionId id)
  6339. {
  6340. querySubscriptionManager(SDS_PUBLISHER)->remove(id);
  6341. }
  6342. void CCovenSDSManager::unsubscribeExact(SubscriptionId id)
  6343. {
  6344. querySubscriptionManager(SDSNODE_PUBLISHER)->remove(id);
  6345. }
  6346. bool CCovenSDSManager::removeNotifyHandler(const char *handlerKey)
  6347. {
  6348. return nodeNotifyHandlers.remove(handlerKey);
  6349. }
  6350. IPropertyTree *CCovenSDSManager::lockStoreRead() const
  6351. {
  6352. PROGLOG("lockStoreRead() called");
  6353. CHECKEDREADLOCKENTER(dataRWLock, readWriteTimeout);
  6354. return root;
  6355. }
  6356. void CCovenSDSManager::unlockStoreRead() const
  6357. {
  6358. PROGLOG("unlockStoreRead() called");
  6359. dataRWLock.unlockRead();
  6360. }
  6361. bool CCovenSDSManager::setSDSDebug(StringArray &params, StringBuffer &reply)
  6362. {
  6363. if (0 == params.ordinality()) return false;
  6364. else if (0 == stricmp("datalockHoldTiming", params.item(0)))
  6365. {
  6366. if (params.ordinality()<2)
  6367. {
  6368. reply.append("datalockHoldTiming currently = ").append(readWriteSlowTracing);
  6369. return false;
  6370. }
  6371. unsigned ms = atoi(params.item(1));
  6372. readWriteSlowTracing = ms;
  6373. PROGLOG("datalock, readWriteSlowTracing timing set to %d", readWriteSlowTracing);
  6374. }
  6375. else if (0 == stricmp("datalockHoldStack", params.item(0)))
  6376. {
  6377. if (params.ordinality()<2)
  6378. {
  6379. reply.append("datalockHoldStack currently set to '").append(readWriteStackTracing?"on":"off").append("'");
  6380. return false;
  6381. }
  6382. readWriteStackTracing = (0 == stricmp("on", params.item(1)));
  6383. PROGLOG("datalock, held time stacks set to '%s'", readWriteStackTracing?"on":"off");
  6384. }
  6385. else if (0 == stricmp("datalockRetryStackTiming", params.item(0)))
  6386. {
  6387. if (params.ordinality()<2)
  6388. {
  6389. reply.append("datalockRetryStackTiming currently =").append(readWriteTimeout);
  6390. return false;
  6391. }
  6392. unsigned ms = atoi(params.item(1));
  6393. readWriteTimeout = ms;
  6394. PROGLOG("datalock, readWriteTimeout timing set to %s", readWriteStackTracing?"on":"off");
  6395. }
  6396. else if (0 == stricmp("fakecritTiming", params.item(0)))
  6397. {
  6398. if (params.ordinality()<2)
  6399. {
  6400. reply.append("fakeCritTimeout currently =").append(fakeCritTimeout);
  6401. return false;
  6402. }
  6403. unsigned ms = atoi(params.item(1));
  6404. fakeCritTimeout = ms;
  6405. PROGLOG("fakecrit, fakeCritTimeout timing set to %d", fakeCritTimeout);
  6406. }
  6407. else
  6408. {
  6409. reply.append("Unknown command");
  6410. return false;
  6411. }
  6412. return true;
  6413. }
  6414. void CCovenSDSManager::saveRequest()
  6415. {
  6416. blockingSave();
  6417. }
  6418. IPropertyTree &CCovenSDSManager::queryProperties() const
  6419. {
  6420. return *properties;
  6421. }
  6422. void CCovenSDSManager::installNotifyHandler(const char *handlerKey, ISDSNotifyHandler *handler)
  6423. {
  6424. nodeNotifyHandlers.replace(* new CSDSNotifyHandlerMapping(handlerKey, *handler));
  6425. }
  6426. // ISDSConnectionManager impl.
  6427. void CCovenSDSManager::commit(CRemoteConnection &connection, bool *disconnectDeleteRoot)
  6428. {
  6429. Owned<CLCWriteLockBlock> lockBlock;
  6430. if (!RTM_MODE(connection.queryMode(), RTM_INTERNAL))
  6431. lockBlock.setown(new CLCWriteLockBlock(dataRWLock, readWriteTimeout, __FILE__, __LINE__));
  6432. CClientRemoteTree *tree = (CClientRemoteTree *) connection.queryRoot();
  6433. bool lazyFetch = connection.setLazyFetch(false);
  6434. Owned<IPropertyTree> changeTree = tree->collateData();
  6435. connection.setLazyFetch(lazyFetch);
  6436. if (NULL == disconnectDeleteRoot && !changeTree) return;
  6437. ConnectionId connectionId = connection.queryConnectionId();
  6438. CServerConnection *serverConnection = queryConnection(connectionId);
  6439. if (!serverConnection)
  6440. throw MakeSDSException(SDSExcpt_ConnectionAbsent, " [commit]");
  6441. try
  6442. {
  6443. CServerRemoteTree *serverTree = (CServerRemoteTree *)serverConnection->queryRoot();
  6444. assertex(serverTree);
  6445. MemoryBuffer newIds, inc;
  6446. if (changeTree && serverTree->processData(*serverConnection, *changeTree, newIds))
  6447. { // something commited, if RTM_Create was used need to remember this.
  6448. StringBuffer path;
  6449. serverConnection->queryPTreePath().getAbsolutePath(path);
  6450. saveDelta(path.str(), *changeTree);
  6451. bool lazyFetch = connection.setLazyFetch(false);
  6452. tree->clearCommitChanges(&newIds);
  6453. assertex(newIds.getPos() == newIds.length()); // must have read it all
  6454. connection.setLazyFetch(lazyFetch);
  6455. }
  6456. }
  6457. catch (IException *)
  6458. {
  6459. if (disconnectDeleteRoot)
  6460. {
  6461. connection.setConnected(false);
  6462. disconnect(connectionId, *disconnectDeleteRoot);
  6463. }
  6464. throw;
  6465. }
  6466. if (disconnectDeleteRoot)
  6467. {
  6468. connection.setConnected(false);
  6469. disconnect(connectionId, *disconnectDeleteRoot);
  6470. }
  6471. }
  6472. CRemoteTreeBase *CCovenSDSManager::get(CRemoteConnection &connection, __int64 serverId)
  6473. {
  6474. Owned<CLCReadLockBlock> lockBlock;
  6475. if (!RTM_MODE(connection.queryMode(), RTM_INTERNAL))
  6476. lockBlock.setown(new CLCReadLockBlock(dataRWLock, readWriteTimeout, __FILE__, __LINE__));
  6477. CDisableFetchChangeBlock block(connection);
  6478. CRemoteTreeBase *connectionRoot = (CRemoteTreeBase *) connection.queryRoot();
  6479. Owned<CServerRemoteTree> tree = getRegisteredTree(connectionRoot->queryServerId());
  6480. if (!tree)
  6481. return NULL;
  6482. CClientRemoteTree *newTree = new CClientRemoteTree(connection);
  6483. serverToClientTree(*tree, *newTree);
  6484. return newTree;
  6485. }
  6486. void CCovenSDSManager::getChildren(CRemoteTreeBase &parent, CRemoteConnection &connection, unsigned levels)
  6487. {
  6488. Owned<CLCReadLockBlock> lockBlock;
  6489. if (!RTM_MODE(connection.queryMode(), RTM_INTERNAL))
  6490. lockBlock.setown(new CLCReadLockBlock(dataRWLock, readWriteTimeout, __FILE__, __LINE__));
  6491. CDisableFetchChangeBlock block(connection);
  6492. Owned<CServerRemoteTree> serverParent = (CServerRemoteTree *)getRegisteredTree(parent.queryServerId());
  6493. if (serverParent)
  6494. _getChildren(parent, *serverParent, connection, levels);
  6495. }
  6496. void CCovenSDSManager::getChildrenFor(CRTArray &childLessList, CRemoteConnection &connection, unsigned levels)
  6497. {
  6498. Owned<CLCReadLockBlock> lockBlock;
  6499. if (!RTM_MODE(connection.queryMode(), RTM_INTERNAL))
  6500. lockBlock.setown(new CLCReadLockBlock(dataRWLock, readWriteTimeout, __FILE__, __LINE__));
  6501. CDisableFetchChangeBlock block(connection);
  6502. ForEachItemIn(f, childLessList)
  6503. {
  6504. CRemoteTreeBase &tree = childLessList.item(f);
  6505. Owned<CServerRemoteTree> serverParent = (CServerRemoteTree *)getRegisteredTree(tree.queryServerId());
  6506. if (serverParent)
  6507. _getChildren(tree, *serverParent, connection, levels);
  6508. }
  6509. }
  6510. static void addServerChildren(CClientRemoteTree &clientParent, CServerRemoteTree &serverParent, bool recurse)
  6511. {
  6512. Owned<IPropertyTreeIterator> iter = serverParent.getElements("*");
  6513. ForEach (*iter)
  6514. {
  6515. CServerRemoteTree &serverChild = (CServerRemoteTree &)iter->query();
  6516. CClientRemoteTree *clientChild = (CClientRemoteTree *)clientParent.create(NULL);
  6517. serverToClientTree(serverChild, *clientChild);
  6518. clientChild = (CClientRemoteTree *)clientParent.addPropTree(clientChild->queryName(), clientChild);
  6519. if (recurse)
  6520. addServerChildren(*clientChild, serverChild, recurse);
  6521. }
  6522. }
  6523. void CCovenSDSManager::matchServerTree(CClientRemoteTree *local, IPropertyTree &matchTree, bool allTail)
  6524. {
  6525. Owned<IPropertyTreeIterator> matchIter = matchTree.getElements("*");
  6526. if (matchIter->first())
  6527. {
  6528. if (local->hasChildren() && NULL == local->queryChildren())
  6529. {
  6530. local->createChildMap();
  6531. Owned<CServerRemoteTree> tree = getRegisteredTree(matchTree.getPropInt64("@serverId"));
  6532. addServerChildren(*local, *tree, false);
  6533. }
  6534. do
  6535. {
  6536. IPropertyTree &elem = matchIter->query();
  6537. StringBuffer path(elem.queryName());
  6538. path.append('[').append(elem.getPropInt("@pos")).append(']');
  6539. CClientRemoteTree *child = (CClientRemoteTree *)local->queryPropTree(path.str());
  6540. if (child) // if not would imply some other thread deleted I think.
  6541. matchServerTree(child, elem, allTail);
  6542. }
  6543. while (matchIter->next());
  6544. }
  6545. else
  6546. {
  6547. if (local->hasChildren() && NULL == local->queryChildren())
  6548. {
  6549. local->createChildMap();
  6550. Owned<CServerRemoteTree> tree = getRegisteredTree(matchTree.getPropInt64("@serverId"));
  6551. addServerChildren(*local, *tree, allTail);
  6552. }
  6553. }
  6554. }
  6555. void CCovenSDSManager::ensureLocal(CRemoteConnection &connection, CRemoteTreeBase &_parent, IPropertyTree *serverMatchTree, IPTIteratorCodes flags)
  6556. {
  6557. CClientRemoteTree &parent = (CClientRemoteTree &)_parent;
  6558. bool getLeaves = iptiter_remotegetbranch == (flags & iptiter_remotegetbranch);
  6559. CDisableFetchChangeBlock block(connection);
  6560. matchServerTree(&parent, *serverMatchTree, getLeaves);
  6561. }
  6562. void CCovenSDSManager::_getChildren(CRemoteTreeBase &parent, CServerRemoteTree &serverParent, CRemoteConnection &connection, unsigned levels)
  6563. {
  6564. Owned<IPropertyTreeIterator> iter = serverParent.getElements("*");
  6565. assertex(iter);
  6566. if (levels && serverParent.getPropBool("@fetchEntire"))
  6567. levels = 0;
  6568. ForEach(*iter)
  6569. {
  6570. CServerRemoteTree &child = (CServerRemoteTree &)iter->query();
  6571. CClientRemoteTree *newChild = new CClientRemoteTree(connection); // clone create
  6572. serverToClientTree(child, *newChild);
  6573. parent.addPropTree(newChild->queryName(), newChild);
  6574. if (0==levels || child.getPropBool("@fetchEntire"))
  6575. _getChildren(*newChild, child, connection, 0);
  6576. }
  6577. }
  6578. IPropertyTreeIterator *CCovenSDSManager::getElements(CRemoteConnection &connection, const char *xpath)
  6579. {
  6580. Owned<CLCReadLockBlock> lockBlock = new CLCReadLockBlock(dataRWLock, readWriteTimeout, __FILE__, __LINE__);
  6581. CDisableFetchChangeBlock block(connection);
  6582. Owned<CServerRemoteTree> serverConnRoot = (CServerRemoteTree *)getRegisteredTree(((CClientRemoteTree *)connection.queryRoot())->queryServerId());
  6583. Owned<DaliPTArrayIterator> elements = new DaliPTArrayIterator();
  6584. Owned<IPropertyTreeIterator> iter = serverConnRoot->getElements(xpath);
  6585. ForEach (*iter)
  6586. {
  6587. CServerRemoteTree &child = (CServerRemoteTree &)iter->query();
  6588. Owned<CClientRemoteTree> newChild = new CClientRemoteTree(connection);
  6589. serverToClientTree(child, *newChild);
  6590. elements->array.append(*LINK(newChild));
  6591. }
  6592. return LINK(elements);
  6593. }
  6594. void CCovenSDSManager::changeMode(CRemoteConnection &connection, unsigned mode, unsigned timeout, bool suppressReloads)
  6595. {
  6596. if (mode & RTM_CREATE_MASK)
  6597. throw MakeSDSException(SDSExcpt_BadMode, "calling changeMode");
  6598. ConnectionId connectionId = connection.queryConnectionId();
  6599. Linked<CServerConnection> serverConnection = queryConnection(connectionId);
  6600. if (!serverConnection)
  6601. throw MakeSDSException(SDSExcpt_ConnectionAbsent, " [changeMode]");
  6602. CServerRemoteTree *connectionRoot = (CServerRemoteTree *) serverConnection->queryRoot();
  6603. unsigned prevMode = connection.queryMode();
  6604. if (RTM_MODE(prevMode, RTM_LOCK_WRITE) && !RTM_MODE(mode, RTM_LOCK_WRITE))
  6605. commit(connection, NULL);
  6606. changeLockMode(*serverConnection, mode, timeout);
  6607. if (!suppressReloads)
  6608. {
  6609. if (RTM_MODE(mode, RTM_LOCK_WRITE) && !RTM_MODE(prevMode, RTM_LOCK_WRITE) && !RTM_MODE(prevMode, RTM_LOCK_READ))
  6610. connection.reload();
  6611. }
  6612. }
  6613. IPropertyTree *CCovenSDSManager::getXPaths(__int64 serverId, const char *xpath, bool getServerIds)
  6614. {
  6615. Owned<CServerRemoteTree> tree = getRegisteredTree(serverId);
  6616. if (!tree)
  6617. return NULL;
  6618. IPropertyTree *matchTree = getXPathMatchTree(*tree, xpath);
  6619. if (!matchTree)
  6620. return NULL;
  6621. if (getServerIds)
  6622. populateWithServerIds(matchTree, tree);
  6623. return matchTree;
  6624. }
  6625. IPropertyTreeIterator *CCovenSDSManager::getXPathsSortLimit(const char *baseXPath, const char *matchXPath, const char *sortBy, bool caseinsensitive, bool ascending, unsigned from, unsigned limit)
  6626. {
  6627. Owned<IPropertyTree> matchTree = getXPathsSortLimitMatchTree(baseXPath, matchXPath, sortBy, caseinsensitive, ascending, from, limit);
  6628. if (!matchTree)
  6629. return createNullPTreeIterator();
  6630. IPropertyTree *baseTree = SDSManager->queryRoot()->queryPropTree(baseXPath);
  6631. return new CXPathIterator(baseTree, matchTree, iptiter_null);
  6632. }
  6633. IPropertyTree *CCovenSDSManager::getXPathsSortLimitMatchTree(const char *baseXPath, const char *matchXPath, const char *sortby, bool caseinsensitive, bool ascending, unsigned from, unsigned limit)
  6634. {
  6635. UNIMPLEMENTED;
  6636. return NULL;
  6637. }
  6638. void CCovenSDSManager::getExternalValue(__int64 index, MemoryBuffer &mb)
  6639. {
  6640. IExternalHandler *handler = queryExternalHandler(EF_BinaryValue);
  6641. assertex(handler);
  6642. StringBuffer name(EXTERNAL_NAME_PREFIX);
  6643. name.append(index);
  6644. handler->readValue(name.str(), mb);
  6645. }
  6646. void CCovenSDSManager::getExternalValueFromServerId(__int64 serverId, MemoryBuffer &mb)
  6647. {
  6648. Owned<CServerRemoteTree> idTree = (CServerRemoteTree *) SDSManager->getRegisteredTree(serverId);
  6649. if (idTree)
  6650. {
  6651. CHECKEDCRITICALBLOCK(extCrit, fakeCritTimeout);
  6652. __int64 index = idTree->getPropInt64(EXT_ATTR);
  6653. if (index)
  6654. getExternalValue(index, mb);
  6655. else
  6656. WARNLOG("External file reference missing (node name='%s', id=%" I64F "d)", idTree->queryName(), serverId);
  6657. }
  6658. }
  6659. IPropertyTreeIterator *CCovenSDSManager::getElementsRaw(const char *xpath,INode *remotedali, unsigned timeout)
  6660. {
  6661. assertex(!remotedali); // only client side
  6662. CHECKEDDALIREADLOCKBLOCK(dataRWLock, readWriteTimeout);
  6663. return root->getElements(xpath);
  6664. }
  6665. void CCovenSDSManager::setConfigOpt(const char *opt, const char *value)
  6666. {
  6667. IPropertyTree &props = queryProperties();
  6668. if (props.hasProp(opt) && (0 == strcmp(value, props.queryProp(opt))))
  6669. return;
  6670. ensurePTree(&queryProperties(), opt);
  6671. queryProperties().setProp(opt, value);
  6672. }
  6673. unsigned CCovenSDSManager::queryCount(const char *xpath)
  6674. {
  6675. unsigned count = 0;
  6676. if (xpath && *xpath == '/')
  6677. ++xpath;
  6678. Owned<IPropertyTreeIterator> iter = root->getElements(xpath);
  6679. ForEach(*iter)
  6680. ++count;
  6681. return count;
  6682. }
  6683. void CCovenSDSManager::start()
  6684. {
  6685. server.start();
  6686. if (coalesce) coalesce->start();
  6687. }
  6688. void CCovenSDSManager::stop()
  6689. {
  6690. server.stop();
  6691. PROGLOG("waiting for coalescer to stop");
  6692. if (coalesce) coalesce->stop();
  6693. }
  6694. void CCovenSDSManager::restart(IException * e)
  6695. {
  6696. LOG(MCwarning, unknownJob, "-------: stopping SDS server");
  6697. StringBuffer msg;
  6698. msg.append("Unhandled exception, restarting: ").append(e->errorCode()).append(": ");
  6699. e->errorMessage(msg);
  6700. stop();
  6701. connections.kill();
  6702. LOG(MCwarning, unknownJob, "-------: stopped");
  6703. LOG(MCwarning, unknownJob, "-------: saving current store . . . . . .");
  6704. saveStore();
  6705. LOG(MCwarning, unknownJob, "-------: store saved.");
  6706. LOG(MCwarning, unknownJob, "-------: restarting SDS server restart");
  6707. start();
  6708. LOG(MCwarning, unknownJob, "-------: restarted");
  6709. }
  6710. CServerConnection *CCovenSDSManager::createConnectionInstance(CRemoteTreeBase *root, SessionId sessionId, unsigned mode, unsigned timeout, const char *xpath, CRemoteTreeBase *&tree, ConnectionId _connectionId, StringAttr *deltaPath, Owned<IPropertyTree> &deltaChange, Owned<CBranchChange> &branchChange, unsigned &additions)
  6711. {
  6712. IPropertyTree *parent = NULL;
  6713. ConnInfoFlags connInfoFlags = (ConnInfoFlags)0;
  6714. const char *_xpath = ('/' == *xpath)?xpath+1:xpath;
  6715. StringBuffer tXpath;
  6716. if (!RTM_MODE(mode, RTM_CREATE_UNIQUE))
  6717. {
  6718. unsigned l = strlen(_xpath);
  6719. if (l && '/' == _xpath[l-1])
  6720. {
  6721. tXpath.append(l-1, _xpath);
  6722. _xpath = tXpath.str();
  6723. }
  6724. }
  6725. bool newNode = false;
  6726. bool replaceNode = false;
  6727. IPropertyTree *created = NULL, *createdParent = NULL;
  6728. StringBuffer head;
  6729. if (mode & RTM_CREATE)
  6730. {
  6731. const char *prop = splitXPath(_xpath, head);
  6732. if (head.length())
  6733. {
  6734. try
  6735. {
  6736. parent = createPropBranch(root, head.str(), true, &created, &createdParent);
  6737. if (created)
  6738. newNode = true;
  6739. }
  6740. catch (IException *) { if (created) createdParent->removeTree(created); throw; }
  6741. catch (DALI_CATCHALL) { if (created) createdParent->removeTree(created); throw; }
  6742. }
  6743. else
  6744. parent = root->splitBranchProp(_xpath, prop);
  6745. if (parent)
  6746. {
  6747. connInfoFlags = ci_newParent;
  6748. StringBuffer uProp;
  6749. if (RTM_MODE(mode, RTM_CREATE_UNIQUE))
  6750. {
  6751. Owned<IPropertyTreeIterator> iter = parent->getElements(prop);
  6752. if (iter->first())
  6753. {
  6754. uProp.append(prop).append('-');
  6755. unsigned l = uProp.length();
  6756. unsigned n=1;
  6757. for (;;)
  6758. {
  6759. n += getRandom() % 5; // better chance of finding a mismatch soon.
  6760. uProp.append(n);
  6761. iter.setown(parent->getElements(uProp.str()));
  6762. if (!iter->first())
  6763. break;
  6764. uProp.setLength(l);
  6765. }
  6766. prop = uProp.str();
  6767. if (head.length())
  6768. tXpath.append(head).append('/');
  6769. _xpath = tXpath.append(prop).str();
  6770. }
  6771. }
  6772. if (RTM_MODE(mode, RTM_CREATE_QUERY))
  6773. tree = (CRemoteTreeBase *) root->queryCreateBranch(parent, prop, &newNode);
  6774. else
  6775. {
  6776. IPropertyTree *newTree = ((CRemoteTreeBase *) parent)->create(NULL);
  6777. if (RTM_MODE(mode, RTM_CREATE_ADD))
  6778. tree = (CRemoteTreeBase *) parent->addPropTree(prop, newTree);
  6779. else // Default - RTM_CREATE - replace existing
  6780. {
  6781. Owned<IPropertyTreeIterator> iter = parent->getElements(prop);
  6782. if (iter->first())
  6783. replaceNode = true;
  6784. tree = (CRemoteTreeBase *) parent->setPropTree(prop, newTree);
  6785. }
  6786. newNode = true;
  6787. }
  6788. }
  6789. else
  6790. tree = NULL;
  6791. }
  6792. else
  6793. {
  6794. if (NULL == _xpath || '\0' == *_xpath)
  6795. tree = root;
  6796. else
  6797. {
  6798. StringBuffer path;
  6799. const char *prop = splitXPath(_xpath, path);
  6800. if (!prop) prop = _xpath;
  6801. // establish parent
  6802. tree = NULL;
  6803. Owned<IPropertyTreeIterator> iter = root->getElements(path.str());
  6804. ForEach (*iter)
  6805. {
  6806. IPropertyTree *_parent = &iter->query();
  6807. IPropertyTree *match = _parent->queryPropTree(prop);
  6808. if (tree)
  6809. {
  6810. if (match)
  6811. throw MakeSDSException(SDSExcpt_AmbiguousXpath, "Ambiguous: %s", _xpath);
  6812. }
  6813. else
  6814. {
  6815. parent = _parent;
  6816. tree = (CRemoteTreeBase *)match;
  6817. }
  6818. }
  6819. }
  6820. }
  6821. if (!tree)
  6822. return NULL;
  6823. assertex(tree->queryServerId());
  6824. ConnectionId connectionId = _connectionId;
  6825. if (!connectionId)
  6826. connectionId = coven.getUniqueId();
  6827. CServerConnection *connection = new CServerConnection(*this, connectionId, _xpath, sessionId, mode, timeout, parent, connInfoFlags);
  6828. Owned<LinkingCriticalBlock> b;
  6829. if (!RTM_MODE(mode, RTM_INTERNAL))
  6830. b.setown(new LinkingCriticalBlock(lockCrit, __FILE__, __LINE__));
  6831. connection->initPTreePath(*root, *tree);
  6832. if (newNode)
  6833. {
  6834. writeTransactions++;
  6835. // add tree into stack temporarily, or add manually at end.
  6836. deltaChange.setown(createPTree(RESERVED_CHANGE_NODE));
  6837. IPropertyTree *tail = deltaChange;
  6838. if (created) // some elems in "head" created
  6839. {
  6840. // iterate stack until createdParent to build up new head path.
  6841. StringBuffer _deltaPath;
  6842. unsigned s = 0;
  6843. StringBuffer headPath;
  6844. connection->queryPTreePath().getAbsolutePath(headPath);
  6845. if (headPath.length() && headPath.charAt(0) == '/')
  6846. headPath.remove(0, 1);
  6847. for (;;)
  6848. {
  6849. _deltaPath.append('/');
  6850. IPropertyTree &tree = connection->queryPTreePath().item(s);
  6851. if (&tree == createdParent)
  6852. {
  6853. ++s;
  6854. break;
  6855. }
  6856. unsigned l = _deltaPath.length();
  6857. const char *t = queryHead(headPath.str(), _deltaPath);
  6858. assertex(l != _deltaPath.length());
  6859. if (t)
  6860. headPath.remove(0, _deltaPath.length());
  6861. else
  6862. headPath.clear();
  6863. if (++s>=connection->queryPTreePath().ordinality())
  6864. break;
  6865. }
  6866. additions = connection->queryPTreePath().ordinality()-s;
  6867. deltaPath->set(_deltaPath.str());
  6868. branchChange.setown(new CBranchChange(*(CRemoteTreeBase *)createdParent));
  6869. CBranchChange *topChange = branchChange;
  6870. // iterate remaining stack, iterate marking as 'new'
  6871. for (;s<connection->queryPTreePath().ordinality();s++)
  6872. {
  6873. IPropertyTree &tree = connection->queryPTreePath().item(s);
  6874. IPropertyTree *n = tail->addPropTree(RESERVED_CHANGE_NODE, createPTree());
  6875. n->setProp("@name", tree.queryName());
  6876. n->setPropBool("@new", true);
  6877. tail = n;
  6878. Owned<CBranchChange> childChange = new CBranchChange((CRemoteTreeBase &)tree);
  6879. childChange->noteChange(PDS_Added, PDS_Added);
  6880. topChange->addChildBranch(*LINK(childChange));
  6881. topChange = childChange;
  6882. }
  6883. }
  6884. else
  6885. {
  6886. StringBuffer s, _deltaPath;
  6887. connection->queryPTreePath().getAbsolutePath(s);
  6888. const char *t = splitXPath(s.str(), _deltaPath);
  6889. deltaPath->set(_deltaPath.str());
  6890. IPropertyTree *n = tail->addPropTree(RESERVED_CHANGE_NODE, createPTree());
  6891. n->setProp("@name", tree->queryName());
  6892. if (replaceNode)
  6893. n->setPropBool("@replace", true);
  6894. else
  6895. n->setPropBool("@new", true);
  6896. branchChange.setown(new CBranchChange(*(CRemoteTreeBase *)tree));
  6897. if (replaceNode)
  6898. branchChange->noteChange(PDS_Data, PDS_Data);
  6899. else
  6900. branchChange->noteChange(PDS_Added, PDS_Added);
  6901. additions=1;
  6902. }
  6903. }
  6904. connection->setRoot(LINK(tree));
  6905. return connection;
  6906. }
  6907. void CCovenSDSManager::clearSDSLocks()
  6908. {
  6909. CHECKEDCRITICALBLOCK(lockCrit, fakeCritTimeout);
  6910. SuperHashIteratorOf<CLock> iter(lockTable.queryBaseTable());
  6911. ICopyArrayOf<CLock> locks;
  6912. ForEach(iter)
  6913. locks.append(iter.query());
  6914. ForEachItemIn(l, locks)
  6915. locks.item(l).unlockAll();
  6916. }
  6917. void CCovenSDSManager::changeLockMode(CServerConnection &connection, unsigned newMode, unsigned timeout)
  6918. {
  6919. CServerRemoteTree *tree = (CServerRemoteTree *) connection.queryRoot();
  6920. ConnectionId connectionId = connection.queryConnectionId();
  6921. __int64 treeId = tree->queryServerId();
  6922. newMode = newMode & (RTM_LOCKBASIC_MASK|RTM_LOCK_SUB);
  6923. newMode |= connection.queryMode() & ~(RTM_LOCKBASIC_MASK|RTM_LOCK_SUB);
  6924. CUnlockCallback callback(connection.queryXPath(), connectionId, *tree);
  6925. {
  6926. CHECKEDCRITICALBLOCK(lockCrit, fakeCritTimeout);
  6927. CLock *lock = queryLock(treeId);
  6928. if (lock)
  6929. {
  6930. lock->changeMode(connectionId, connection.querySessionId(), newMode, timeout, callback);
  6931. connection.setMode(newMode);
  6932. return;
  6933. }
  6934. }
  6935. // no existing lock for connection
  6936. lock(*tree, connection.queryXPath(), connectionId, connection.querySessionId(), newMode, timeout, callback);
  6937. connection.setMode(newMode);
  6938. }
  6939. bool CCovenSDSManager::unlock(__int64 connectionId, bool close, StringBuffer &connectionInfo)
  6940. {
  6941. Owned<CServerConnection> connection = getConnection(connectionId);
  6942. if (!connection) return false;
  6943. MemoryBuffer connInfo;
  6944. connection->getInfo(connInfo);
  6945. formatConnectionInfo(connInfo, connectionInfo);
  6946. if (close)
  6947. {
  6948. PROGLOG("forcing unlock & disconnection of connection : %s", connectionInfo.str());
  6949. Owned<CLCLockBlock> lockBlock = new CLCWriteLockBlock(dataRWLock, readWriteTimeout, __FILE__, __LINE__);
  6950. SDSManager->disconnect(connectionId, false);
  6951. }
  6952. else // leave connection open, just unlock
  6953. {
  6954. PROGLOG("forcing unlock for connection : %s", connectionInfo.str());
  6955. __int64 nodeId = ((CRemoteTreeBase *)connection->queryRoot())->queryServerId();
  6956. CHECKEDCRITICALBLOCK(lockCrit, fakeCritTimeout);
  6957. CLock *lock = queryLock(nodeId);
  6958. if (lock)
  6959. lock->unlock(connectionId);
  6960. }
  6961. return true;
  6962. }
  6963. bool CCovenSDSManager::unlock(__int64 treeId, ConnectionId connectionId, bool delayDelete)
  6964. {
  6965. CHECKEDCRITICALBLOCK(lockCrit, fakeCritTimeout);
  6966. CLock *lock = queryLock(treeId);
  6967. if (lock)
  6968. return lock->unlock(connectionId, delayDelete);
  6969. return false;
  6970. }
  6971. void CCovenSDSManager::unlockAll(__int64 treeId)
  6972. {
  6973. CHECKEDCRITICALBLOCK(lockCrit, fakeCritTimeout);
  6974. CLock *lock = queryLock(treeId);
  6975. if (lock)
  6976. lock->unlockAll();
  6977. }
  6978. LockStatus CCovenSDSManager::establishLock(CLock &lock, __int64 treeId, ConnectionId connectionId, SessionId sessionId, unsigned mode, unsigned timeout, IUnlockCallback &lockCallback)
  6979. {
  6980. LockStatus res = lock.lock(mode, timeout, connectionId, sessionId, lockCallback);
  6981. if (res == LockSucceeded && server.queryStopped())
  6982. {
  6983. lock.unlock(connectionId);
  6984. throw MakeSDSException(SDSExcpt_ServerStoppedLockAborted);
  6985. }
  6986. return res;
  6987. }
  6988. void CCovenSDSManager::lock(CServerRemoteTree &tree, const char *xpath, ConnectionId connectionId, SessionId sessionId, unsigned mode, unsigned timeout, IUnlockCallback &callback)
  6989. {
  6990. if (0 == ((RTM_LOCK_READ | RTM_LOCK_WRITE) & mode)) // no point in creating lock.
  6991. return;
  6992. CLock *lock = NULL;
  6993. StringBuffer sxpath;
  6994. if ('/' != *xpath)
  6995. {
  6996. sxpath.append('/').append(xpath);
  6997. if ('/' != sxpath.charAt(sxpath.length()-1))
  6998. sxpath.append('/');
  6999. xpath = sxpath.str();
  7000. }
  7001. else if ('/' != xpath[strlen(xpath)-1])
  7002. {
  7003. sxpath.append(xpath);
  7004. sxpath.append('/');
  7005. xpath = sxpath.str();
  7006. }
  7007. __int64 treeId = tree.queryServerId();
  7008. CHECKEDCRITICALBLOCK(lockCrit, fakeCritTimeout);
  7009. lock = lockTable.find(&treeId);
  7010. if (!lock)
  7011. {
  7012. IdPath idPath;
  7013. lock = new CLock(lockTable, treeId, idPath, xpath, mode, connectionId, sessionId);
  7014. lockTable.replace(*lock);
  7015. }
  7016. else
  7017. {
  7018. Linked<CLock> tmp = lock; // keep it alive could be destroyed whilst blocked in call below.
  7019. LockStatus result = establishLock(*lock, treeId, connectionId, sessionId, mode, timeout, callback);
  7020. if (result != LockSucceeded)
  7021. {
  7022. if (!queryConnection(connectionId)) return; // connection aborted.
  7023. StringBuffer s;
  7024. switch (result)
  7025. {
  7026. case LockFailed:
  7027. throw MakeSDSException(SDSExcpt_ConnectionAbsent, "Lost connection trying to establish lock on connection to : %s", xpath);
  7028. case LockTimedOut:
  7029. throw MakeSDSException(SDSExcpt_LockTimeout, "Lock timeout trying to establish lock to %s, existing lock info: %s", xpath, lock->toString(s).str());
  7030. case LockHeld:
  7031. throw MakeSDSException(SDSExcpt_LockHeld, "Lock is held trying to establish lock to %s, existing lock info: %s", xpath, lock->toString(s).str());
  7032. }
  7033. }
  7034. }
  7035. }
  7036. void CCovenSDSManager::createConnection(SessionId sessionId, unsigned mode, unsigned timeout, const char *xpath, CServerRemoteTree *&tree, ConnectionId &connectionId, bool primary, Owned<LinkingCriticalBlock> &connectCritBlock)
  7037. {
  7038. CRemoteTreeBase *_tree;
  7039. Linked<CRemoteTreeBase> linkedTree;
  7040. Owned<CServerConnection> connection;
  7041. StringBuffer _xpath;
  7042. if (!xpath || '/'!=*xpath)
  7043. xpath = _xpath.append('/').append(xpath).str();
  7044. struct CFreeExistingLocks
  7045. {
  7046. ~CFreeExistingLocks() { clear(); }
  7047. void clear()
  7048. {
  7049. ForEachItemIn(l, existingLockTrees)
  7050. SDSManager->unlock(existingLockTrees.item(l).queryServerId(), connId);
  7051. existingLockTrees.kill();
  7052. }
  7053. void setConnectionId(ConnectionId _connId) { connId = _connId; }
  7054. void add(CServerRemoteTree &tree)
  7055. {
  7056. tree.Link();
  7057. existingLockTrees.append(tree);
  7058. }
  7059. void remove(CServerRemoteTree &tree)
  7060. {
  7061. existingLockTrees.zap(tree);
  7062. }
  7063. bool isExisting(CServerRemoteTree &tree) { return NotFound != existingLockTrees.find(tree); }
  7064. ConnectionId connId;
  7065. IArrayOf<CServerRemoteTree> existingLockTrees;
  7066. } freeExistingLocks;
  7067. StringAttr deltaPath;
  7068. Owned<IPropertyTree> deltaChange;
  7069. Owned<CBranchChange> branchChange;
  7070. unsigned additions = 0;
  7071. try
  7072. {
  7073. struct LockUnblock
  7074. {
  7075. LockUnblock(ReadWriteLock &_rWLock) : rWLock(_rWLock)
  7076. {
  7077. lockedForWrite = rWLock.queryWriteLocked();
  7078. if (lockedForWrite) rWLock.unlockWrite();
  7079. else rWLock.unlockRead();
  7080. }
  7081. ~LockUnblock() { if (lockedForWrite) rWLock.lockWrite(); else rWLock.lockRead(); }
  7082. bool lockedForWrite;
  7083. ReadWriteLock &rWLock;
  7084. };
  7085. bool locked = false;
  7086. class CConnectExistingLockCallback : implements IUnlockCallback
  7087. {
  7088. CUnlockCallback lcb;
  7089. CheckedCriticalSection *connectCrit;
  7090. public:
  7091. CConnectExistingLockCallback(const char *xpath, ConnectionId connectionId, CServerRemoteTree &tree, CheckedCriticalSection *_connectCrit) : lcb(xpath, connectionId, tree), connectCrit(_connectCrit) { }
  7092. virtual void block()
  7093. {
  7094. if (connectCrit)
  7095. CHECKEDCRITENTER(*connectCrit, fakeCritTimeout);
  7096. lcb.block();
  7097. }
  7098. virtual void unblock()
  7099. {
  7100. lcb.unblock();
  7101. if (connectCrit)
  7102. CHECKEDCRITLEAVE(*connectCrit);
  7103. }
  7104. };
  7105. if (!RTM_MODE(mode, RTM_CREATE_ADD) && !RTM_MODE(mode, RTM_CREATE_UNIQUE)) // cannot be pending on existing locked nodes in these cases
  7106. {
  7107. CTimeMon tm(timeout);
  7108. connectionId = coven.getUniqueId();
  7109. Owned<CServerConnection> tmpConn = new CServerConnection(*this, connectionId, xpath, sessionId, mode, timeout, NULL, (ConnInfoFlags)0);
  7110. { CHECKEDCRITICALBLOCK(cTableCrit, fakeCritTimeout);
  7111. connections.replace(*LINK(tmpConn));
  7112. }
  7113. if (sessionId)
  7114. {
  7115. { LockUnblock b(dataRWLock);
  7116. tmpConn->subscribe(sessionId);
  7117. }
  7118. if (!queryConnection(connectionId)) // aborted
  7119. {
  7120. connectionId = 0;
  7121. throw MakeSDSException(SDSExcpt_AbortDuringConnection, " during connect");
  7122. }
  7123. }
  7124. freeExistingLocks.setConnectionId(connectionId);
  7125. try
  7126. {
  7127. for (;;)
  7128. {
  7129. try
  7130. {
  7131. Owned<IPropertyTreeIterator> iter = root->getElements(xpath+1);
  7132. iter->first();
  7133. while (iter->isValid())
  7134. {
  7135. CServerRemoteTree &existing = (CServerRemoteTree &) iter->query();
  7136. if (freeExistingLocks.isExisting(existing))
  7137. iter->next();
  7138. else
  7139. {
  7140. freeExistingLocks.add(existing);
  7141. {
  7142. unsigned remaining;
  7143. if (timeout)
  7144. {
  7145. if (tm.timedout(&remaining))
  7146. {
  7147. Linked<CLock> lock;
  7148. VStringBuffer timeoutMsg("Failed to establish lock to %s, timeout whilst retrying connection to orphaned connection path", xpath);
  7149. ForEachItemIn(f, freeExistingLocks.existingLockTrees)
  7150. {
  7151. CServerRemoteTree &e = freeExistingLocks.existingLockTrees.item(f);
  7152. {
  7153. CHECKEDCRITICALBLOCK(lockCrit, fakeCritTimeout);
  7154. CLock *_lock = queryLock(e.queryServerId());
  7155. if (_lock)
  7156. {
  7157. if (!lock)
  7158. timeoutMsg.append(", existing lock info: ");
  7159. timeoutMsg.newline();
  7160. lock.set(_lock);
  7161. }
  7162. }
  7163. if (lock)
  7164. lock->toString(timeoutMsg);
  7165. }
  7166. throw MakeSDSException(SDSExcpt_LockTimeout, "%s", timeoutMsg.str());
  7167. }
  7168. }
  7169. else
  7170. remaining = 0; // a timeout of 0 means fail immediately if locked
  7171. CConnectExistingLockCallback connectLockCallback(xpath, connectionId, existing, &connectCrit);
  7172. lock(existing, xpath, connectionId, sessionId, mode, remaining, connectLockCallback);
  7173. }
  7174. if (!queryConnection(connectionId)) // aborted
  7175. {
  7176. connectionId = 0;
  7177. throw MakeSDSException(SDSExcpt_AbortDuringConnection, " during connect");
  7178. }
  7179. iter.setown(root->getElements(xpath+1));
  7180. iter->first();
  7181. }
  7182. }
  7183. break;
  7184. }
  7185. catch (ISDSException *e) // don't treat waiting on a now orpahned node an error, since trying to lock to create (retry)
  7186. {
  7187. if (SDSExcpt_OrphanedNode != e->errorCode())
  7188. throw;
  7189. else
  7190. e->Release();
  7191. }
  7192. freeExistingLocks.clear();
  7193. }
  7194. }
  7195. catch (IException *)
  7196. {
  7197. CHECKEDCRITICALBLOCK(cTableCrit, fakeCritTimeout);
  7198. tmpConn->unsubscribeSession();
  7199. connections.removeExact(tmpConn);
  7200. connectionId = 0;
  7201. throw;
  7202. }
  7203. { CHECKEDCRITICALBLOCK(cTableCrit, fakeCritTimeout);
  7204. tmpConn->unsubscribeSession();
  7205. connections.removeExact(tmpConn);
  7206. }
  7207. }
  7208. try
  7209. {
  7210. if (RTM_MODE(mode, RTM_CREATE_ADD) || RTM_MODE(mode, RTM_CREATE_UNIQUE))
  7211. {
  7212. CHECKEDCRITICALBLOCK(blockedSaveCrit, fakeCritTimeout);
  7213. connection.setown(createConnectionInstance(root, sessionId, mode, timeout, xpath, _tree, connectionId, &deltaPath, deltaChange, branchChange, additions));
  7214. }
  7215. else
  7216. connection.setown(createConnectionInstance(root, sessionId, mode, timeout, xpath, _tree, connectionId, &deltaPath, deltaChange, branchChange, additions));
  7217. }
  7218. catch (IException *) // do not want to miss recording change to inc, under any circumstances.
  7219. {
  7220. if (deltaChange.get())
  7221. {
  7222. PROGLOG("Exception on RTM_CREATE caused call to saveDelta, xpath=%s", xpath);
  7223. saveDelta(deltaPath, *deltaChange);
  7224. }
  7225. throw;
  7226. }
  7227. linkedTree.set(_tree);
  7228. if (!connection)
  7229. {
  7230. connectionId = 0;
  7231. return;
  7232. }
  7233. assertex(_tree);
  7234. if (deltaChange.get())
  7235. {
  7236. CPTStack stack = connection->queryPTreePath();
  7237. if (connection->queryRoot() == SDSManager->queryRoot())
  7238. stack.pop();
  7239. stack.popn(additions);
  7240. connection->notify();
  7241. SDSManager->startNotification(*deltaChange, stack, *branchChange);
  7242. saveDelta(deltaPath, *deltaChange);
  7243. }
  7244. connectionId = connection->queryConnectionId();
  7245. if (freeExistingLocks.isExisting(*(CServerRemoteTree *)_tree))
  7246. {
  7247. locked = true;
  7248. freeExistingLocks.remove(*(CServerRemoteTree *)_tree);
  7249. connectCritBlock.clear();
  7250. }
  7251. { CHECKEDCRITICALBLOCK(cTableCrit, fakeCritTimeout);
  7252. connections.replace(*LINK(connection));
  7253. }
  7254. try
  7255. {
  7256. if (!locked)
  7257. {
  7258. CConnectExistingLockCallback connectLockCallback(xpath, connectionId, *(CServerRemoteTree *)_tree, connectCritBlock.get()?&connectCrit:NULL);
  7259. lock(*(CServerRemoteTree *)_tree, xpath, connectionId, sessionId, mode, timeout, connectLockCallback);
  7260. }
  7261. }
  7262. catch (IException *)
  7263. {
  7264. { CHECKEDCRITICALBLOCK(cTableCrit, fakeCritTimeout);
  7265. connections.removeExact(connection);
  7266. }
  7267. throw;
  7268. }
  7269. catch (DALI_CATCHALL)
  7270. {
  7271. { CHECKEDCRITICALBLOCK(cTableCrit, fakeCritTimeout);
  7272. connections.removeExact(connection);
  7273. }
  7274. throw;
  7275. }
  7276. if (sessionId)
  7277. {
  7278. // unlock global lock whilst subscription occurs, incase it calls me back inline (e.g. during an immediate abort)
  7279. {
  7280. LockUnblock b(dataRWLock);
  7281. connection->subscribe(sessionId);
  7282. }
  7283. // subscription may have already disconnected by this stage.
  7284. if (!queryConnection(connectionId))
  7285. {
  7286. connectionId = 0;
  7287. throw MakeSDSException(SDSExcpt_AbortDuringConnection, " during connect");
  7288. }
  7289. }
  7290. }
  7291. catch (IException *e)
  7292. {
  7293. if (SDSExcpt_OrphanedNode != e->errorCode())
  7294. throw;
  7295. e->Release();
  7296. connectionId = 0;
  7297. return;
  7298. }
  7299. // could have been blocked, now freed but in the meantime *this* connection has been aborted.
  7300. if (!queryConnection(connectionId))
  7301. {
  7302. unlock(_tree->queryServerId(), connectionId);
  7303. connectionId = 0;
  7304. throw MakeSDSException(SDSExcpt_AbortDuringConnection, " during connect");
  7305. }
  7306. connection->setEstablished();
  7307. tree = (CServerRemoteTree *) LINK(_tree);
  7308. connectionId = connection->queryConnectionId();
  7309. }
  7310. CServerConnection *CCovenSDSManager::queryConnection(ConnectionId id)
  7311. {
  7312. CHECKEDCRITICALBLOCK(cTableCrit, fakeCritTimeout);
  7313. return (CServerConnection *)connections.find(&id);
  7314. }
  7315. CServerConnection *CCovenSDSManager::getConnection(ConnectionId id)
  7316. {
  7317. CHECKEDCRITICALBLOCK(cTableCrit, fakeCritTimeout);
  7318. CServerConnection *conn = (CServerConnection *)connections.find(&id);
  7319. if (conn) conn->Link();
  7320. return conn;
  7321. }
  7322. void CCovenSDSManager::disconnect(ConnectionId id, bool deleteRoot, Owned<CLCLockBlock> *lockBlock)
  7323. {
  7324. Linked<CServerConnection> connection;
  7325. { CHECKEDCRITICALBLOCK(cTableCrit, fakeCritTimeout);
  7326. connection.set(queryConnection(id));
  7327. if (!connection)
  7328. return;
  7329. connections.removeExact(connection);
  7330. }
  7331. Linked<CServerRemoteTree> tree = (CServerRemoteTree *)connection->queryRootUnvalidated();
  7332. if (!tree) return;
  7333. unsigned index = (unsigned)-1;
  7334. if (connection->queryParent())
  7335. {
  7336. if (deleteRoot || RTM_MODE(connection->queryMode(), RTM_DELETE_ON_DISCONNECT))
  7337. {
  7338. CHECKEDCRITICALBLOCK(lockCrit, fakeCritTimeout);
  7339. CLock *lock = queryLock(tree->queryServerId());
  7340. if (lock)
  7341. {
  7342. deleteRoot = false;
  7343. lock->setDROLR((CServerRemoteTree *)connection->queryParent(), tree);
  7344. }
  7345. else
  7346. deleteRoot = true;
  7347. }
  7348. if ((unsigned)-1 == index)
  7349. index = connection->queryParent()->queryChildIndex(connection->queryRootUnvalidated());
  7350. }
  7351. else
  7352. deleteRoot = false;
  7353. bool orphaned = ((CServerRemoteTree*)connection->queryRootUnvalidated())->isOrphaned();
  7354. StringBuffer path;
  7355. if (!orphaned) // see below, path/delta only recorded if not orphaned. Cannot calulate path if connection root has already been deleted
  7356. connection->queryPTreePath().getAbsolutePath(path);
  7357. else
  7358. DBGLOG("Disconnecting orphaned connection: %s, deleteRoot=%s", connection->queryXPath(), deleteRoot?"true":"false");
  7359. // Still want disconnection to be performed & recorded, if orphaned
  7360. if (!deleteRoot && unlock(tree->queryServerId(), id, true)) // unlock returns true if last unlock and there was a setDROLR on it
  7361. deleteRoot = true;
  7362. if (deleteRoot)
  7363. {
  7364. if (lockBlock)
  7365. {
  7366. lockBlock->clear();
  7367. lockBlock->setown(new CLCWriteLockBlock(dataRWLock, readWriteTimeout, __FILE__, __LINE__));
  7368. }
  7369. connection->queryParent()->removeTree(tree);
  7370. writeTransactions++;
  7371. if (!orphaned)
  7372. {
  7373. Owned<IPropertyTree> changeTree = createPTree(RESERVED_CHANGE_NODE);
  7374. IPropertyTree *d = changeTree->setPropTree(DELETE_TAG, createPTree());
  7375. d->setProp("@name", tree->queryName());
  7376. d->setPropInt("@pos", index+1);
  7377. Owned<CBranchChange> branchChange = new CBranchChange(*tree);
  7378. branchChange->noteChange(PDS_Deleted, PDS_Deleted);
  7379. CPTStack stack = connection->queryPTreePath();
  7380. stack.pop();
  7381. if (connection->queryRootUnvalidated() == SDSManager->queryRoot())
  7382. stack.pop();
  7383. if (!RTM_MODE(connection->queryMode(), RTM_INTERNAL))
  7384. {
  7385. connection->notify();
  7386. SDSManager->startNotification(*changeTree, stack, *branchChange);
  7387. }
  7388. StringBuffer head;
  7389. const char *tail = splitXPath(path.str(), head);
  7390. CHECKEDCRITICALBLOCK(blockedSaveCrit, fakeCritTimeout);
  7391. if (NotFound != index)
  7392. saveDelta(head.str(), *changeTree);
  7393. else
  7394. { // NB: don't believe this can happen, but last thing want to do is save duff delete delta.
  7395. WARNLOG("** CCovenSDSManager::disconnect - index position lost **");
  7396. PrintStackReport();
  7397. }
  7398. }
  7399. }
  7400. tree.clear();
  7401. connection->unsubscribeSession();
  7402. }
  7403. StringBuffer &formatUsageStats(MemoryBuffer &src, StringBuffer &out)
  7404. {
  7405. unsigned c;
  7406. src.read(c);
  7407. out.append("Connections : ").append(c).newline();
  7408. src.read(c);
  7409. out.append("Locks : ").append(c).newline();
  7410. src.read(c);
  7411. out.append("Subscribers : ").append(c).newline();
  7412. src.read(c);
  7413. out.append("Connection subscriptions : ").append(c).newline();
  7414. return out;
  7415. }
  7416. StringBuffer &formatConnectionInfo(MemoryBuffer &src, StringBuffer &out)
  7417. {
  7418. ConnectionId connectionId;
  7419. StringAttr xpath;
  7420. SessionId sessionId;
  7421. unsigned mode;
  7422. unsigned timeout;
  7423. bool established;
  7424. src.read(connectionId).read(xpath).read(sessionId).read(mode).read(timeout).read(established);
  7425. out.append("ConnectionId=").appendf("%" I64F "x", connectionId).append(", xpath=").append(xpath).append(", sessionId=").appendf("%" I64F "x", sessionId).append(", mode=").append(mode).append(", timeout=");
  7426. if (INFINITE == timeout)
  7427. out.append("INFINITE");
  7428. else
  7429. out.append(timeout);
  7430. out.append(established?"":" [BLOCKED]");
  7431. return out;
  7432. }
  7433. StringBuffer &formatSubscriberInfo(MemoryBuffer &src, StringBuffer &out)
  7434. {
  7435. SubscriptionId subscriptionId;
  7436. bool sub;
  7437. StringAttr xpath;
  7438. src.read(subscriptionId).read(sub).read(xpath);
  7439. out.append("SubscriptionId=").appendf("%" I64F "x", subscriptionId).append(", xpath=").append(xpath).append(", sub=").append(sub?"true":"false");
  7440. return out;
  7441. }
  7442. StringBuffer &formatNodeSubscriberInfo(MemoryBuffer &src, StringBuffer &out)
  7443. {
  7444. SubscriptionId subscriptionId;
  7445. StringAttr xpath;
  7446. unsigned nodeCount;
  7447. src.read(subscriptionId).read(xpath).read(nodeCount);
  7448. out.append("SubscriptionId=").appendf("%" I64F "x", subscriptionId).append(", xpath=").append(xpath).append(", nodes=").append(nodeCount);
  7449. return out;
  7450. }
  7451. StringBuffer &formatConnections(MemoryBuffer &src, StringBuffer &out)
  7452. {
  7453. unsigned count;
  7454. src.read(count);
  7455. if (count)
  7456. {
  7457. while (count--)
  7458. {
  7459. formatConnectionInfo(src, out);
  7460. if (count) out.newline();
  7461. }
  7462. }
  7463. else
  7464. out.append("No current connections");
  7465. return out;
  7466. }
  7467. StringBuffer &formatSubscribers(MemoryBuffer &src, StringBuffer &out)
  7468. {
  7469. unsigned sdsSubscribers, sdsNodeSubscribers=0;
  7470. src.read(sdsSubscribers);
  7471. unsigned s = sdsSubscribers;
  7472. while (s--)
  7473. {
  7474. formatSubscriberInfo(src, out);
  7475. if (s) out.newline();
  7476. }
  7477. if (src.remaining())
  7478. {
  7479. src.read(sdsNodeSubscribers);
  7480. s = sdsNodeSubscribers;
  7481. while (s--)
  7482. {
  7483. formatNodeSubscriberInfo(src, out);
  7484. if (s) out.newline();
  7485. }
  7486. }
  7487. out.newline().appendf("%d xpath subscribers, %d node subscribers\n", sdsSubscribers, sdsNodeSubscribers);
  7488. return out;
  7489. }
  7490. unsigned CCovenSDSManager::countConnections()
  7491. {
  7492. CHECKEDCRITICALBLOCK(cTableCrit, fakeCritTimeout);
  7493. return connections.count();
  7494. }
  7495. unsigned CCovenSDSManager::countActiveLocks()
  7496. {
  7497. unsigned activeLocks = 0;
  7498. CHECKEDCRITICALBLOCK(lockCrit, fakeCritTimeout);
  7499. SuperHashIteratorOf<CLock> iter(lockTable.queryBaseTable());
  7500. ForEach(iter) {
  7501. CLock &lock = iter.query();
  7502. if (lock.lockCount()) activeLocks++;
  7503. }
  7504. return activeLocks;
  7505. }
  7506. ILockInfoCollection *CCovenSDSManager::getLocks(const char *ipPattern, const char *xpathPattern)
  7507. {
  7508. Owned<ILockInfoCollection> lockInfoCollection = createLockInfoCollection();
  7509. bool filteredConnections = !isEmptyString(ipPattern);
  7510. bool filteredXPaths = !isEmptyString(xpathPattern);
  7511. CLockInfoArray locks;
  7512. {
  7513. CHECKEDCRITICALBLOCK(lockCrit, fakeCritTimeout);
  7514. SuperHashIteratorOf<CLock> iter(lockTable.queryBaseTable());
  7515. ForEach(iter)
  7516. {
  7517. CLock &lock = iter.query();
  7518. if (lock.lockCount())
  7519. {
  7520. if (!filteredXPaths || WildMatch(lock.queryXPath(), xpathPattern))
  7521. locks.append(* lock.getLockInfo());
  7522. }
  7523. }
  7524. }
  7525. if (filteredConnections)
  7526. {
  7527. ForEachItemIn(c, locks)
  7528. {
  7529. ILockInfo &lockInfo = locks.item(c);
  7530. lockInfo.prune(ipPattern);
  7531. }
  7532. }
  7533. ForEachItemIn(l, locks)
  7534. lockInfoCollection->add(* LINK(&locks.item(l)));
  7535. return lockInfoCollection.getClear();
  7536. }
  7537. MemoryBuffer &CCovenSDSManager::collectUsageStats(MemoryBuffer &out)
  7538. {
  7539. { CHECKEDCRITICALBLOCK(cTableCrit, fakeCritTimeout);
  7540. out.append(connections.count());
  7541. }
  7542. unsigned activeLocks = 0;
  7543. { CHECKEDCRITICALBLOCK(lockCrit, fakeCritTimeout);
  7544. SuperHashIteratorOf<CLock> iter(lockTable.queryBaseTable());
  7545. ForEach(iter)
  7546. {
  7547. CLock &lock = iter.query();
  7548. if (lock.lockCount()) activeLocks++;
  7549. }
  7550. }
  7551. out.append(activeLocks);
  7552. out.append(subscribers.count());
  7553. out.append(connectionSubscriptionManager->querySubscribers());
  7554. return out;
  7555. }
  7556. MemoryBuffer &CCovenSDSManager::collectConnections(MemoryBuffer &out)
  7557. {
  7558. CHECKEDCRITICALBLOCK(cTableCrit, fakeCritTimeout);
  7559. out.append(connections.count());
  7560. SuperHashIteratorOf<CServerConnection> iter(connections.queryBaseTable());
  7561. ForEach(iter)
  7562. iter.query().getInfo(out);
  7563. return out;
  7564. }
  7565. MemoryBuffer &CCovenSDSManager::collectSubscribers(MemoryBuffer &out)
  7566. {
  7567. CHECKEDCRITICALBLOCK(sTableCrit, fakeCritTimeout);
  7568. out.append(subscribers.count());
  7569. SuperHashIteratorOf<CSubscriberContainer> sdsIter(subscribers.queryBaseTable());
  7570. ForEach(sdsIter)
  7571. sdsIter.query().getInfo(out);
  7572. nodeSubscriptionManager->collectSubscribers(out);
  7573. return out;
  7574. }
  7575. void CCovenSDSManager::blockingSave(unsigned *writeTransactions)
  7576. {
  7577. CHECKEDDALIREADLOCKBLOCK(SDSManager->dataRWLock, readWriteTimeout); // block all write actions whilst saving
  7578. CHECKEDCRITICALBLOCK(blockedSaveCrit, fakeCritTimeout);
  7579. if (writeTransactions)
  7580. *writeTransactions = SDSManager->writeTransactions;
  7581. // JCS - could in theory, not block, but abort save.
  7582. SDSManager->saveStore();
  7583. }
  7584. bool CCovenSDSManager::updateEnvironment(IPropertyTree *newEnv, bool forceGroupUpdate, StringBuffer &response)
  7585. {
  7586. Owned<IRemoteConnection> conn = querySDS().connect("/",myProcessSession(),0, INFINITE);
  7587. if (conn)
  7588. {
  7589. Owned<IPropertyTree> root = conn->getRoot();
  7590. Owned<IPropertyTree> oldEnvironment = root->getPropTree("Environment");
  7591. if (oldEnvironment.get())
  7592. {
  7593. StringBuffer bakname;
  7594. Owned<IFileIO> io = createUniqueFile(NULL, "environment", "bak", bakname);
  7595. Owned<IFileIOStream> fstream = createBufferedIOStream(io);
  7596. toXML(oldEnvironment, *fstream); // formatted (default)
  7597. root->removeTree(oldEnvironment);
  7598. }
  7599. root->addPropTree("Environment", LINK(newEnv));
  7600. root.clear();
  7601. conn->commit();
  7602. conn->close();
  7603. StringBuffer messages;
  7604. initClusterGroups(forceGroupUpdate, messages, oldEnvironment);
  7605. response.append(messages);
  7606. PROGLOG("Environment and node groups updated");
  7607. }
  7608. return true;
  7609. }
  7610. // TODO
  7611. StringBuffer &CCovenSDSManager::getExternalReport(StringBuffer &out)
  7612. {
  7613. return out;
  7614. }
  7615. StringBuffer &CCovenSDSManager::getConnections(StringBuffer &out)
  7616. {
  7617. MemoryBuffer mb;
  7618. formatConnections(collectConnections(mb), out);
  7619. return out;
  7620. }
  7621. StringBuffer &CCovenSDSManager::getSubscribers(StringBuffer &out)
  7622. {
  7623. MemoryBuffer mb;
  7624. formatSubscribers(collectSubscribers(mb), out);
  7625. return out;
  7626. }
  7627. StringBuffer &CCovenSDSManager::getUsageStats(StringBuffer &out)
  7628. {
  7629. MemoryBuffer mb;
  7630. formatUsageStats(collectUsageStats(mb), out);
  7631. return out;
  7632. }
  7633. void CCovenSDSManager::handleNodeNotify(notifications n, CServerRemoteTree &tree)
  7634. {
  7635. const char *handlerKey = tree.queryProp(NOTIFY_ATTR);
  7636. assertex(handlerKey);
  7637. CSDSNotifyHandlerMapping *m = nodeNotifyHandlers.find(handlerKey);
  7638. if (!m)
  7639. {
  7640. LOG(MCwarning, unknownJob, "Unknown notify handler name \"%s\", handing event %s", handlerKey, notificationStr(n));
  7641. return;
  7642. }
  7643. switch (n)
  7644. {
  7645. case notify_delete:
  7646. try { m->query().removed(tree); }
  7647. catch (IException *e)
  7648. {
  7649. StringBuffer s("Exception calling ISDSNotifyHandler->removed(<tree>), for handler\"");
  7650. s.append(notificationStr(n)).append("\"");
  7651. EXCLOG(e, s.str());
  7652. e->Release();
  7653. }
  7654. break;
  7655. default:
  7656. LOG(MCerror, unknownJob, "Unknown notification type (%d)", n);
  7657. break;
  7658. }
  7659. }
  7660. void CCovenSDSManager::handleNotify(CSubscriberContainerBase *_subscriber, MemoryBuffer &notifyData)
  7661. {
  7662. Owned<CSubscriberContainerBase> subscriber = _subscriber;
  7663. class CNotifyPoolFactory : public IThreadFactory, public CInterface
  7664. {
  7665. class CNotifyHandler : implements IPooledThread, public CInterface
  7666. {
  7667. DECL_NAMEDCOUNT;
  7668. public:
  7669. IMPLEMENT_IINTERFACE;
  7670. CNotifyHandler() { INIT_NAMEDCOUNT; }
  7671. void init(void *startInfo)
  7672. {
  7673. n.setown((CSubscriberNotifier *)startInfo);
  7674. }
  7675. void main()
  7676. {
  7677. try
  7678. {
  7679. n->notify();
  7680. }
  7681. catch (IException *e)
  7682. {
  7683. EXCLOG(e, "CNotifyHandler");
  7684. e->Release();
  7685. }
  7686. n.clear();
  7687. }
  7688. bool canReuse()
  7689. {
  7690. return true;
  7691. }
  7692. bool stop()
  7693. {
  7694. return true;
  7695. }
  7696. private:
  7697. Linked<CSubscriberNotifier> n;
  7698. };
  7699. public:
  7700. IMPLEMENT_IINTERFACE;
  7701. IPooledThread *createNew()
  7702. {
  7703. return new CNotifyHandler();
  7704. }
  7705. };
  7706. if (!notifyPool)
  7707. {
  7708. CNotifyPoolFactory *factory = new CNotifyPoolFactory;
  7709. notifyPool.setown(createThreadPool("SDS Notification Pool", factory, this, SUBNTFY_POOL_SIZE));
  7710. factory->Release();
  7711. }
  7712. CHECKEDCRITICALBLOCK(nfyTableCrit, fakeCritTimeout);
  7713. SubscriptionId id = subscriber->queryId();
  7714. CSubscriberNotifier *notifier = subscriberNotificationTable.find(id);
  7715. /* Must clear 'subscriber' before leaving ntyTableCrit block, so that the notifier thread owns and destroys it
  7716. * It cannot be destroyed here, because this method may have been called inside the SDS lock during a node
  7717. * delete and do not want to call into subscriber manager as that can deadlock if processing a request
  7718. * already that requires SDS lock.
  7719. */
  7720. if (notifier)
  7721. {
  7722. subscriber.clear();
  7723. notifier->queueChange(notifyData);
  7724. }
  7725. else
  7726. {
  7727. Owned<CSubscriberNotifier> _notifier = new CSubscriberNotifier(subscriberNotificationTable, *subscriber.getClear(), notifyData);
  7728. subscriberNotificationTable.replace(*_notifier);
  7729. notifyPool->start(_notifier.getClear()); // NB: takes ownership (during init())
  7730. }
  7731. }
  7732. /////////////////
  7733. class CSubscriberNotifyScanner : public CInterface
  7734. {
  7735. DECL_NAMEDCOUNT;
  7736. CPTStack stack;
  7737. Linked<IPropertyTree> changeTree;
  7738. Linked<CBranchChange> rootChanges;
  7739. StringBuffer xpath;
  7740. CSubscriberArray subs;
  7741. public:
  7742. struct PushPop
  7743. {
  7744. PushPop(CPTStack &_stack, CRemoteTreeBase &tree) : stack(_stack) { tree.Link(); stack.append(tree); }
  7745. ~PushPop() { stack.pop(); }
  7746. CPTStack &stack;
  7747. };
  7748. CSubscriberNotifyScanner(IPropertyTree &_changeTree, CPTStack &_stack, CBranchChange &_rootChanges) : changeTree(&_changeTree), rootChanges(&_rootChanges), stack(_stack)
  7749. {
  7750. INIT_NAMEDCOUNT;
  7751. SDSManager->querySubscriberTable().getSubscribers(subs);
  7752. }
  7753. enum SubCommitType { subCommitNone, subCommitExact, subCommitBelow, subCommitAbove };
  7754. SubCommitType match(const char *head, const char *path)
  7755. {
  7756. bool wild = false;
  7757. for (;;)
  7758. {
  7759. if (wild)
  7760. {
  7761. if (*head == *path)
  7762. {
  7763. path++;
  7764. wild = false;
  7765. }
  7766. else if ('/' == *head)
  7767. wild = false;
  7768. }
  7769. else if ('*' == *path)
  7770. {
  7771. path++;
  7772. wild = true;
  7773. }
  7774. else if (*head != *path)
  7775. return subCommitNone;
  7776. else
  7777. path++;
  7778. head++;
  7779. if ('\0' == *path)
  7780. {
  7781. if ('\0' == *head)
  7782. return subCommitExact; // absolute match
  7783. else if (!wild && '/' != *head) // e.g. change=/a/bc, subscriber=/a/b
  7784. return subCommitNone;
  7785. return subCommitBelow; // e.g. change=/a/b/c, subscriber=/a/b
  7786. }
  7787. else
  7788. {
  7789. if ('\0' == *head)
  7790. return subCommitAbove; // e.g. change=/a/b, subscriber=/a/b/c - not matched yet, but returning true keeps it from being pruned
  7791. }
  7792. }
  7793. }
  7794. void scan()
  7795. {
  7796. xpath.clear();
  7797. stack.toString(xpath);
  7798. if (stack.ordinality() && (rootChanges->tree == &(stack.tos()))) stack.pop();
  7799. CSubscriberArray pruned;
  7800. scan(*rootChanges, stack, pruned);
  7801. }
  7802. bool prune(const char *xpath, CSubscriberCopyArray &candidates, CSubscriberArray &pruned)
  7803. {
  7804. ForEachItemInRev(s, subs)
  7805. {
  7806. CSubscriberContainer &subscriber = subs.item(s);
  7807. SubCommitType subCommit;
  7808. if (subscriber.isUnsubscribed())
  7809. subCommit = subCommitNone;
  7810. else
  7811. subCommit = match(xpath, subscriber.queryXPath());
  7812. switch (subCommit)
  7813. {
  7814. case subCommitNone:
  7815. {
  7816. pruned.append(*LINK(&subscriber));
  7817. subs.remove(s);
  7818. break;
  7819. }
  7820. case subCommitExact:
  7821. {
  7822. candidates.append(subscriber);
  7823. break;
  7824. }
  7825. case subCommitBelow: // e.g. change=/a/b/c, subscriber=/a/b
  7826. {
  7827. if (!subscriber.querySub())
  7828. {
  7829. pruned.append(*LINK(&subscriber));
  7830. subs.remove(s);
  7831. }
  7832. else
  7833. candidates.append(subscriber);
  7834. break;
  7835. }
  7836. case subCommitAbove: // e.g. change=/a/b, subscriber=/a/b/c
  7837. break; // keep in subs, deeper changes may match
  7838. default:
  7839. throwUnexpected();
  7840. }
  7841. }
  7842. return (subs.ordinality() > 0);
  7843. }
  7844. // recurse down all matching subscription stubs while qualified
  7845. void scanAll(PDState state, CBranchChange &changes, CPTStack &stack, CSubscriberArray &pruned)
  7846. {
  7847. CSubscriberCopyArray candidates;
  7848. if (prune(xpath.str(), candidates, pruned))
  7849. {
  7850. MemoryBuffer notifyData;
  7851. if (candidates.ordinality())
  7852. {
  7853. ForEachItemInRev(s, candidates)
  7854. {
  7855. CSubscriberContainer &subscriber = candidates.item(s);
  7856. if (!subscriber.isUnsubscribed())
  7857. {
  7858. if (subscriber.qualify(stack, true))
  7859. {
  7860. if (0 == notifyData.length())
  7861. buildNotifyData(notifyData, state, &stack, NULL);
  7862. SDSManager->handleNotify(LINK(&subscriber), notifyData);
  7863. }
  7864. else
  7865. pruned.append(*LINK(&subscriber));
  7866. }
  7867. subs.zap(subscriber);
  7868. }
  7869. }
  7870. else
  7871. {
  7872. if (0 == changes.children.ordinality())
  7873. {
  7874. ForEachItemInRev(s, subs)
  7875. {
  7876. CSubscriberContainer &subscriber = subs.item(s);
  7877. if (!subscriber.isUnsubscribed())
  7878. {
  7879. if (subscriber.qualify(stack, true))
  7880. {
  7881. if (0 == notifyData.length())
  7882. buildNotifyData(notifyData, state, &stack, NULL);
  7883. SDSManager->handleNotify(LINK(&subscriber), notifyData);
  7884. }
  7885. else
  7886. pruned.append(*LINK(&subscriber));
  7887. }
  7888. subs.remove(s);
  7889. }
  7890. }
  7891. else
  7892. {
  7893. ForEachItemIn (c, changes.children)
  7894. {
  7895. CBranchChange &childChange = changes.children.item(c);
  7896. PushPop pp(stack, *childChange.tree);
  7897. size32_t parentLength = xpath.length();
  7898. xpath.append('/').append(childChange.tree->queryName());
  7899. CSubscriberArray _pruned;
  7900. scanAll(state, childChange, stack, _pruned);
  7901. ForEachItemIn(i, _pruned) subs.append(*LINK(&_pruned.item(i)));
  7902. if (0 == subs.ordinality())
  7903. break;
  7904. xpath.setLength(parentLength);
  7905. }
  7906. }
  7907. }
  7908. }
  7909. }
  7910. void scan(CBranchChange &changes, CPTStack &stack, CSubscriberArray &pruned)
  7911. {
  7912. CSubscriberCopyArray candidates;
  7913. if (!prune(xpath.str(), candidates, pruned))
  7914. return;
  7915. PushPop pp(stack, *changes.tree);
  7916. if (PDS_Deleted == (changes.local & PDS_Deleted))
  7917. {
  7918. scanAll(changes.local, changes, stack, pruned);
  7919. return;
  7920. }
  7921. else if (candidates.ordinality()) // xpath matched some subscribers, and/or below some, need to check for sub subscribers
  7922. {
  7923. bool ret = false;
  7924. // avoid notifying on PDS_Structure only, which signifies changes deeper down only
  7925. MemoryBuffer notifyData;
  7926. if (changes.state && changes.local && (changes.local != PDS_Structure))
  7927. {
  7928. int lastSendValue = -1;
  7929. ForEachItemInRev(s, candidates)
  7930. {
  7931. CSubscriberContainer &subscriber = candidates.item(s);
  7932. if (!subscriber.isUnsubscribed())
  7933. {
  7934. if (subscriber.qualify(stack, false))
  7935. {
  7936. if (subscriber.querySendValue())
  7937. {
  7938. if (1 != lastSendValue)
  7939. {
  7940. MemoryBuffer mb;
  7941. changes.tree->getPropBin(NULL, mb);
  7942. buildNotifyData(notifyData.clear(), changes.state, &stack, &mb);
  7943. lastSendValue = 1;
  7944. }
  7945. }
  7946. else
  7947. {
  7948. if (0 != lastSendValue)
  7949. {
  7950. buildNotifyData(notifyData.clear(), changes.state, &stack, NULL);
  7951. lastSendValue = 0;
  7952. }
  7953. }
  7954. SDSManager->handleNotify(LINK(&subscriber), notifyData);
  7955. }
  7956. else
  7957. pruned.append(*LINK(&subscriber));
  7958. }
  7959. subs.zap(subscriber);
  7960. }
  7961. }
  7962. }
  7963. ForEachItemIn(c, changes.children)
  7964. {
  7965. CBranchChange &childChanges = changes.children.item(c);
  7966. size32_t parentLength = xpath.length();
  7967. xpath.append('/').append(childChanges.tree->queryName());
  7968. CSubscriberArray pruned;
  7969. scan(childChanges, stack, pruned);
  7970. ForEachItemIn(i, pruned) subs.append(*LINK(&pruned.item(i)));
  7971. if (0 == subs.ordinality())
  7972. break;
  7973. xpath.setLength(parentLength);
  7974. }
  7975. }
  7976. };
  7977. void CCovenSDSManager::startNotification(IPropertyTree &changeTree, CPTStack &stack, CBranchChange &changes)
  7978. {
  7979. class CScanNotifyPoolFactory : public IThreadFactory, public CInterface
  7980. {
  7981. class CScanNotifyHandler : implements IPooledThread, public CInterface
  7982. {
  7983. public:
  7984. IMPLEMENT_IINTERFACE;
  7985. void init(void *startInfo)
  7986. {
  7987. n.set((CSubscriberNotifyScanner *)startInfo);
  7988. }
  7989. void main()
  7990. {
  7991. try
  7992. {
  7993. n->scan();
  7994. }
  7995. catch (IException *e)
  7996. {
  7997. EXCLOG(e, "CScanNotifyHandler");
  7998. e->Release();
  7999. }
  8000. n.clear();
  8001. }
  8002. bool canReuse()
  8003. {
  8004. return true;
  8005. }
  8006. bool stop()
  8007. {
  8008. return true;
  8009. }
  8010. private:
  8011. Linked<CSubscriberNotifyScanner> n;
  8012. };
  8013. public:
  8014. IMPLEMENT_IINTERFACE;
  8015. IPooledThread *createNew()
  8016. {
  8017. return new CScanNotifyHandler();
  8018. }
  8019. };
  8020. if (!scanNotifyPool)
  8021. {
  8022. CScanNotifyPoolFactory *factory = new CScanNotifyPoolFactory;
  8023. scanNotifyPool.setown(createThreadPool("SDS Scan-Notification Pool", factory, this, SUBSCAN_POOL_SIZE));
  8024. factory->Release();
  8025. }
  8026. Owned<CSubscriberNotifyScanner> scan = new CSubscriberNotifyScanner(changeTree, stack, changes);
  8027. scanNotifyPool->start(scan.get());
  8028. }
  8029. void CCovenSDSManager::deleteExternal(__int64 index)
  8030. {
  8031. if (server.queryStopped()) return;
  8032. CHECKEDCRITICALBLOCK(extCrit, fakeCritTimeout);
  8033. IExternalHandler *handler = queryExternalHandler(EF_BinaryValue);
  8034. assertex(handler);
  8035. StringBuffer name(EXTERNAL_NAME_PREFIX);
  8036. name.append(index);
  8037. handler->remove(name.str());
  8038. }
  8039. void CCovenSDSManager::serializeExternal(__int64 index, IPropertyTree &owner, MemoryBuffer &mb, bool withValue)
  8040. {
  8041. CHECKEDCRITICALBLOCK(extCrit, fakeCritTimeout);
  8042. IExternalHandler *extHandler = queryExternalHandler(EF_BinaryValue);
  8043. assertex(extHandler);
  8044. try
  8045. {
  8046. StringBuffer name(EXTERNAL_NAME_PREFIX);
  8047. name.append(index);
  8048. extHandler->read(name.str(), owner, mb, withValue);
  8049. }
  8050. catch (IException *)
  8051. {
  8052. extHandler->resetAsExternal(owner);
  8053. throw;
  8054. }
  8055. }
  8056. void CCovenSDSManager::writeExternal(CServerRemoteTree &tree, bool direct, __int64 existing)
  8057. {
  8058. CHECKEDCRITICALBLOCK(extCrit, fakeCritTimeout);
  8059. __int64 index;
  8060. if (existing)
  8061. index = existing;
  8062. else
  8063. index = getNextExternal();
  8064. IExternalHandler *extHandler = queryExternalHandler(EF_BinaryValue);
  8065. assertex(extHandler);
  8066. tree.removeProp(EXT_ATTR);
  8067. StringBuffer name(EXTERNAL_NAME_PREFIX);
  8068. extHandler->write(name.append(index).str(), tree);
  8069. tree.setPropInt64(EXT_ATTR, index);
  8070. extHandler->resetAsExternal(tree);
  8071. // setPropInt64(EXT_ATTR, index); // JCSMORE not necessary
  8072. }
  8073. // ISubscriptionManager
  8074. void CCovenSDSManager::add(ISubscription *sub, SubscriptionId id)
  8075. {
  8076. CHECKEDCRITICALBLOCK(sTableCrit, fakeCritTimeout);
  8077. subscribers.replace(* new CSubscriberContainer(sub, id));
  8078. }
  8079. void CCovenSDSManager::remove(SubscriptionId id)
  8080. {
  8081. CHECKEDCRITICALBLOCK(sTableCrit, fakeCritTimeout);
  8082. subscribers.remove(&id);
  8083. }
  8084. // IExceptionHandler
  8085. static bool processingUnhandled = false;
  8086. static bool handled = false;
  8087. static CheckedCriticalSection unhandledCrit;
  8088. bool CCovenSDSManager::fireException(IException *e)
  8089. {
  8090. //This code is rather dodgy (and causes more problems than it solves)
  8091. // so ignore unhandled exceptions for the moment!
  8092. LOG(MCoperatorError, unknownJob, e, "Caught unhandled exception!");
  8093. return true;
  8094. { CHECKEDCRITICALBLOCK(unhandledCrit, fakeCritTimeout);
  8095. if (processingUnhandled)
  8096. {
  8097. if (handled)
  8098. {
  8099. LOG(MCdisaster, unknownJob, e, "FATAL, too many exceptions");
  8100. return false; // did not successfully handle.
  8101. }
  8102. LOG(MCoperatorError, unknownJob, e, "Exception while restarting or shutting down");
  8103. return true;
  8104. }
  8105. handled = false;
  8106. processingUnhandled = true;
  8107. }
  8108. // Handle exception on a separate thread, to avoid complication with joining/restarting deadlocks.
  8109. class CHandleException : public Thread
  8110. {
  8111. CCovenSDSManager &manager;
  8112. Linked<IException> e;
  8113. bool restart;
  8114. public:
  8115. CHandleException(CCovenSDSManager &_manager, IException *_e, bool _restart) : Thread("sds:CHandleException"), manager(_manager), e(_e), restart(_restart)
  8116. {
  8117. start();
  8118. }
  8119. virtual int run()
  8120. {
  8121. handled=true;
  8122. try
  8123. {
  8124. if (restart)
  8125. manager.restart(e);
  8126. else
  8127. {
  8128. StringBuffer msg;
  8129. msg.append("Unhandled exception, shutting down: ").append(e->errorCode()).append(": ");
  8130. e->errorMessage(msg);
  8131. manager.stop();
  8132. manager.saveStore();
  8133. }
  8134. manager.unhandledThread.clear();
  8135. }
  8136. catch (IException *_e) { LOG(MCoperatorError, unknownJob, _e, "Exception while restarting or shutting down"); _e->Release(); }
  8137. catch (DALI_CATCHALL) { LOG(MCoperatorError, unknownJob, "Unknown exception while restarting or shutting down"); }
  8138. if (!restart)
  8139. {
  8140. e->Link();
  8141. throw e.get();
  8142. }
  8143. processingUnhandled = false;
  8144. handled = false;
  8145. return 0;
  8146. }
  8147. };
  8148. unhandledThread.setown(new CHandleException(*this, e, restartOnError));
  8149. return true;
  8150. }
  8151. void CCovenSDSManager::addNodeSubscriber(ISubscription *sub, SubscriptionId id)
  8152. {
  8153. MemoryBuffer mb;
  8154. mb.setBuffer(sub->queryData().length(), (void *)sub->queryData().get());
  8155. StringAttr xpath;
  8156. bool sendValue;
  8157. mb.read(xpath);
  8158. mb.read(sendValue);
  8159. Owned<IPropertyTreeIterator> iter = root->getElements(xpath+1);
  8160. if (!iter->first())
  8161. throw MakeSDSException(SDSExcpt_SubscriptionNoMatch, "Failed to match any nodes: %s", xpath.get());
  8162. else
  8163. {
  8164. Owned<CNodeSubscriberContainer> subscriber = new CNodeSubscriberContainer(sub, id, sendValue, xpath);
  8165. do
  8166. {
  8167. CServerRemoteTree &node = (CServerRemoteTree &)iter->query();
  8168. node.setSubscribed(true);
  8169. subscriber->add(node);
  8170. }
  8171. while (iter->next());
  8172. nodeSubscriptionManager->associateSubscriber(*subscriber);
  8173. }
  8174. }
  8175. void CCovenSDSManager::removeNodeSubscriber(SubscriptionId id)
  8176. {
  8177. nodeSubscriptionManager->removeSubscriberAssociation(id);
  8178. }
  8179. void CCovenSDSManager::notifyNodeDelete(CServerRemoteTree &node)
  8180. {
  8181. nodeSubscriptionManager->notifyDelete(&node);
  8182. }
  8183. void CCovenSDSManager::notifyNode(CServerRemoteTree &node, PDState state)
  8184. {
  8185. nodeSubscriptionManager->notify(node, state);
  8186. }
  8187. ///////////////////////
  8188. class CDaliSDSServer: implements IDaliServer, public CInterface
  8189. {
  8190. public:
  8191. IMPLEMENT_IINTERFACE;
  8192. CDaliSDSServer(IPropertyTree *_config) : config(_config)
  8193. {
  8194. manager = NULL;
  8195. cancelLoad = false;
  8196. storeLoaded = false;
  8197. }
  8198. ~CDaliSDSServer()
  8199. {
  8200. delete manager;
  8201. }
  8202. void start()
  8203. {
  8204. CriticalBlock b(crit);
  8205. ICoven &coven=queryCoven();
  8206. assertex(coven.inCoven()); // must be member of coven
  8207. if (config)
  8208. sdsConfig.setown(config->getPropTree("SDS"));
  8209. if (!sdsConfig)
  8210. sdsConfig.setown(createPTree());
  8211. manager = new CCovenSDSManager(coven, *sdsConfig, config?config->queryProp("@dataPath"):NULL);
  8212. SDSManager = manager;
  8213. addThreadExceptionHandler(manager);
  8214. try { manager->loadStore(NULL, &cancelLoad); }
  8215. catch (IException *)
  8216. {
  8217. LOG(MCdebugInfo(100), unknownJob, "Failed to load main store");
  8218. throw;
  8219. }
  8220. storeLoaded = true;
  8221. manager->start();
  8222. }
  8223. void ready()
  8224. {
  8225. #ifdef TEST_NOTIFY_HANDLER
  8226. class CTestHan : public CInterface, implements ISDSNotifyHandler
  8227. {
  8228. public:
  8229. IMPLEMENT_IINTERFACE;
  8230. virtual void removed(IPropertyTree &tree)
  8231. {
  8232. PrintLog("Hello, tree(%s) handler(%s), being deleted", tree.queryName(), queryNotifyHandlerName(&tree));
  8233. }
  8234. };
  8235. Owned<ISDSNotifyHandler> myHan = new CTestHan();
  8236. ISDSManagerServer &sdsManager = querySDSServer();
  8237. sdsManager.installNotifyHandler("testHandler", myHan);
  8238. Owned<IRemoteConnection> conn = manager->connect("/", 0, 0, 0);
  8239. IPropertyTree *root = conn->queryRoot();
  8240. IPropertyTree *tree = root->setPropTree("test", createPTree());
  8241. setNotifyHandlerName("testHandler", tree);
  8242. conn->commit();
  8243. root->removeProp("test");
  8244. conn->commit();
  8245. sdsManager.removeNotifyHandler("testHandler");
  8246. #endif
  8247. }
  8248. void suspend()
  8249. {
  8250. }
  8251. void stop()
  8252. {
  8253. cancelLoad = true; // if in progress
  8254. CriticalBlock b(crit);
  8255. if (storeLoaded)
  8256. {
  8257. manager->stop();
  8258. manager->saveStore();
  8259. }
  8260. removeThreadExceptionHandler(manager);
  8261. ::Release(manager);
  8262. manager = NULL;
  8263. }
  8264. void nodeDown(rank_t rank)
  8265. {
  8266. TBD;
  8267. }
  8268. private:
  8269. CCovenSDSManager *manager;
  8270. Linked<IPropertyTree> config;
  8271. Owned<IPropertyTree> sdsConfig;
  8272. bool cancelLoad, storeLoaded;
  8273. CriticalSection crit;
  8274. };
  8275. unsigned SDSLockTimeoutCount = 0;
  8276. unsigned querySDSLockTimeoutCount()
  8277. {
  8278. return SDSLockTimeoutCount;
  8279. }
  8280. ISDSException *MakeSDSException(int errorCode)
  8281. {
  8282. ISDSException *ret = new CSDSException(errorCode);
  8283. return ret;
  8284. }
  8285. ISDSException *MakeSDSException(int errorCode, const char *errorMsg, ...)
  8286. {
  8287. if(errorCode == SDSExcpt_LockTimeout)
  8288. SDSLockTimeoutCount++;
  8289. va_list args;
  8290. va_start(args, errorMsg);
  8291. ISDSException *ret = new CSDSException(errorCode, errorMsg, args);
  8292. va_end(args);
  8293. return ret;
  8294. }
  8295. IDaliServer *createDaliSDSServer(IPropertyTree *config)
  8296. {
  8297. return new CDaliSDSServer(config);
  8298. }
  8299. //////////////////////
  8300. bool applyXmlDeltas(IPropertyTree &root, IIOStream &stream, bool stopOnError)
  8301. {
  8302. class CDeltaProcessor : implements IPTreeNotifyEvent, public CInterface
  8303. {
  8304. unsigned level;
  8305. IPTreeMaker *maker;
  8306. IPropertyTree &store;
  8307. offset_t sectionEndOffset;
  8308. StringAttr headerPath;
  8309. bool stopOnError;
  8310. public:
  8311. IMPLEMENT_IINTERFACE;
  8312. bool hadError;
  8313. CDeltaProcessor(IPropertyTree &_store, bool _stopOnError) : store(_store), stopOnError(_stopOnError), level(0)
  8314. {
  8315. sectionEndOffset = 0;
  8316. hadError = false;
  8317. maker = createRootLessPTreeMaker();
  8318. }
  8319. ~CDeltaProcessor()
  8320. {
  8321. ::Release(maker);
  8322. }
  8323. void apply(IPropertyTree &change, IPropertyTree &currentBranch)
  8324. {
  8325. if (change.getPropBool("@localValue"))
  8326. {
  8327. bool binary = change.isBinary(NULL);
  8328. if (binary)
  8329. {
  8330. MemoryBuffer mb;
  8331. change.getPropBin(NULL, mb);
  8332. currentBranch.setPropBin(NULL, mb.length(), mb.toByteArray());
  8333. }
  8334. else
  8335. currentBranch.setProp(NULL, change.queryProp(NULL));
  8336. }
  8337. else if (change.getPropBool("@appendValue"))
  8338. {
  8339. if (change.queryProp(NULL))
  8340. {
  8341. bool binary=change.isBinary(NULL);
  8342. __int64 index = currentBranch.getPropInt64(EXT_ATTR);
  8343. MemoryBuffer mb;
  8344. if (index && QUERYINTERFACE(&currentBranch, CServerRemoteTree))
  8345. {
  8346. MemoryBuffer mbv;
  8347. SDSManager->getExternalValue(index, mbv);
  8348. CPTValue v(mbv);
  8349. v.getValue(mb, binary);
  8350. }
  8351. else
  8352. currentBranch.getPropBin(NULL, mb);
  8353. change.getPropBin(NULL, mb);
  8354. if (binary)
  8355. currentBranch.setPropBin(NULL, mb.length(), mb.toByteArray());
  8356. else
  8357. currentBranch.setProp(NULL, (const char *)mb.toByteArray());
  8358. }
  8359. }
  8360. Owned<IPropertyTreeIterator> iter = change.getElements(RENAME_TAG);
  8361. ForEach (*iter)
  8362. {
  8363. IPropertyTree &d = iter->query();
  8364. StringBuffer xpath(d.queryProp("@from"));
  8365. xpath.append('[').append(d.queryProp("@pos")).append(']');
  8366. verifyex(currentBranch.renameProp(xpath.str(), d.queryProp("@to")));
  8367. }
  8368. iter.setown(change.getElements(DELETE_TAG));
  8369. ForEach (*iter)
  8370. {
  8371. IPropertyTree &d = iter->query();
  8372. StringBuffer xpath(d.queryProp("@name"));
  8373. xpath.append('[').append(d.queryProp("@pos")).append(']');
  8374. if (!currentBranch.removeProp(xpath.str()))
  8375. LOG(MCoperatorWarning, unknownJob, "Property '%s' missing, but recorded as being present at time of delete, in section '%s'", xpath.str(), headerPath.get());
  8376. }
  8377. IPropertyTree *ac = change.queryPropTree(ATTRCHANGE_TAG);
  8378. if (ac)
  8379. {
  8380. Owned<IAttributeIterator> aIter = ac->getAttributes();
  8381. ForEach (*aIter)
  8382. currentBranch.setProp(aIter->queryName(), aIter->queryValue());
  8383. }
  8384. IPropertyTree *ad = change.queryPropTree(ATTRDELETE_TAG);
  8385. if (ad)
  8386. {
  8387. Owned<IAttributeIterator> aIter = ad->getAttributes();
  8388. ForEach (*aIter)
  8389. {
  8390. if (!currentBranch.removeProp(aIter->queryName()))
  8391. LOG(MCoperatorWarning, unknownJob, "Property '%s' missing, but recorded as being present at time of delete, in section '%s'", aIter->queryName(), headerPath.get());
  8392. }
  8393. }
  8394. processChildren(change, currentBranch);
  8395. }
  8396. void processChildren(IPropertyTree &change, IPropertyTree &currentBranch)
  8397. {
  8398. // process children
  8399. Owned<IPropertyTreeIterator> iter = change.getElements("T");
  8400. ForEach (*iter)
  8401. {
  8402. try
  8403. {
  8404. IPropertyTree &child = iter->query();
  8405. const char *name = child.queryProp("@name");
  8406. if (child.getPropBool("@new"))
  8407. {
  8408. IPropertyTree *newBranch = currentBranch.addPropTree(name, createPTree());
  8409. apply(child, *newBranch);
  8410. }
  8411. else if (child.getPropBool("@replace"))
  8412. {
  8413. IPropertyTree *newBranch = currentBranch.setPropTree(name, createPTree());
  8414. apply(child, *newBranch);
  8415. }
  8416. else
  8417. {
  8418. const char *pos = child.queryProp("@pos");
  8419. if (!pos)
  8420. throw MakeStringException(0, "Missing position attribute in child reference, section end offset=%" I64F "d", sectionEndOffset);
  8421. StringBuffer xpath(name);
  8422. xpath.append('[').append(pos).append(']');
  8423. IPropertyTree *existingBranch = currentBranch.queryPropTree(xpath.str());
  8424. if (!existingBranch)
  8425. throw MakeStringException(0, "Failed to locate delta change in %s, section end offset=%" I64F "d", xpath.str(), sectionEndOffset);
  8426. apply(child, *existingBranch);
  8427. }
  8428. }
  8429. catch (IException *e)
  8430. {
  8431. StringBuffer s("Error processing delta section: sectionEndOffset=");
  8432. LOG(MCoperatorError, unknownJob, e, s.append(sectionEndOffset).str());
  8433. if (stopOnError) throw;
  8434. hadError = true;
  8435. e->Release();
  8436. }
  8437. }
  8438. }
  8439. void process(IPropertyTree &match, offset_t endOffset)
  8440. {
  8441. sectionEndOffset = endOffset;
  8442. const char *xpath = match.queryProp("@path");
  8443. if (xpath && '/' == *xpath)
  8444. xpath++;
  8445. IPropertyTree *root = store.queryPropTree(xpath);
  8446. if (!root)
  8447. throw MakeStringException(0, "Failed to locate header xpath = %s", xpath);
  8448. IPropertyTree *start = match.queryPropTree("Delta/T");
  8449. if (!start)
  8450. throw MakeStringException(0, "Badly constructed delta format (missing Delta/T) in header path=%s, section end offset=%" I64F "d", xpath, endOffset);
  8451. headerPath.set(xpath);
  8452. apply(*start, *root);
  8453. }
  8454. // IPTreeNotifyEvent
  8455. virtual void beginNode(const char *tag, offset_t startOffset) { maker->beginNode(tag, startOffset); }
  8456. virtual void newAttribute(const char *name, const char *value) { maker->newAttribute(name, value); }
  8457. virtual void beginNodeContent(const char *tag) { level++; }
  8458. virtual void endNode(const char *tag, unsigned length, const void *value, bool binary, offset_t endOffset)
  8459. {
  8460. try
  8461. {
  8462. --level;
  8463. IPropertyTree *match = NULL;
  8464. if (0 == level)
  8465. {
  8466. if (0 == strcmp("Header", tag))
  8467. match = maker->queryCurrentNode();
  8468. }
  8469. maker->endNode(tag, length, value, binary, endOffset);
  8470. if (match)
  8471. {
  8472. process(*match, endOffset);
  8473. verifyex(maker->queryRoot()->removeTree(match)); // no longer needed.
  8474. }
  8475. }
  8476. catch (IException *e)
  8477. {
  8478. StringBuffer s("Error processing delta section: sectionEndOffset=");
  8479. LOG(MCoperatorError, unknownJob, e, s.append(endOffset).str());
  8480. if (stopOnError) throw;
  8481. hadError = true;
  8482. e->Release();
  8483. }
  8484. }
  8485. } deltaProcessor(root, stopOnError);
  8486. Owned<IPullPTreeReader> xmlReader = createPullXMLStreamReader(stream, deltaProcessor, (PTreeReaderOptions)((unsigned)ptr_ignoreWhiteSpace+(unsigned)ptr_noRoot), false);
  8487. try
  8488. {
  8489. xmlReader->load();
  8490. }
  8491. catch (IException *e)
  8492. {
  8493. if (stopOnError)
  8494. throw;
  8495. LOG(MCoperatorError, unknownJob, e, "XML parse error on delta load - load truncated");
  8496. e->Release();
  8497. }
  8498. return !deltaProcessor.hadError;
  8499. }
  8500. void LogRemoteConn(IRemoteConnection *conn)
  8501. {
  8502. CConnectionBase *conbase = QUERYINTERFACE(conn,CConnectionBase);
  8503. if (!conbase)
  8504. {
  8505. PROGLOG("Could not get base for %x",(unsigned)(memsize_t)conn);
  8506. return;
  8507. }
  8508. IPropertyTree *root = conn->queryRoot();
  8509. CRemoteTreeBase *remotetree = root?QUERYINTERFACE(root,CRemoteTreeBase):NULL;
  8510. unsigned rcount = remotetree?remotetree->getLinkCount()-1:((unsigned)-1);
  8511. PROGLOG("CONN(%x,%" I64F "x,%" I64F "x) path = '%s' mode = %x, link %d,%d",
  8512. (unsigned)(memsize_t)conn,
  8513. (__int64)conbase->querySessionId(),
  8514. (__int64)conbase->queryConnectionId(),
  8515. conbase->queryXPath(),
  8516. conbase->queryMode(),
  8517. conbase->getLinkCount()-1,
  8518. rcount);
  8519. }
  8520. #ifdef _POOLED_SERVER_REMOTE_TREE
  8521. MODULE_INIT(INIT_PRIORITY_STANDARD)
  8522. {
  8523. CServerRemoteTree_Allocator = new CFixedSizeAllocator(sizeof(CServerRemoteTree));
  8524. return true;
  8525. }
  8526. MODULE_EXIT()
  8527. {
  8528. delete CServerRemoteTree_Allocator;
  8529. }
  8530. #endif