dasds.cpp 329 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383638463856386638763886389639063916392639363946395639663976398639964006401640264036404640564066407640864096410641164126413641464156416641764186419642064216422642364246425642664276428642964306431643264336434643564366437643864396440644164426443644464456446644764486449645064516452645364546455645664576458645964606461646264636464646564666467646864696470647164726473647464756476647764786479648064816482648364846485648664876488648964906491649264936494649564966497649864996500650165026503650465056506650765086509651065116512651365146515651665176518651965206521652265236524652565266527652865296530653165326533653465356536653765386539654065416542654365446545654665476548654965506551655265536554655565566557655865596560656165626563656465656566656765686569657065716572657365746575657665776578657965806581658265836584658565866587658865896590659165926593659465956596659765986599660066016602660366046605660666076608660966106611661266136614661566166617661866196620662166226623662466256626662766286629663066316632663366346635663666376638663966406641664266436644664566466647664866496650665166526653665466556656665766586659666066616662666366646665666666676668666966706671667266736674667566766677667866796680668166826683668466856686668766886689669066916692669366946695669666976698669967006701670267036704670567066707670867096710671167126713671467156716671767186719672067216722672367246725672667276728672967306731673267336734673567366737673867396740674167426743674467456746674767486749675067516752675367546755675667576758675967606761676267636764676567666767676867696770677167726773677467756776677767786779678067816782678367846785678667876788678967906791679267936794679567966797679867996800680168026803680468056806680768086809681068116812681368146815681668176818681968206821682268236824682568266827682868296830683168326833683468356836683768386839684068416842684368446845684668476848684968506851685268536854685568566857685868596860686168626863686468656866686768686869687068716872687368746875687668776878687968806881688268836884688568866887688868896890689168926893689468956896689768986899690069016902690369046905690669076908690969106911691269136914691569166917691869196920692169226923692469256926692769286929693069316932693369346935693669376938693969406941694269436944694569466947694869496950695169526953695469556956695769586959696069616962696369646965696669676968696969706971697269736974697569766977697869796980698169826983698469856986698769886989699069916992699369946995699669976998699970007001700270037004700570067007700870097010701170127013701470157016701770187019702070217022702370247025702670277028702970307031703270337034703570367037703870397040704170427043704470457046704770487049705070517052705370547055705670577058705970607061706270637064706570667067706870697070707170727073707470757076707770787079708070817082708370847085708670877088708970907091709270937094709570967097709870997100710171027103710471057106710771087109711071117112711371147115711671177118711971207121712271237124712571267127712871297130713171327133713471357136713771387139714071417142714371447145714671477148714971507151715271537154715571567157715871597160716171627163716471657166716771687169717071717172717371747175717671777178717971807181718271837184718571867187718871897190719171927193719471957196719771987199720072017202720372047205720672077208720972107211721272137214721572167217721872197220722172227223722472257226722772287229723072317232723372347235723672377238723972407241724272437244724572467247724872497250725172527253725472557256725772587259726072617262726372647265726672677268726972707271727272737274727572767277727872797280728172827283728472857286728772887289729072917292729372947295729672977298729973007301730273037304730573067307730873097310731173127313731473157316731773187319732073217322732373247325732673277328732973307331733273337334733573367337733873397340734173427343734473457346734773487349735073517352735373547355735673577358735973607361736273637364736573667367736873697370737173727373737473757376737773787379738073817382738373847385738673877388738973907391739273937394739573967397739873997400740174027403740474057406740774087409741074117412741374147415741674177418741974207421742274237424742574267427742874297430743174327433743474357436743774387439744074417442744374447445744674477448744974507451745274537454745574567457745874597460746174627463746474657466746774687469747074717472747374747475747674777478747974807481748274837484748574867487748874897490749174927493749474957496749774987499750075017502750375047505750675077508750975107511751275137514751575167517751875197520752175227523752475257526752775287529753075317532753375347535753675377538753975407541754275437544754575467547754875497550755175527553755475557556755775587559756075617562756375647565756675677568756975707571757275737574757575767577757875797580758175827583758475857586758775887589759075917592759375947595759675977598759976007601760276037604760576067607760876097610761176127613761476157616761776187619762076217622762376247625762676277628762976307631763276337634763576367637763876397640764176427643764476457646764776487649765076517652765376547655765676577658765976607661766276637664766576667667766876697670767176727673767476757676767776787679768076817682768376847685768676877688768976907691769276937694769576967697769876997700770177027703770477057706770777087709771077117712771377147715771677177718771977207721772277237724772577267727772877297730773177327733773477357736773777387739774077417742774377447745774677477748774977507751775277537754775577567757775877597760776177627763776477657766776777687769777077717772777377747775777677777778777977807781778277837784778577867787778877897790779177927793779477957796779777987799780078017802780378047805780678077808780978107811781278137814781578167817781878197820782178227823782478257826782778287829783078317832783378347835783678377838783978407841784278437844784578467847784878497850785178527853785478557856785778587859786078617862786378647865786678677868786978707871787278737874787578767877787878797880788178827883788478857886788778887889789078917892789378947895789678977898789979007901790279037904790579067907790879097910791179127913791479157916791779187919792079217922792379247925792679277928792979307931793279337934793579367937793879397940794179427943794479457946794779487949795079517952795379547955795679577958795979607961796279637964796579667967796879697970797179727973797479757976797779787979798079817982798379847985798679877988798979907991799279937994799579967997799879998000800180028003800480058006800780088009801080118012801380148015801680178018801980208021802280238024802580268027802880298030803180328033803480358036803780388039804080418042804380448045804680478048804980508051805280538054805580568057805880598060806180628063806480658066806780688069807080718072807380748075807680778078807980808081808280838084808580868087808880898090809180928093809480958096809780988099810081018102810381048105810681078108810981108111811281138114811581168117811881198120812181228123812481258126812781288129813081318132813381348135813681378138813981408141814281438144814581468147814881498150815181528153815481558156815781588159816081618162816381648165816681678168816981708171817281738174817581768177817881798180818181828183818481858186818781888189819081918192819381948195819681978198819982008201820282038204820582068207820882098210821182128213821482158216821782188219822082218222822382248225822682278228822982308231823282338234823582368237823882398240824182428243824482458246824782488249825082518252825382548255825682578258825982608261826282638264826582668267826882698270827182728273827482758276827782788279828082818282828382848285828682878288828982908291829282938294829582968297829882998300830183028303830483058306830783088309831083118312831383148315831683178318831983208321832283238324832583268327832883298330833183328333833483358336833783388339834083418342834383448345834683478348834983508351835283538354835583568357835883598360836183628363836483658366836783688369837083718372837383748375837683778378837983808381838283838384838583868387838883898390839183928393839483958396839783988399840084018402840384048405840684078408840984108411841284138414841584168417841884198420842184228423842484258426842784288429843084318432843384348435843684378438843984408441844284438444844584468447844884498450845184528453845484558456845784588459846084618462846384648465846684678468846984708471847284738474847584768477847884798480848184828483848484858486848784888489849084918492849384948495849684978498849985008501850285038504850585068507850885098510851185128513851485158516851785188519852085218522852385248525852685278528852985308531853285338534853585368537853885398540854185428543854485458546854785488549855085518552855385548555855685578558855985608561856285638564856585668567856885698570857185728573857485758576857785788579858085818582858385848585858685878588858985908591859285938594859585968597859885998600860186028603860486058606860786088609861086118612861386148615861686178618861986208621862286238624862586268627862886298630863186328633863486358636863786388639864086418642864386448645864686478648864986508651865286538654865586568657865886598660866186628663866486658666866786688669867086718672867386748675867686778678867986808681868286838684868586868687868886898690869186928693869486958696869786988699870087018702870387048705870687078708870987108711871287138714871587168717871887198720872187228723872487258726872787288729873087318732873387348735873687378738873987408741874287438744874587468747874887498750875187528753875487558756875787588759876087618762876387648765876687678768876987708771877287738774877587768777877887798780878187828783878487858786878787888789879087918792879387948795879687978798879988008801880288038804880588068807880888098810881188128813881488158816881788188819882088218822882388248825882688278828882988308831883288338834883588368837883888398840884188428843884488458846884788488849885088518852885388548855885688578858885988608861886288638864886588668867886888698870887188728873887488758876887788788879888088818882888388848885888688878888888988908891889288938894889588968897889888998900890189028903890489058906890789088909891089118912891389148915891689178918891989208921892289238924892589268927892889298930893189328933893489358936893789388939894089418942894389448945894689478948894989508951895289538954895589568957895889598960896189628963896489658966896789688969897089718972897389748975897689778978897989808981898289838984898589868987898889898990899189928993899489958996899789988999900090019002900390049005900690079008900990109011901290139014901590169017901890199020902190229023902490259026902790289029903090319032903390349035903690379038903990409041904290439044904590469047904890499050905190529053905490559056905790589059
  1. /*##############################################################################
  2. HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. ############################################################################## */
  13. #define da_decl DECL_EXPORT
  14. #include "platform.h"
  15. #include "jhash.hpp"
  16. #include "jlib.hpp"
  17. #include "jfile.hpp"
  18. #include "jregexp.hpp"
  19. #include "jthread.hpp"
  20. #include "javahash.hpp"
  21. #include "javahash.tpp"
  22. #include "jmisc.hpp"
  23. #include "jlog.hpp"
  24. #include "mplog.hpp"
  25. #include "jptree.ipp"
  26. #include "jqueue.tpp"
  27. #include "dautils.hpp"
  28. #include "dadfs.hpp"
  29. #define DEBUG_DIR "debug"
  30. #define DEFAULT_KEEP_LASTN_STORES 1
  31. #define MAXDELAYS 5
  32. static const char *deltaHeader = "<CRC>0000000000</CRC><SIZE>0000000000000000</SIZE>"; // fill in later
  33. static unsigned deltaHeaderCrcOff = 5;
  34. static unsigned deltaHeaderSizeStart = 21;
  35. static unsigned deltaHeaderSizeOff = 27;
  36. static unsigned readWriteSlowTracing = 10000; // 10s default
  37. static bool readWriteStackTracing = false;
  38. static unsigned fakeCritTimeout = 60000;
  39. static unsigned readWriteTimeout = 60000;
  40. // #define NODELETE
  41. // #define DISABLE_COALESCE_QUIETTIME
  42. #define LEGACY_CLIENT_RESPONSE
  43. #define ENABLE_INSPOS
  44. #include "mpbuff.hpp"
  45. #include "mpcomm.hpp"
  46. #include "mputil.hpp"
  47. #include "dacoven.hpp"
  48. #include "daserver.hpp"
  49. #include "daclient.hpp"
  50. #include "dacsds.ipp"
  51. #include "dasds.ipp"
  52. #define ALWAYSLAZY_NOTUSED
  53. #define NoMoreChildrenMarker ((__int64)-1)
  54. #define DEFAULT_MAXCLOSEDOWNRETRIES 20 // really do not want to give up.
  55. #define DEFAULT_MAXRETRIES 3
  56. #define DEFAULT_RETRYDELAY (2) // (seconds)
  57. #define DELTANAME "daliinc"
  58. #define DELTADETACHED "dalidet"
  59. #define DELTAINPROGRESS "delta.progress"
  60. #define DETACHINPROGRESS "detach.progress"
  61. #define TMPSAVENAME "dali_store_tmp.xml"
  62. #define INIT_NODETABLE_SIZE 0x380000
  63. #define DEFAULT_LCIDLE_PERIOD (60*10) // time has to be quiet for before blocking/saving (when using @lightweightCoalesce)
  64. #define DEFAULT_LCMIN_TIME (24*60*60) // don't save more than once a 'DEFAULT_LCMIN_TIME' period.
  65. #define DEFAULT_LCIDLE_RATE 1 // 1 write transactions per idle period. <= this rate is deemed idle (suitable for save)
  66. #define STORENOTSAVE_WARNING_PERIOD 72 // hours
  67. static unsigned msgCount=(unsigned)-1;
  68. // #define TEST_NOTIFY_HANDLER
  69. #define TRACE_QWAITING
  70. #define CRC_VALIDATION
  71. #define SUBNTFY_POOL_SIZE 400
  72. #define SUBSCAN_POOL_SIZE 100
  73. #define RTM_INTERNAL 0x80000000 // marker for internal connection (performed within a transaction)
  74. #define DEFAULT_EXTERNAL_SIZE_THRESHOLD (10*1024)
  75. #define NOTIFY_ATTR "@sds:notify"
  76. #define FETCH_ENTIRE -1
  77. #define FETCH_ENTIRE_COND -2
  78. #define TIMEOUT_ON_CLOSEDOWN 120000 // On closedown, give up on trying to join a thread in CSDSTransactionServer after two minutes
  79. #define _POOLED_SERVER_REMOTE_TREE // use a pool for CServerRemoteTree allocations
  80. #ifdef _POOLED_SERVER_REMOTE_TREE
  81. static CFixedSizeAllocator *CServerRemoteTree_Allocator;
  82. #endif
  83. enum notifications { notify_delete=1 };
  84. static const char *notificationStr(notifications n)
  85. {
  86. switch (n)
  87. {
  88. case notify_delete: return "Notify Delete";
  89. default: return "UNKNOWN NOTIFY TYPE";
  90. }
  91. }
  92. const char *queryNotifyHandlerName(IPropertyTree *tree)
  93. {
  94. return tree->queryProp(NOTIFY_ATTR);
  95. }
  96. bool setNotifyHandlerName(const char *handlerName, IPropertyTree *tree)
  97. {
  98. #ifdef _DEBUG
  99. CClientRemoteTree *_tree = QUERYINTERFACE(tree, CClientRemoteTree);
  100. if (!_tree) return false; // has to a SDS tree!
  101. #endif
  102. tree->setProp(NOTIFY_ATTR, handlerName);
  103. return true;
  104. }
  105. StringBuffer &getSdsCmdText(SdsCommand cmd, StringBuffer &ret)
  106. {
  107. switch (cmd)
  108. {
  109. case DAMP_SDSCMD_CONNECT:
  110. return ret.append("DAMP_SDSCMD_CONNECT");
  111. case DAMP_SDSCMD_GET:
  112. return ret.append("DAMP_SDSCMD_GET");
  113. case DAMP_SDSCMD_GETCHILDREN:
  114. return ret.append("DAMP_SDSCMD_GETCHILDREN");
  115. case DAMP_SDSCMD_REVISIONS:
  116. return ret.append("DAMP_SDSCMD_REVISIONS");
  117. case DAMP_SDSCMD_DATA:
  118. return ret.append("DAMP_SDSCMD_DATA");
  119. case DAMP_SDSCMD_DISCONNECT:
  120. return ret.append("DAMP_SDSCMD_DISCONNECT");
  121. case DAMP_SDSCMD_CONNECTSERVER:
  122. return ret.append("DAMP_SDSCMD_CONNECTSERVER");
  123. case DAMP_SDSCMD_DATASERVER:
  124. return ret.append("DAMP_SDSCMD_DATASERVER");
  125. case DAMP_SDSCMD_DISCONNECTSERVER:
  126. return ret.append("DAMP_SDSCMD_DISCONNECTSERVER");
  127. case DAMP_SDSCMD_CHANGEMODE:
  128. return ret.append("DAMP_SDSCMD_CHANGEMODE");
  129. case DAMP_SDSCMD_CHANGEMODESERVER:
  130. return ret.append("DAMP_SDSCMD_CHANGEMODESERVER");
  131. case DAMP_SDSCMD_EDITION:
  132. return ret.append("DAMP_SDSCMD_EDITION");
  133. case DAMP_SDSCMD_GETSTORE:
  134. return ret.append("DAMP_SDSCMD_GETSTORE");
  135. case DAMP_SDSCMD_VERSION:
  136. return ret.append("DAMP_SDSCMD_VERSION");
  137. case DAMP_SDSCMD_DIAGNOSTIC:
  138. return ret.append("DAMP_SDSCMD_DIAGNOSTIC");
  139. case DAMP_SDSCMD_GETELEMENTS:
  140. return ret.append("DAMP_SDSCMD_GETELEMENTS");
  141. case DAMP_SDSCMD_MCONNECT:
  142. return ret.append("DAMP_SDSCMD_MCONNECT");
  143. case DAMP_SDSCMD_GETCHILDREN2:
  144. return ret.append("DAMP_SDSCMD_GETCHILDREN2");
  145. case DAMP_SDSCMD_GET2:
  146. return ret.append("DAMP_SDSCMD_GET2");
  147. case DAMP_SDSCMD_GETPROPS:
  148. return ret.append("DAMP_SDSCMD_GETPROPS");
  149. case DAMP_SDSCMD_GETXPATHS:
  150. return ret.append("DAMP_SDSCMD_GETXPATHS");
  151. case DAMP_SDSCMD_GETEXTVALUE:
  152. return ret.append("DAMP_SDSCMD_GETEXTVALUE");
  153. case DAMP_SDSCMD_GETXPATHSPLUSIDS:
  154. return ret.append("DAMP_SDSCMD_GETXPATHSPLUSIDS");
  155. case DAMP_SDSCMD_GETXPATHSCRITERIA:
  156. return ret.append("DAMP_SDSCMD_GETXPATHSCRITERIA");
  157. case DAMP_SDSCMD_GETELEMENTSRAW:
  158. return ret.append("DAMP_SDSCMD_GETELEMENTSRAW");
  159. case DAMP_SDSCMD_GETCOUNT:
  160. return ret.append("DAMP_SDSCMD_GETCOUNT");
  161. default:
  162. return ret.append("UNKNOWN");
  163. };
  164. return ret;
  165. }
  166. #ifdef USECHECKEDCRITICALSECTIONS
  167. class LinkingCriticalBlock : public CheckedCriticalBlock, public CInterface
  168. {
  169. public:
  170. LinkingCriticalBlock(CheckedCriticalSection &crit, const char *file, unsigned line) : CheckedCriticalBlock(crit, fakeCritTimeout, file, line) { }
  171. };
  172. class CLCLockBlock : public CInterface
  173. {
  174. ReadWriteLock &lock;
  175. bool readLocked; // false == writeLocked
  176. unsigned got, lnum;
  177. public:
  178. CLCLockBlock(ReadWriteLock &_lock, bool readLock, unsigned timeout, const char *fname, unsigned _lnum) : lock(_lock), lnum(_lnum)
  179. {
  180. got = msTick();
  181. loop
  182. {
  183. if (readLock)
  184. {
  185. if (lock.lockRead(timeout))
  186. break;
  187. }
  188. else
  189. {
  190. if (lock.lockWrite(timeout))
  191. break;
  192. }
  193. PROGLOG("CLCLockBlock(write=%d) timeout %s(%d), took %d ms",!readLock,fname,lnum,got-msTick());
  194. PrintStackReport();
  195. }
  196. got = msTick();
  197. readLocked = readLock; // false == writeLocked
  198. };
  199. ~CLCLockBlock()
  200. {
  201. if (readLocked)
  202. lock.unlockRead();
  203. else
  204. lock.unlockWrite();
  205. unsigned e=msTick()-got;
  206. if (e>readWriteSlowTracing)
  207. {
  208. StringBuffer s("TIME: CLCLockBlock(write=");
  209. s.append(!readLocked).append(",lnum=").append(lnum).append(") took ").append(e).append(" ms");
  210. DBGLOG("%s", s.str());
  211. if (readWriteStackTracing)
  212. PrintStackReport();
  213. }
  214. }
  215. };
  216. #else
  217. class LinkingCriticalBlock : public CriticalBlock, public CInterface, implements IInterface
  218. {
  219. public:
  220. IMPLEMENT_IINTERFACE;
  221. LinkingCriticalBlock(CriticalSection &crit, const char *file, unsigned line) : CriticalBlock(crit) { }
  222. };
  223. class CLCLockBlock : public CInterface
  224. {
  225. ReadWriteLock &lock;
  226. bool readLocked; // false == writeLocked
  227. public:
  228. CLCLockBlock(ReadWriteLock &_lock, bool readLock, unsigned timeout, const char *fname, unsigned lnum) : lock(_lock)
  229. {
  230. if (readLock)
  231. lock.lockRead();
  232. else
  233. lock.lockWrite();
  234. readLocked = readLock; // false == writeLocked
  235. };
  236. ~CLCLockBlock()
  237. {
  238. if (readLocked)
  239. lock.unlockRead();
  240. else
  241. lock.unlockWrite();
  242. }
  243. };
  244. #endif
  245. class CLCReadLockBlock : public CLCLockBlock
  246. {
  247. public:
  248. CLCReadLockBlock(ReadWriteLock &lock, unsigned timeout, const char *fname, unsigned lnum) : CLCLockBlock(lock, true, timeout, fname, lnum) { }
  249. };
  250. class CLCWriteLockBlock : public CLCLockBlock
  251. {
  252. public:
  253. CLCWriteLockBlock(ReadWriteLock &lock, unsigned timeout, const char *fname, unsigned lnum) : CLCLockBlock(lock, false, timeout, fname, lnum) { }
  254. };
  255. #ifdef USECHECKEDCRITICALSECTIONS
  256. #define CHECKEDDALIREADLOCKBLOCK(l,timeout) Owned<CLCReadLockBlock> glue(block,__LINE__) = new CLCReadLockBlock(l,timeout,__FILE__,__LINE__)
  257. #define CHECKEDDALIWRITELOCKBLOCK(l,timeout) Owned<CLCWriteLockBlock> glue(block,__LINE__) = new CLCWriteLockBlock(l,timeout,__FILE__,__LINE__)
  258. #else
  259. #define CHECKEDDALIREADLOCKBLOCK(l,timeout) ReadLockBlock glue(block,__LINE__)(l)
  260. #define CHECKEDDALIWRITELOCKBLOCK(l,timeout) WriteLockBlock glue(block,__LINE__)(l)
  261. #endif
  262. #define OVERFLOWSIZE 50000
  263. class CFitArray
  264. {
  265. unsigned size;
  266. CRemoteTreeBase **ptrs; // offset 0 not used
  267. CriticalSection crit;
  268. unsigned nextId, chk;
  269. unsigned freeChainHead;
  270. inline void _ensure(unsigned _size)
  271. {
  272. if (size<_size) {
  273. ptrs = (CRemoteTreeBase **)checked_realloc(ptrs, _size * sizeof(CRemoteTreeBase *), size * sizeof(CRemoteTreeBase *), -100);
  274. memset(&ptrs[size], 0, (_size-size)*sizeof(CRemoteTreeBase *));
  275. size = _size;
  276. }
  277. }
  278. unsigned getId()
  279. {
  280. if (freeChainHead)
  281. {
  282. unsigned nF = freeChainHead;
  283. freeChainHead = (CRemoteTreeBase **)ptrs[nF]-ptrs;
  284. return nF;
  285. }
  286. if (++nextId >= size)
  287. _ensure(nextId+OVERFLOWSIZE);
  288. return nextId;
  289. }
  290. CRemoteTreeBase *_queryElem(__int64 id)
  291. {
  292. unsigned i = (unsigned) id;
  293. if (i>=size)
  294. return NULL;
  295. CRemoteTreeBase *ret = ptrs[i];
  296. if (!ret)
  297. return NULL;
  298. if ((memsize_t)ret >= (memsize_t)&ptrs[0] && (memsize_t)ret < (memsize_t)&ptrs[size])
  299. return NULL; // a ptr within the table is part of the free chain
  300. if (id != ret->queryServerId()) // then obj at index is not the same object.
  301. return NULL;
  302. return ret;
  303. }
  304. public:
  305. CFitArray() : ptrs(NULL), chk(0) { reset(); }
  306. CFitArray(unsigned initSize) : ptrs(NULL), chk(0) { reset(); ensure(initSize); }
  307. ~CFitArray() { free(ptrs); }
  308. void reset()
  309. {
  310. if (ptrs)
  311. free(ptrs);
  312. size = 0;
  313. ptrs = NULL;
  314. nextId = 1;
  315. freeChainHead = 0;
  316. }
  317. void ensure(unsigned size)
  318. {
  319. CriticalBlock b(crit);
  320. _ensure(size);
  321. }
  322. void addElem(CRemoteTreeBase *member)
  323. {
  324. CriticalBlock b(crit);
  325. __int64 id = getId();
  326. ptrs[id] = member;
  327. if (++chk==0x80000000)
  328. chk = 0;
  329. id |= ((__int64)chk << 32);
  330. member->setServerId(id);
  331. }
  332. void freeElem(__int64 id)
  333. {
  334. CriticalBlock b(crit);
  335. unsigned i = (unsigned) id;
  336. assertex(i<size);
  337. ptrs[i] = (CRemoteTreeBase *)&ptrs[freeChainHead];
  338. freeChainHead = i;
  339. }
  340. CRemoteTreeBase *queryElem(__int64 id)
  341. {
  342. CriticalBlock b(crit);
  343. return _queryElem(id);
  344. }
  345. CRemoteTreeBase *getElem(__int64 id)
  346. {
  347. CriticalBlock b(crit);
  348. return LINK(_queryElem(id));
  349. }
  350. unsigned maxElements() const { return nextId; } // actual is nextId - however many in free chain
  351. };
  352. ////////////////
  353. enum IncInfo { IINull=0x00, IncData=0x01, IncDetails=0x02, IncConnect=0x04, IncDisconnect=0x08, IncDisconnectDelete=0x24 };
  354. StringBuffer &constructStoreName(const char *storeBase, unsigned e, StringBuffer &res)
  355. {
  356. res.append(storeBase);
  357. if (e)
  358. res.append(e);
  359. res.append(".xml");
  360. return res;
  361. }
  362. ////////////////
  363. static CheckedCriticalSection loadStoreCrit, saveStoreCrit, saveIncCrit, nfyTableCrit, extCrit, blockedSaveCrit;
  364. class CCovenSDSManager;
  365. static CCovenSDSManager *SDSManager;
  366. static StringAttr remoteBackupLocation;
  367. /////////////////
  368. class TimingStats
  369. {
  370. public:
  371. TimingStats() : count(0), totalTime(0), maxTime(0), minTime((unsigned long)-1), totalSize(0) {}
  372. inline void record(unsigned long interval)
  373. {
  374. count++;
  375. totalTime += interval;
  376. if(interval>maxTime) maxTime = interval;
  377. if(interval<minTime) minTime = interval;
  378. }
  379. inline void recordSize(unsigned long size)
  380. {
  381. totalSize += size;
  382. }
  383. unsigned long queryCount() const { return count; }
  384. unsigned long queryMeanTime() const { return count ? (unsigned long)(((double)totalTime*0.1)/count + 0.5) : 0; }
  385. unsigned long queryMaxTime() const { return maxTime/10; }
  386. unsigned long queryMinTime() const { return count ? minTime/10 : 0; }
  387. unsigned long queryMeanSize() const { return count ? (unsigned long)(((double)totalSize)/count + 0.5) : 0; }
  388. private:
  389. unsigned long count;
  390. unsigned long totalTime;
  391. unsigned long maxTime;
  392. unsigned long minTime;
  393. unsigned long totalSize;
  394. };
  395. class TimingBlock
  396. {
  397. public:
  398. TimingBlock(TimingStats & _stats) : stats(_stats) { start = msTick(); }
  399. ~TimingBlock() { stats.record(msTick()-start); }
  400. protected:
  401. TimingStats & stats;
  402. private:
  403. unsigned long start;
  404. };
  405. class TimingSizeBlock : public TimingBlock
  406. {
  407. public:
  408. TimingSizeBlock(TimingStats & _stats) : TimingBlock(_stats), size(0) {}
  409. ~TimingSizeBlock() { stats.recordSize(size); }
  410. inline void recordSize(unsigned long _size) { size = _size; }
  411. private:
  412. unsigned long size;
  413. };
  414. class CSDSTransactionServer : public Thread, public CTransactionLogTracker
  415. {
  416. public:
  417. CSDSTransactionServer(CCovenSDSManager &_manager);
  418. void stop();
  419. void processMessage(CMessageBuffer &mb);
  420. const bool &queryStopped() const { return stopped; }
  421. inline TimingStats const & queryXactTimingStats() const { return xactTimingStats; }
  422. inline TimingStats const & queryConnectTimingStats() const { return connectTimingStats; }
  423. inline TimingStats const & queryCommitTimingStats() const { return commitTimingStats; }
  424. // Thread
  425. virtual int run();
  426. // CTransactionLogTracker
  427. virtual StringBuffer &getCmdText(unsigned cmd, StringBuffer &ret) const
  428. {
  429. return getSdsCmdText((SdsCommand)cmd, ret);
  430. }
  431. private:
  432. TimingStats xactTimingStats;
  433. TimingStats connectTimingStats;
  434. TimingStats commitTimingStats;
  435. bool stopped;
  436. CCovenSDSManager &manager;
  437. };
  438. //////////////
  439. class CSubscriberContainerBase : public CInterfaceOf<IInterface>
  440. {
  441. DECL_NAMEDCOUNT;
  442. protected:
  443. bool unsubscribed;
  444. Owned<ISubscription> subscriber;
  445. SubscriptionId id;
  446. public:
  447. CSubscriberContainerBase(ISubscription *_subscriber, SubscriptionId _id) :
  448. subscriber(_subscriber), id(_id)
  449. {
  450. INIT_NAMEDCOUNT;
  451. unsubscribed = false;
  452. }
  453. bool notify(MemoryBuffer &mb) const
  454. {
  455. try
  456. {
  457. subscriber->notify(mb);
  458. return true;
  459. }
  460. catch (IException *e)
  461. {
  462. LOG(MCuserWarning, e, "SDS: Error notifying subscriber");
  463. e->Release();
  464. }
  465. return false; // unsubscribe
  466. }
  467. const SubscriptionId &queryId() const { return id; }
  468. const void *queryFindParam() const
  469. {
  470. return (const void *) &id;
  471. }
  472. bool isUnsubscribed() { return unsubscribed || subscriber->aborted(); }
  473. void setUnsubscribed() { unsubscribed = true; }
  474. };
  475. /////////////////
  476. class CConnectionSubscriberContainer : public CSubscriberContainerBase
  477. {
  478. public:
  479. CConnectionSubscriberContainer(ISubscription *subscriber, SubscriptionId id) : CSubscriberContainerBase(subscriber, id) { }
  480. bool notify() { MemoryBuffer mb; return CSubscriberContainerBase::notify(mb); }
  481. };
  482. //////////////
  483. enum ConnInfoFlags { ci_newParent = 0x01 };
  484. //////////////
  485. class CServerConnection : public CConnectionBase, implements ISessionNotify
  486. {
  487. DECL_NAMEDCOUNT;
  488. public:
  489. IMPLEMENT_IINTERFACE;
  490. CServerConnection(ISDSConnectionManager &manager, ConnectionId connectionId, const char *xpath, SessionId id, unsigned mode, unsigned timeout, IPropertyTree *_parent, ConnInfoFlags _connInfoFlags)
  491. : CConnectionBase(manager, connectionId, xpath, id, mode, timeout), parent(_parent), connInfoFlags(_connInfoFlags)
  492. {
  493. INIT_NAMEDCOUNT;
  494. subsid = 0;
  495. established = false;
  496. }
  497. ~CServerConnection();
  498. void initPTreePath(PTree &root, PTree &tail)
  499. {
  500. if (&root != &tail)
  501. {
  502. StringBuffer head;
  503. const char *_tail = splitXPath(xpath, head);
  504. if (_tail == xpath)
  505. ptreePath.append(*LINK(&root));
  506. else
  507. ptreePath.fill(root, head.str(), tail);
  508. #define DEBUG_HPCC_11202
  509. #ifdef DEBUG_HPCC_11202
  510. PTree &parent = ptreePath.tos();
  511. aindex_t pos = parent.queryChildIndex(&tail);
  512. if (pos == NotFound)
  513. {
  514. StringBuffer msg;
  515. msg.append("ConnectionId=").appendf("%" I64F "x", connectionId).append(", xpath=").append(xpath).append(", sessionId=").appendf("%" I64F "x", sessionId).append(", mode=").append(mode).append(", timeout=");
  516. if (INFINITE == timeout)
  517. msg.append("INFINITE");
  518. else
  519. msg.append(timeout);
  520. ERRLOG("Invalid connection: %s", msg.str());
  521. ForEachItemIn(i, ptreePath)
  522. {
  523. PTree &tree = ptreePath.item(i);
  524. DBGLOG("PTree path item %d = %s", i, tree.queryName());
  525. }
  526. assertex(false); // stack report may be useful
  527. }
  528. #endif
  529. }
  530. ptreePath.append(*LINK(&tail));
  531. }
  532. CPTStack &queryPTreePath() { return ptreePath; }
  533. IPropertyTree *queryRootUnvalidated()
  534. {
  535. return root;
  536. }
  537. MemoryBuffer &getInfo(MemoryBuffer &out)
  538. {
  539. out.append(connectionId).append(xpath).append(sessionId).append(mode).append(timeout).append(established);
  540. return out;
  541. }
  542. void subscribe(SessionId id)
  543. {
  544. subsid = querySessionManager().subscribeSession(id, this);
  545. }
  546. void addConnInfoFlag(ConnInfoFlags flag) { connInfoFlags = (ConnInfoFlags) (((unsigned)connInfoFlags) | ((unsigned) flag)); }
  547. void closed(SessionId id)
  548. {
  549. LOG(MCwarning, unknownJob, "Connection (%" I64F "x) was leaked by exiting client (%" I64F "x) path=%s", connectionId, id, queryXPath());
  550. aborted(id);
  551. subsid=0;
  552. }
  553. void aborted(SessionId id);
  554. void unsubscribeSession()
  555. {
  556. if (subsid) {
  557. querySessionManager().unsubscribeSession(subsid);
  558. subsid = 0;
  559. }
  560. }
  561. void notify()
  562. {
  563. ForEachItemInRev(s, subscriptions)
  564. {
  565. CConnectionSubscriberContainer &sub = subscriptions.item(s);
  566. if (!sub.notify())
  567. subscriptions.remove(s);
  568. }
  569. }
  570. void addSubscriber(CConnectionSubscriberContainer &sub)
  571. {
  572. subscriptions.append(sub);
  573. }
  574. void removeSubscriber(SubscriptionId id)
  575. {
  576. ForEachItemIn(s, subscriptions) // do not expect a lot of subscribers per connection - probably ~ 1.
  577. {
  578. if (id == subscriptions.item(s).queryId())
  579. {
  580. subscriptions.remove(s);
  581. break;
  582. }
  583. }
  584. }
  585. bool queryEstablished() { return established; }
  586. void setEstablished() { established = true; }
  587. IPropertyTree *queryParent() { return parent; }
  588. void removeRoot()
  589. {
  590. if (parent)
  591. parent->removeTree(root);
  592. }
  593. virtual IPropertyTree *queryRoot();
  594. private:
  595. ConnInfoFlags connInfoFlags;
  596. IPropertyTree *parent;
  597. SubscriptionId subsid;
  598. CPTStack ptreePath;
  599. IArrayOf<CConnectionSubscriberContainer> subscriptions;
  600. bool established;
  601. };
  602. /////////////////
  603. class CQualifiers : public CInterfaceOf<IInterface>
  604. {
  605. StringArray qualifiers;
  606. public:
  607. inline void add(const char *qualifier) { qualifiers.append(qualifier); }
  608. inline unsigned count() const { return qualifiers.ordinality(); }
  609. inline const char *item(unsigned i) const { return qualifiers.item(i); }
  610. };
  611. class CSubscriberContainer : public CSubscriberContainerBase
  612. {
  613. StringAttr xpath, fullXpath;
  614. IPointerArrayOf<CQualifiers> qualifierStack;
  615. bool sub, sendValue;
  616. unsigned depth;
  617. public:
  618. CSubscriberContainer(ISubscription *subscriber, SubscriptionId id) : CSubscriberContainerBase(subscriber, id)
  619. {
  620. const MemoryAttr &ma = subscriber->queryData();
  621. MemoryBuffer mb(ma.length(), ma.get());
  622. mb.read(xpath);
  623. mb.read(sub);
  624. if (mb.remaining()) // remaining
  625. mb.read(sendValue);
  626. else
  627. sendValue = false;
  628. const char *path = xpath;
  629. const char *nextSep = path+1;
  630. StringBuffer head;
  631. depth = 1; // root
  632. loop
  633. {
  634. nextSep = queryHead(nextSep, head.clear());
  635. ++depth; // inc last
  636. if (!nextSep)
  637. break;
  638. }
  639. StringBuffer strippedXpath;
  640. loop
  641. {
  642. const char *startQ;
  643. if (NULL == (startQ = queryNextUnquoted(path, '['))) // escaped '[]' chars??
  644. {
  645. if (strippedXpath.length()) strippedXpath.append(path);
  646. break;
  647. }
  648. const char *nextSep = path+1;
  649. loop
  650. {
  651. nextSep = queryHead(nextSep, head.clear());
  652. if (!nextSep || startQ < nextSep)
  653. break;
  654. qualifierStack.append(NULL); // no qualifier for this segment.
  655. }
  656. Owned<CQualifiers> qualifiers = new CQualifiers;
  657. strippedXpath.append(startQ-path, path);
  658. loop
  659. {
  660. const char *endQ = queryNextUnquoted(startQ+1, ']');
  661. if (!endQ)
  662. throw MakeSDSException(SDSExcpt_SubscriptionParseError, "Missing closing brace: %s", xpath.get());
  663. StringAttr qualifier(startQ+1, endQ-startQ-1);
  664. qualifiers->add(qualifier);
  665. path = endQ+1;
  666. if ('[' != *path)
  667. break;
  668. startQ = path;
  669. }
  670. qualifierStack.append(qualifiers.getClear());
  671. }
  672. fullXpath.set(xpath);
  673. if (strippedXpath.length()) // some qualifications
  674. xpath.set(strippedXpath.str());
  675. // Notes on qualified (e.g. 'a/b[x="test"]/c' etc.)
  676. // 1) strip out all qualifies from xpath into 'qualifierStack' and translate xpath to simple absolute path.
  677. // 2) collate IPropertyTree stack in processData; to pass to querySubscribers.
  678. // 3) querySubscribers becomes getSubscribers, builds up a list of matching subscribers,
  679. // initially obtained with findList, but pruned based on IPT stack and qualifierStack.
  680. // by walking IPT stack performing either PTree->checkPattern or for index's parent->findChild etc.
  681. }
  682. const char *queryXPath() const { return xpath; }
  683. bool querySub() const { return sub; }
  684. bool querySendValue() const { return sendValue; }
  685. unsigned queryDepth() const { return depth; }
  686. bool qualify(CPTStack &stack, bool matchIfPartial)
  687. {
  688. ForEachItemIn(q, qualifierStack)
  689. {
  690. if (stack.ordinality() <= q+1)
  691. {
  692. // No more stack available (e.g. because deleted below this point)
  693. return matchIfPartial; // NB: return true if matchIfPartial=true (meaning head of subscriber path matched commit stack)
  694. }
  695. PTree &item = stack.item(q+1); // stack +1, top is root unqualified.
  696. CQualifiers *qualifiers = qualifierStack.item(q);
  697. if (qualifiers)
  698. {
  699. for (unsigned q2=0; q2<qualifiers->count(); q2++)
  700. {
  701. const char *qualifier = qualifiers->item(q2);
  702. const char *q = qualifier;
  703. bool numeric = true;
  704. loop
  705. {
  706. if ('\0' == *q) break;
  707. else if (!isdigit(*q)) { numeric = false; break; }
  708. else q++;
  709. }
  710. if (numeric)
  711. {
  712. unsigned qnum = atoi(qualifier);
  713. if (!item.queryParent())
  714. {
  715. if (qnum != 1)
  716. return false;
  717. }
  718. else if (((PTree *)item.queryParent())->findChild(&item) != qnum-1)
  719. return false;
  720. }
  721. else if (!item.checkPattern(qualifier))
  722. return false;
  723. }
  724. }
  725. }
  726. return true;
  727. }
  728. MemoryBuffer &getInfo(MemoryBuffer &out)
  729. {
  730. out.append(id).append(sub).append(fullXpath);
  731. return out;
  732. }
  733. };
  734. typedef IArrayOf<CSubscriberContainer> CSubscriberArray;
  735. typedef ICopyArrayOf<CSubscriberContainer> CSubscriberCopyArray;
  736. class CSubscriberContainerList : public CInterface, public CSubscriberArray
  737. {
  738. public:
  739. CSubscriberContainerList() { }
  740. CSubscriberContainerList(const char *_xpath) : xpath(_xpath) { _xpath = xpath.get(); }
  741. const char *queryXPath() const { return xpath; }
  742. const char *queryFindString() const { return queryXPath(); }
  743. private:
  744. StringAttr xpath;
  745. };
  746. typedef OwningStringSuperHashTableOf<CSubscriberContainerList> CSubscriberXPathTable;
  747. class CSubscriberTable : public ThreadSafeSimpleHashTableOf<CSubscriberContainer, SubscriptionId>
  748. {
  749. public:
  750. ~CSubscriberTable() { _releaseAll(); }
  751. virtual void onAdd(void *et)
  752. {
  753. CSubscriberContainer *subscriber = (CSubscriberContainer *) et;
  754. CSubscriberContainerList *list = xpathTable.find(subscriber->queryXPath());
  755. if (!list)
  756. {
  757. list = new CSubscriberContainerList(subscriber->queryXPath());
  758. xpathTable.replace(*list);
  759. }
  760. list->append(*subscriber); // give over ownership.
  761. }
  762. virtual void onRemove(void *et)
  763. {
  764. CSubscriberContainer *subscriber = (CSubscriberContainer *) et;
  765. subscriber->setUnsubscribed();
  766. CSubscriberContainerList *list = xpathTable.find(subscriber->queryXPath());
  767. assertex(list);
  768. verifyex(list->zap(*subscriber));
  769. if (!list->ordinality())
  770. xpathTable.removeExact(list);
  771. }
  772. CSubscriberContainerList *getQualifiedList(const char *xpath, CPTStack &stack)
  773. {
  774. CriticalBlock b(crit);
  775. CSubscriberContainerList *list = xpathTable.find(xpath);
  776. if (!list) return NULL;
  777. CSubscriberContainerList *results = NULL;
  778. ForEachItemIn(s, *list)
  779. {
  780. CSubscriberContainer &subscriber = list->item(s);
  781. if (subscriber.qualify(stack, false))
  782. {
  783. if (!results) results = new CSubscriberContainerList(xpath);
  784. subscriber.Link();
  785. results->append(subscriber);
  786. }
  787. }
  788. return results;
  789. }
  790. void getSubscribers(CSubscriberArray &subs)
  791. {
  792. CriticalBlock b(crit);
  793. SuperHashIteratorOf<CSubscriberContainer> iter(queryBaseTable());
  794. ForEach(iter)
  795. {
  796. CSubscriberContainer &sub = iter.query();
  797. sub.Link();
  798. subs.append(sub);
  799. }
  800. }
  801. private:
  802. CSubscriberXPathTable xpathTable;
  803. };
  804. #ifdef _DEBUG
  805. struct DebugInfo
  806. {
  807. DebugInfo() { clearExclusive(); }
  808. void clearExclusive() { ExclOwningThread =0; ExclOwningConnection=0; ExclOwningSession=0; }
  809. ThreadId ExclOwningThread;
  810. ConnectionId ExclOwningConnection;
  811. SessionId ExclOwningSession;
  812. };
  813. #endif
  814. class BoolSetBlock
  815. {
  816. public:
  817. BoolSetBlock(bool &_b, bool state=true) : b(_b) { o = b; b = state; }
  818. ~BoolSetBlock() { b = o; }
  819. private:
  820. bool o, &b;
  821. };
  822. //////////////
  823. template <class ET>
  824. class LinkedStringHTMapping : public OwningStringHTMapping<ET>
  825. {
  826. public:
  827. LinkedStringHTMapping(const char *fp, ET &et) : OwningStringHTMapping<ET>(fp, et) { this->et.Link(); }
  828. };
  829. typedef LinkedStringHTMapping<IExternalHandler> CExternalHandlerMapping;
  830. typedef OwningStringSuperHashTableOf<CExternalHandlerMapping> CExternalHandlerTable;
  831. void serializeVisibleAttributes(IPropertyTree &tree, MemoryBuffer &mb)
  832. {
  833. IAttributeIterator *aIter = tree.getAttributes();
  834. if (aIter->first())
  835. {
  836. loop
  837. {
  838. const char *attr = aIter->queryName();
  839. if (0 != strcmp(EXT_ATTR, attr))
  840. {
  841. mb.append(attr);
  842. mb.append(aIter->queryValue());
  843. }
  844. if (!aIter->next())
  845. break;
  846. }
  847. }
  848. aIter->Release();
  849. mb.append(""); // attribute terminator. i.e. blank attr name.
  850. }
  851. void writeDelta(StringBuffer &xml, IFile &iFile, const char *msg="", unsigned retrySecs=0, unsigned retryAttempts=10)
  852. {
  853. Owned<IException> exception;
  854. OwnedIFileIO iFileIO;
  855. unsigned _retryAttempts = retryAttempts;
  856. Owned<IFileIOStream> stream;
  857. offset_t lastGood = 0;
  858. unsigned startCrc = ~0;
  859. MemoryBuffer header;
  860. char strNum[17];
  861. loop
  862. {
  863. header.append(deltaHeader);
  864. try
  865. {
  866. iFileIO.setown(iFile.open(IFOreadwrite));
  867. stream.setown(createIOStream(iFileIO));
  868. if (lastGood)
  869. {
  870. PROGLOG("Resetting delta file size");
  871. iFileIO->setSize(lastGood);
  872. }
  873. else
  874. {
  875. if (iFileIO->size())
  876. {
  877. iFileIO->read(deltaHeaderCrcOff, 10, strNum);
  878. startCrc = ~(unsigned)atoi64_l(strNum, 10);
  879. }
  880. else
  881. stream->write(strlen(deltaHeader), deltaHeader);
  882. lastGood = iFileIO->size();
  883. }
  884. stream->seek(0, IFSend);
  885. stream->write(xml.length(), xml.str());
  886. stream->flush();
  887. stream.clear();
  888. offset_t fLen = lastGood + xml.length();
  889. unsigned crc = crc32(xml.str(), xml.length(), startCrc);
  890. char *headerPtr = (char *)header.bufferBase();
  891. sprintf(strNum, "%010u", ~crc);
  892. memcpy(headerPtr + deltaHeaderCrcOff, strNum, 10);
  893. sprintf(strNum, "%016" I64F "X", fLen);
  894. memcpy(headerPtr + deltaHeaderSizeOff, strNum, 16);
  895. iFileIO->write(0, strlen(deltaHeader), headerPtr);
  896. }
  897. catch (IException *e)
  898. {
  899. exception.setown(e);
  900. StringBuffer s(msg);
  901. LOG(MCoperatorError, unknownJob, e, s.append("writeDelta, failed").str());
  902. }
  903. if (!exception.get())
  904. break;
  905. if (0 == retrySecs)
  906. return;
  907. if (0 == --_retryAttempts)
  908. {
  909. WARNLOG("writeDelta, too many retry attempts [%d]", retryAttempts);
  910. return;
  911. }
  912. exception.clear();
  913. WARNLOG("writeDelta, retrying");
  914. MilliSleep(retrySecs*1000);
  915. }
  916. }
  917. struct BackupQueueItem
  918. {
  919. static unsigned typeMask;
  920. enum flagt { f_delta=0x1, f_addext=0x2, f_delext=0x3, f_first=0x10 };
  921. BackupQueueItem() : edition((unsigned)-1), flags(0) { text = new StringBuffer; dataLength = 0; data = NULL; }
  922. ~BackupQueueItem()
  923. {
  924. delete text;
  925. if (data) free(data);
  926. }
  927. StringBuffer *text;
  928. unsigned edition;
  929. unsigned dataLength;
  930. void *data;
  931. byte flags;
  932. };
  933. unsigned BackupQueueItem::typeMask = 0x0f;
  934. class CBackupHandler : public CInterface, implements IThreaded
  935. {
  936. typedef QueueOf<BackupQueueItem, false> BackupQueue;
  937. CThreaded threaded;
  938. BackupQueue itemQueue, freeQueue;
  939. Semaphore pending, softQueueLimitSem;
  940. bool aborted, waiting, addWaiting, async;
  941. unsigned currentEdition, throttleCounter;
  942. CriticalSection queueCrit, freeQueueCrit;
  943. StringAttr backupPath;
  944. unsigned freeQueueLimit; // how many BackupQueueItems to cache for reuse
  945. unsigned largeWarningThreshold; // point at which to start warning about large queue
  946. unsigned softQueueLimit; // threshold over which primary transactions will be delay by small delay, to allow backup catchup.
  947. unsigned softQueueLimitDelay; // delay for above
  948. CTimeMon warningTime;
  949. unsigned recentTimeThrottled;
  950. unsigned lastNumWarnItems;
  951. IPropertyTree &config;
  952. const unsigned defaultFreeQueueLimit = 50;
  953. const unsigned defaultLargeWarningThreshold = 50;
  954. const unsigned defaultSoftQueueLimit = 200;
  955. const unsigned defaultSoftQueueLimitDelay = 200;
  956. BackupQueueItem *getFreeItem()
  957. {
  958. BackupQueueItem *item;
  959. {
  960. CriticalBlock b(freeQueueCrit);
  961. item = freeQueue.dequeue();
  962. }
  963. if (!item)
  964. item = new BackupQueueItem;
  965. return item;
  966. }
  967. void clearQueue(BackupQueue &queue)
  968. {
  969. loop
  970. {
  971. BackupQueueItem *item = queue.dequeue();
  972. if (!item) break;
  973. delete item;
  974. }
  975. }
  976. void writeExt(const char *name, const unsigned length, const void *data, unsigned retrySecs=0, unsigned retryAttempts=10)
  977. {
  978. Owned<IException> exception;
  979. unsigned _retryAttempts = retryAttempts;
  980. StringBuffer rL(remoteBackupLocation);
  981. loop
  982. {
  983. try
  984. {
  985. rL.append(name);
  986. Owned<IFile> iFile = createIFile(rL.str());
  987. Owned<IFileIO> fileIO = iFile->open(IFOcreate);
  988. fileIO->write(0, length, data);
  989. }
  990. catch (IException *e)
  991. {
  992. exception.setown(e);
  993. StringBuffer err("Saving external (backup): ");
  994. LOG(MCoperatorError, unknownJob, e, err.append(rL).str());
  995. }
  996. if (!exception.get())
  997. break;
  998. if (0 == retrySecs)
  999. return;
  1000. if (0 == --_retryAttempts)
  1001. {
  1002. WARNLOG("writeExt, too many retry attempts [%d]", retryAttempts);
  1003. return;
  1004. }
  1005. exception.clear();
  1006. WARNLOG("writeExt, retrying");
  1007. MilliSleep(retrySecs*1000);
  1008. }
  1009. }
  1010. void deleteExt(const char *name, unsigned retrySecs=0, unsigned retryAttempts=10)
  1011. {
  1012. Owned<IException> exception;
  1013. unsigned _retryAttempts = retryAttempts;
  1014. StringBuffer rL(remoteBackupLocation);
  1015. loop
  1016. {
  1017. try
  1018. {
  1019. rL.append(name);
  1020. Owned<IFile> iFile = createIFile(rL.str());
  1021. iFile->remove();
  1022. }
  1023. catch (IException *e)
  1024. {
  1025. exception.setown(e);
  1026. StringBuffer err("Removing external (backup): ");
  1027. LOG(MCoperatorWarning, unknownJob, e, err.append(rL).str());
  1028. }
  1029. if (!exception.get())
  1030. break;
  1031. if (0 == retrySecs)
  1032. return;
  1033. if (0 == --_retryAttempts)
  1034. {
  1035. WARNLOG("deleteExt, too many retry attempts [%d]", retryAttempts);
  1036. return;
  1037. }
  1038. exception.clear();
  1039. WARNLOG("deleteExt, retrying");
  1040. MilliSleep(retrySecs*1000);
  1041. }
  1042. }
  1043. bool writeDelta(StringBuffer &xml, unsigned edition, bool first)
  1044. {
  1045. StringBuffer deltaFilename(backupPath);
  1046. constructStoreName(DELTANAME, edition, deltaFilename);
  1047. OwnedIFile iFile = createIFile(deltaFilename.str());
  1048. if (!first && !iFile->exists())
  1049. return false; // discard
  1050. ::writeDelta(xml, *iFile, "CBackupHandler - ", 60, 30);
  1051. return true;
  1052. }
  1053. void clearOld()
  1054. {
  1055. CriticalBlock b(queueCrit);
  1056. loop
  1057. {
  1058. BackupQueueItem *item = itemQueue.dequeue();
  1059. if (!item) break;
  1060. if (BackupQueueItem::f_delta == (item->flags & BackupQueueItem::typeMask))
  1061. {
  1062. item->text->clear();
  1063. if (freeQueue.ordinality() < freeQueueLimit)
  1064. freeQueue.enqueue(item);
  1065. else
  1066. delete item;
  1067. }
  1068. }
  1069. if (addWaiting && itemQueue.ordinality()<softQueueLimit)
  1070. softQueueLimitSem.signal();
  1071. }
  1072. public:
  1073. CBackupHandler(IPropertyTree &_config) : config(_config), threaded("CBackupHandler")
  1074. {
  1075. currentEdition = (unsigned)-1;
  1076. addWaiting = waiting = async = false;
  1077. aborted = true;
  1078. throttleCounter = 0;
  1079. recentTimeThrottled = 0;
  1080. lastNumWarnItems = 0;
  1081. freeQueueLimit = config.getPropInt("@backupFreeQueueLimit", defaultFreeQueueLimit);
  1082. largeWarningThreshold = config.getPropInt("@backupLargeWarningThreshold", defaultLargeWarningThreshold);
  1083. softQueueLimit = config.getPropInt("@backupSoftQueueLimit", defaultSoftQueueLimit);
  1084. softQueueLimitDelay = config.getPropInt("@backupSoftQueueLimitDelay", defaultSoftQueueLimitDelay);
  1085. }
  1086. ~CBackupHandler()
  1087. {
  1088. clearQueue(freeQueue);
  1089. clearQueue(itemQueue);
  1090. }
  1091. void init(const char *_backupPath, bool _async)
  1092. {
  1093. backupPath.set(_backupPath);
  1094. async = _async;
  1095. aborted = false;
  1096. PROGLOG("BackupHandler started, async=%s", async?"true":"false");
  1097. threaded.init(this);
  1098. }
  1099. void stop()
  1100. {
  1101. if (!aborted)
  1102. {
  1103. aborted = true;
  1104. pending.signal();
  1105. threaded.join();
  1106. }
  1107. }
  1108. void removeExt(const char *fname)
  1109. {
  1110. if (aborted) return;
  1111. if (!async)
  1112. {
  1113. deleteExt(fname);
  1114. return;
  1115. }
  1116. BackupQueueItem *item = getFreeItem();
  1117. item->text->append(fname);
  1118. item->flags = BackupQueueItem::f_delext;
  1119. add(item);
  1120. }
  1121. void addExt(const char *fname, unsigned length, void *data)
  1122. {
  1123. if (aborted) return;
  1124. if (!async)
  1125. {
  1126. writeExt(fname, length, data);
  1127. return;
  1128. }
  1129. BackupQueueItem *item = getFreeItem();
  1130. item->text->append(fname);
  1131. item->dataLength = length;
  1132. item->data = data; // take ownership
  1133. item->flags = BackupQueueItem::f_addext;
  1134. add(item);
  1135. }
  1136. void addDelta(StringBuffer &xml, unsigned edition, bool first)
  1137. {
  1138. if (aborted) return;
  1139. if (!async)
  1140. {
  1141. writeDelta(xml, edition, first);
  1142. if (xml.length() > 0x100000)
  1143. xml.kill();
  1144. else
  1145. xml.clear();
  1146. return;
  1147. }
  1148. if (edition != currentEdition)
  1149. {
  1150. clearOld();
  1151. currentEdition = edition;
  1152. }
  1153. BackupQueueItem *item = getFreeItem();
  1154. xml.swapWith(*item->text);
  1155. item->edition = edition;
  1156. item->flags = BackupQueueItem::f_delta | BackupQueueItem::f_first;
  1157. add(item);
  1158. }
  1159. void add(BackupQueueItem *item)
  1160. {
  1161. CriticalBlock b(queueCrit);
  1162. itemQueue.enqueue(item);
  1163. unsigned items=itemQueue.ordinality();
  1164. if (0==items%largeWarningThreshold)
  1165. {
  1166. if (items>lastNumWarnItems) // track as they go up
  1167. {
  1168. LOG(MCoperatorWarning, "Backup thread has a high # (%d) of pending transaction queued to write", items);
  1169. lastNumWarnItems = items;
  1170. }
  1171. else if (warningTime.elapsed() >= 60000) // if falling, avoid logging too much
  1172. {
  1173. LOG(MCoperatorWarning, "Backup thread has a high # (%d) of pending transaction queued to write", items);
  1174. lastNumWarnItems = 0;
  1175. warningTime.reset(0);
  1176. }
  1177. }
  1178. if (items>=softQueueLimit)
  1179. {
  1180. addWaiting = true;
  1181. unsigned ms = msTick();
  1182. {
  1183. CriticalUnblock b(queueCrit);
  1184. softQueueLimitSem.wait(softQueueLimitDelay);
  1185. }
  1186. addWaiting = false;
  1187. recentTimeThrottled += (msTick()-ms); // reset when queue < largeWarningThreshold
  1188. if (recentTimeThrottled >= softQueueLimitDelay && (0 == throttleCounter % 10)) // softQueueLimit exceeded - log every 10 transactions if recentTimeThrottled >= softQueueLimitDelay (1 unsignalled delay)
  1189. LOG(MCoperatorWarning, "Primary transactions are being delayed by lagging backup, currently %d queued, recent total throttle delay=%d", items, recentTimeThrottled);
  1190. ++throttleCounter; // also reset when queue < largeWarningThreshold
  1191. }
  1192. if (waiting)
  1193. pending.signal();
  1194. }
  1195. bool doIt(BackupQueueItem &item)
  1196. {
  1197. try
  1198. {
  1199. switch (item.flags & BackupQueueItem::typeMask)
  1200. {
  1201. case BackupQueueItem::f_delta:
  1202. return writeDelta(*item.text, item.edition, 0 != (item.flags & BackupQueueItem::f_first));
  1203. case BackupQueueItem::f_addext:
  1204. writeExt(item.text->str(), item.dataLength, item.data, 60, 30);
  1205. return true;
  1206. case BackupQueueItem::f_delext:
  1207. deleteExt(item.text->str(), 60, 30);
  1208. return true;
  1209. }
  1210. }
  1211. catch (IException *e)
  1212. {
  1213. LOG(MCoperatorWarning, e, "BackupHandler(async) write operation failed, possible backup data loss");
  1214. e->Release();
  1215. }
  1216. return false;
  1217. }
  1218. // IThreaded
  1219. void main()
  1220. {
  1221. loop
  1222. {
  1223. BackupQueueItem *item=NULL;
  1224. do
  1225. {
  1226. CriticalBlock b(queueCrit);
  1227. if (itemQueue.ordinality())
  1228. {
  1229. item = itemQueue.dequeue();
  1230. if (addWaiting && itemQueue.ordinality()<softQueueLimit)
  1231. softQueueLimitSem.signal();
  1232. if (itemQueue.ordinality() < largeWarningThreshold) // reset stats when falls below
  1233. {
  1234. recentTimeThrottled = 0;
  1235. throttleCounter = 0;
  1236. }
  1237. }
  1238. else
  1239. {
  1240. waiting = true;
  1241. {
  1242. CriticalUnblock b(queueCrit);
  1243. if (!aborted)
  1244. pending.wait();
  1245. }
  1246. waiting = false;
  1247. }
  1248. if (aborted)
  1249. {
  1250. if (item) delete item;
  1251. PROGLOG("BackupHandler stopped");
  1252. return;
  1253. }
  1254. }
  1255. while (!item);
  1256. if (!doIt(*item))
  1257. clearOld();
  1258. CriticalBlock b(freeQueueCrit);
  1259. if (freeQueue.ordinality() < freeQueueLimit)
  1260. {
  1261. if (item->text->length() > 0x100000)
  1262. item->text->kill();
  1263. else
  1264. item->text->clear();
  1265. if (item->data)
  1266. {
  1267. free(item->data);
  1268. item->data = NULL;
  1269. item->dataLength = 0;
  1270. }
  1271. freeQueue.enqueue(item);
  1272. }
  1273. else
  1274. delete item;
  1275. }
  1276. }
  1277. };
  1278. class CExternalFile : public CInterface
  1279. {
  1280. StringAttr ext, dataPath;
  1281. protected:
  1282. CBackupHandler &backupHandler;
  1283. public:
  1284. CExternalFile(const char *_ext, const char *_dataPath, CBackupHandler &_backupHandler) : ext(_ext), dataPath(_dataPath), backupHandler(_backupHandler) { }
  1285. const char *queryExt() { return ext; }
  1286. StringBuffer &getName(StringBuffer &fName, const char *base)
  1287. {
  1288. return fName.append(base).append(ext);
  1289. }
  1290. StringBuffer &getFilename(StringBuffer &fName, const char *base)
  1291. {
  1292. return fName.append(dataPath).append(base).append(ext);
  1293. }
  1294. bool isValid(const char *name)
  1295. {
  1296. StringBuffer filename;
  1297. getFilename(filename, name);
  1298. Owned<IFile> iFile = createIFile(filename.str());
  1299. return iFile->exists();
  1300. }
  1301. void remove(const char *name)
  1302. {
  1303. StringBuffer filename;
  1304. getFilename(filename, name);
  1305. Owned<IFile> iFile = createIFile(filename.str());
  1306. iFile->remove();
  1307. if (remoteBackupLocation.length())
  1308. {
  1309. StringBuffer fname(name);
  1310. backupHandler.removeExt(fname.append(queryExt()).str());
  1311. }
  1312. }
  1313. };
  1314. class CLegacyBinaryFileExternal : public CExternalFile, implements IExternalHandler
  1315. {
  1316. public:
  1317. IMPLEMENT_IINTERFACE;
  1318. CLegacyBinaryFileExternal(const char *dataPath, CBackupHandler &backupHandler) : CExternalFile("." EF_LegacyBinaryValue, dataPath, backupHandler) { }
  1319. virtual void resetAsExternal(IPropertyTree &tree)
  1320. {
  1321. tree.setProp(NULL, (char *)NULL);
  1322. }
  1323. virtual void readValue(const char *name, MemoryBuffer &mb)
  1324. {
  1325. StringBuffer filename;
  1326. getFilename(filename, name);
  1327. Owned<IFile> iFile = createIFile(filename.str());
  1328. size32_t sz = (size32_t)iFile->size();
  1329. if ((unsigned)-1 == sz)
  1330. {
  1331. StringBuffer s("Missing external file ");
  1332. Owned<IException> e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str());
  1333. LOG(MCoperatorWarning, unknownJob, e, s.str());
  1334. StringBuffer str("EXTERNAL BINARY FILE: \"");
  1335. str.append(filename.str()).append("\" MISSING");
  1336. CPTValue v(str.length()+1, str.str(), false);
  1337. v.serialize(mb);
  1338. }
  1339. else
  1340. {
  1341. Owned<IFileIO> fileIO = iFile->open(IFOread);
  1342. MemoryBuffer vmb;
  1343. verifyex(sz == ::read(fileIO, 0, sz, vmb));
  1344. CPTValue v(sz, vmb.toByteArray(), true);
  1345. v.serialize(mb);
  1346. }
  1347. }
  1348. virtual void read(const char *name, IPropertyTree &owner, MemoryBuffer &mb, bool withValue)
  1349. {
  1350. StringBuffer filename;
  1351. getFilename(filename, name);
  1352. const char *_name = owner.queryName();
  1353. if (!_name) _name = "";
  1354. mb.append(_name);
  1355. byte flags = ((PTree &)owner).queryFlags();
  1356. mb.append(IptFlagSet(flags, ipt_binary));
  1357. serializeVisibleAttributes(owner, mb);
  1358. Owned<IFile> iFile = createIFile(filename.str());
  1359. size32_t sz = (size32_t)iFile->size();
  1360. if ((unsigned)-1 == sz)
  1361. {
  1362. StringBuffer s("Missing external file ");
  1363. if (*_name)
  1364. s.append("in property ").append(_name);
  1365. Owned<IException> e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str());
  1366. LOG(MCoperatorWarning, unknownJob, e, s.str());
  1367. if (withValue)
  1368. {
  1369. StringBuffer str("EXTERNAL BINARY FILE: \"");
  1370. str.append(filename.str()).append("\" MISSING");
  1371. CPTValue v(str.length()+1, str.str(), false);
  1372. v.serialize(mb);
  1373. }
  1374. else
  1375. mb.append((size32_t)0);
  1376. }
  1377. else
  1378. {
  1379. if (withValue)
  1380. {
  1381. MemoryBuffer vmb;
  1382. Owned<IFileIO> fileIO = iFile->open(IFOread);
  1383. verifyex(sz == ::read(fileIO, 0, sz, vmb));
  1384. CPTValue v(sz, vmb.toByteArray(), true);
  1385. v.serialize(mb);
  1386. }
  1387. else
  1388. mb.append((size32_t)0);
  1389. }
  1390. }
  1391. virtual void write(const char *name, IPropertyTree &tree)
  1392. {
  1393. StringBuffer filename;
  1394. getFilename(filename, name);
  1395. Owned<IFile> iFile = createIFile(filename.str());
  1396. Owned<IFileIO> fileIO = iFile->open(IFOcreate);
  1397. MemoryBuffer out;
  1398. ((PTree &)tree).queryValue()->serialize(out);
  1399. const char *data = out.toByteArray();
  1400. unsigned length = out.length();
  1401. fileIO->write(0, length, data);
  1402. if (remoteBackupLocation.length())
  1403. {
  1404. StringBuffer fname(name);
  1405. backupHandler.addExt(fname.append(queryExt()).str(), length, out.detach());
  1406. }
  1407. }
  1408. virtual void remove(const char *name) { CExternalFile::remove(name); }
  1409. virtual bool isValid(const char *name) { return CExternalFile::isValid(name); }
  1410. virtual StringBuffer &getName(StringBuffer &fName, const char *base) { return CExternalFile::getName(fName, base); }
  1411. virtual StringBuffer &getFilename(StringBuffer &fName, const char *base) { return CExternalFile::getFilename(fName, base); }
  1412. };
  1413. class CBinaryFileExternal : public CExternalFile, implements IExternalHandler
  1414. {
  1415. public:
  1416. IMPLEMENT_IINTERFACE;
  1417. CBinaryFileExternal(const char *dataPath, CBackupHandler &backupHandler) : CExternalFile("." EF_BinaryValue, dataPath, backupHandler) { }
  1418. virtual void resetAsExternal(IPropertyTree &tree)
  1419. {
  1420. tree.setProp(NULL, (char *)NULL);
  1421. }
  1422. virtual void readValue(const char *name, MemoryBuffer &mb)
  1423. {
  1424. StringBuffer filename;
  1425. getFilename(filename, name);
  1426. Owned<IFile> iFile = createIFile(filename.str());
  1427. size32_t sz = (size32_t)iFile->size();
  1428. if ((unsigned)-1 == sz)
  1429. {
  1430. StringBuffer s("Missing external file ");
  1431. Owned<IException> e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str());
  1432. LOG(MCoperatorWarning, unknownJob, e, s.str());
  1433. StringBuffer str("EXTERNAL BINARY FILE: \"");
  1434. str.append(filename.str()).append("\" MISSING");
  1435. CPTValue v(str.length()+1, str.str(), false);
  1436. v.serialize(mb);
  1437. }
  1438. else
  1439. {
  1440. Owned<IFileIO> fileIO = iFile->open(IFOread);
  1441. verifyex(sz == ::read(fileIO, 0, sz, mb));
  1442. }
  1443. }
  1444. virtual void read(const char *name, IPropertyTree &owner, MemoryBuffer &mb, bool withValue)
  1445. {
  1446. StringBuffer filename;
  1447. getFilename(filename, name);
  1448. const char *_name = owner.queryName();
  1449. if (!_name) _name = "";
  1450. mb.append(_name);
  1451. Owned<IFile> iFile = createIFile(filename.str());
  1452. size32_t sz = (size32_t)iFile->size();
  1453. if ((unsigned)-1 == sz)
  1454. {
  1455. byte flags = ((PTree &)owner).queryFlags();
  1456. IptFlagClr(flags, ipt_binary);
  1457. mb.append(flags);
  1458. serializeVisibleAttributes(owner, mb);
  1459. StringBuffer s("Missing external file ");
  1460. if (*_name)
  1461. s.append("in property ").append(_name);
  1462. Owned<IException> e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str());
  1463. LOG(MCoperatorWarning, unknownJob, e, s.str());
  1464. if (withValue)
  1465. {
  1466. StringBuffer str("EXTERNAL BINARY FILE: \"");
  1467. str.append(filename.str()).append("\" MISSING");
  1468. CPTValue v(str.length()+1, str.str(), false);
  1469. v.serialize(mb);
  1470. }
  1471. else
  1472. mb.append((size32_t)0);
  1473. }
  1474. else
  1475. {
  1476. byte flags = ((PTree &)owner).queryFlags();
  1477. mb.append(flags);
  1478. serializeVisibleAttributes(owner, mb);
  1479. if (withValue)
  1480. {
  1481. Owned<IFileIO> fileIO = iFile->open(IFOread);
  1482. verifyex(sz == ::read(fileIO, 0, sz, mb));
  1483. }
  1484. else
  1485. mb.append((size32_t)0);
  1486. }
  1487. }
  1488. virtual void write(const char *name, IPropertyTree &tree)
  1489. {
  1490. StringBuffer filename;
  1491. getFilename(filename, name);
  1492. Owned<IFile> iFile = createIFile(filename.str());
  1493. Owned<IFileIO> fileIO = iFile->open(IFOcreate);
  1494. MemoryBuffer out;
  1495. ((PTree &)tree).queryValue()->serialize(out);
  1496. const char *data = out.toByteArray();
  1497. unsigned length = out.length();
  1498. fileIO->write(0, length, data);
  1499. if (remoteBackupLocation.length())
  1500. {
  1501. StringBuffer fname(name);
  1502. backupHandler.addExt(fname.append(queryExt()).str(), length, out.detach());
  1503. }
  1504. }
  1505. virtual void remove(const char *name) { CExternalFile::remove(name); }
  1506. virtual bool isValid(const char *name) { return CExternalFile::isValid(name); }
  1507. virtual StringBuffer &getName(StringBuffer &fName, const char *base) { return CExternalFile::getName(fName, base); }
  1508. virtual StringBuffer &getFilename(StringBuffer &fName, const char *base) { return CExternalFile::getFilename(fName, base); }
  1509. };
  1510. class CXMLFileExternal : public CExternalFile, implements IExternalHandler
  1511. {
  1512. public:
  1513. IMPLEMENT_IINTERFACE;
  1514. CXMLFileExternal(const char *dataPath, CBackupHandler &backupHandler) : CExternalFile("." EF_XML, dataPath, backupHandler) { }
  1515. virtual void resetAsExternal(IPropertyTree &_tree)
  1516. {
  1517. PTree &tree = *QUERYINTERFACE(&_tree, PTree);
  1518. tree.clear();
  1519. }
  1520. virtual void readValue(const char *name, MemoryBuffer &mb)
  1521. {
  1522. StringBuffer filename;
  1523. getFilename(filename, name);
  1524. OwnedIFile ifile = createIFile(filename.str());
  1525. if (!ifile->exists())
  1526. {
  1527. StringBuffer s("Missing external file ");
  1528. Owned<IException> e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str());
  1529. LOG(MCoperatorWarning, unknownJob, e, s.str());
  1530. StringBuffer str("EXTERNAL XML FILE: \"");
  1531. str.append(filename.str()).append("\" MISSING");
  1532. CPTValue v(str.length()+1, str.str(), false);
  1533. v.serialize(mb);
  1534. }
  1535. else
  1536. {
  1537. Owned<IPropertyTree> tree;
  1538. tree.setown(createPTreeFromXMLFile(filename.str()));
  1539. IPTArrayValue *v = ((PTree *)tree.get())->queryValue();
  1540. if (v)
  1541. v->serialize(mb);
  1542. else
  1543. mb.append((size32_t)0);
  1544. }
  1545. }
  1546. virtual void read(const char *name, IPropertyTree &owner, MemoryBuffer &mb, bool withValue)
  1547. {
  1548. StringBuffer filename;
  1549. getFilename(filename, name);
  1550. Owned<IPropertyTree> tree;
  1551. OwnedIFile ifile = createIFile(filename.str());
  1552. if (!ifile->exists())
  1553. {
  1554. StringBuffer s("Missing external file ");
  1555. const char *name = owner.queryName();
  1556. if (name && *name)
  1557. s.append("in property ").append(name);
  1558. Owned<IException> e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str());
  1559. LOG(MCoperatorWarning, unknownJob, e, s.str());
  1560. StringBuffer str("EXTERNAL XML FILE: \"");
  1561. str.append(filename.str()).append("\" MISSING");
  1562. tree.setown(createPTree(owner.queryName()));
  1563. if (withValue)
  1564. tree->setProp(NULL, str.str());
  1565. }
  1566. else
  1567. {
  1568. tree.setown(createPTreeFromXMLFile(filename.str()));
  1569. if (!withValue)
  1570. tree->setProp(NULL, (char *)NULL);
  1571. }
  1572. ((PTree *)tree.get())->serializeSelf(mb);
  1573. }
  1574. virtual void write(const char *name, IPropertyTree &tree)
  1575. {
  1576. StringBuffer filename;
  1577. getFilename(filename, name);
  1578. Owned<IFile> iFile = createIFile(filename.str());
  1579. Owned<IFileIO> fileIO = iFile->open(IFOcreate);
  1580. Owned<IFileIOStream> fstream = createBufferedIOStream(fileIO);
  1581. toXML(&tree, *fstream);
  1582. if (remoteBackupLocation.length())
  1583. {
  1584. StringBuffer fname(name);
  1585. StringBuffer str;
  1586. toXML(&tree, str);
  1587. unsigned l = str.length();
  1588. backupHandler.addExt(fname.append(queryExt()).str(), l, str.detach());
  1589. }
  1590. }
  1591. virtual void remove(const char *name) { CExternalFile::remove(name); }
  1592. virtual bool isValid(const char *name) { return CExternalFile::isValid(name); }
  1593. virtual StringBuffer &getName(StringBuffer &fName, const char *base) { return CExternalFile::getName(fName, base); }
  1594. virtual StringBuffer &getFilename(StringBuffer &fName, const char *base) { return CExternalFile::getFilename(fName, base); }
  1595. };
  1596. //////////////
  1597. class CBranchChange : public CInterface
  1598. {
  1599. DECL_NAMEDCOUNT;
  1600. typedef CIArrayOf<CBranchChange> CBranchChangeChildren;
  1601. public:
  1602. CBranchChange(CRemoteTreeBase &_tree) : tree(&_tree), local(PDS_None), state(PDS_None) { INIT_NAMEDCOUNT; }
  1603. void noteChange(PDState _local, PDState _state) { local = _local; state = _state; }
  1604. void addChildBranch(CBranchChange &child) { children.append(child); }
  1605. const void *queryFindParam() const { return (const void *) &tree; }
  1606. CBranchChangeChildren children;
  1607. Linked<CRemoteTreeBase> tree;
  1608. PDState local, state; // change info
  1609. };
  1610. SDSNotifyFlags translatePDState(PDState state)
  1611. {
  1612. return (SDSNotifyFlags) state; // mirrored for now.
  1613. }
  1614. void buildNotifyData(MemoryBuffer &notifyData, PDState state, CPTStack *stack, MemoryBuffer *data)
  1615. {
  1616. if (stack)
  1617. {
  1618. notifyData.append('/');
  1619. PTree *parent = &stack->item(0); // root
  1620. unsigned n = stack->ordinality();
  1621. if (n>1)
  1622. {
  1623. unsigned s = 1;
  1624. loop
  1625. {
  1626. PTree &child = stack->item(s);
  1627. const char *str = child.queryName();
  1628. notifyData.append(strlen(str), str);
  1629. if (child.queryParent())
  1630. {
  1631. char temp[12];
  1632. unsigned written = numtostr(temp, parent->findChild(&child)+1);
  1633. notifyData.append('[').append(written, temp).append(']');
  1634. }
  1635. else
  1636. notifyData.append(3, "[1]");
  1637. parent = &child;
  1638. s++;
  1639. if (s<n)
  1640. notifyData.append('/');
  1641. else
  1642. break;
  1643. }
  1644. }
  1645. notifyData.append('\0');
  1646. }
  1647. notifyData.append((int)translatePDState(state));
  1648. if (data)
  1649. {
  1650. notifyData.append(true);
  1651. notifyData.append(data->length());
  1652. notifyData.append(*data);
  1653. }
  1654. else
  1655. notifyData.append(false);
  1656. }
  1657. class CSubscriberNotifier;
  1658. typedef SimpleHashTableOf<CSubscriberNotifier, SubscriptionId> CSubscriberNotifierTable;
  1659. class CSubscriberNotifier : public CInterface
  1660. {
  1661. DECL_NAMEDCOUNT;
  1662. class CChange : public CInterface
  1663. {
  1664. public:
  1665. CChange(MemoryBuffer &_notifyData) : notifyData(_notifyData.length(), _notifyData.toByteArray()) { }
  1666. MemoryBuffer notifyData;
  1667. };
  1668. public:
  1669. CSubscriberNotifier(CSubscriberNotifierTable &_table, CSubscriberContainerBase &_subscriber, MemoryBuffer &notifyData)
  1670. : table(_table), subscriber(_subscriber) //NB: takes ownership of subscriber
  1671. {
  1672. INIT_NAMEDCOUNT;
  1673. change.setown(new CChange(notifyData));
  1674. }
  1675. ~CSubscriberNotifier() { subscriber.Release(); }
  1676. void queueChange(MemoryBuffer &notifyData)
  1677. {
  1678. changeQueue.append(*new CChange(notifyData));
  1679. }
  1680. void notify()
  1681. {
  1682. loop
  1683. {
  1684. if (!subscriber.notify(change->notifyData))
  1685. {
  1686. subscriber.setUnsubscribed();
  1687. break;
  1688. }
  1689. else if (subscriber.isUnsubscribed())
  1690. break;
  1691. CHECKEDCRITICALBLOCK(nfyTableCrit, fakeCritTimeout);
  1692. if (changeQueue.ordinality())
  1693. {
  1694. change.set(&changeQueue.item(0));
  1695. changeQueue.remove(0);
  1696. }
  1697. else
  1698. {
  1699. table.removeExact(this);
  1700. break;
  1701. }
  1702. }
  1703. if (subscriber.isUnsubscribed())
  1704. {
  1705. { CHECKEDCRITICALBLOCK(nfyTableCrit, fakeCritTimeout);
  1706. table.removeExact(this);
  1707. }
  1708. querySubscriptionManager(SDS_PUBLISHER)->remove(subscriber.queryId());
  1709. }
  1710. }
  1711. const void *queryFindParam() const
  1712. {
  1713. return &(subscriber.queryId());
  1714. }
  1715. private:
  1716. Linked<CChange> change;
  1717. CIArrayOf<CChange> changeQueue;
  1718. CSubscriberContainerBase &subscriber;
  1719. MemoryAttr notifyData;
  1720. CSubscriberNotifierTable &table;
  1721. };
  1722. ////////////////
  1723. typedef MapBetween<SubscriptionId, SubscriptionId, ConnectionId, ConnectionId> SubConnMap;
  1724. class CConnectionSubscriptionManager : implements ISubscriptionManager, CInterface
  1725. {
  1726. public:
  1727. IMPLEMENT_IINTERFACE;
  1728. CConnectionSubscriptionManager()
  1729. {
  1730. }
  1731. unsigned querySubscribers()
  1732. {
  1733. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  1734. return subConnMap.count();
  1735. }
  1736. virtual void add(ISubscription *sub, SubscriptionId id);
  1737. virtual void remove(SubscriptionId id);
  1738. private:
  1739. SubConnMap subConnMap;
  1740. CheckedCriticalSection crit;
  1741. };
  1742. //////////
  1743. class CLock;
  1744. typedef ThreadSafeOwningSimpleHashTableOf<CLock, __int64> CLockTable;
  1745. //////////
  1746. interface IUnlockCallback
  1747. {
  1748. virtual void block() = 0;
  1749. virtual void unblock() = 0;
  1750. };
  1751. //////////
  1752. typedef LinkedStringHTMapping<ISDSNotifyHandler> CSDSNotifyHandlerMapping;
  1753. typedef OwningStringSuperHashTableOf<CSDSNotifyHandlerMapping> CNotifyHandlerTable;
  1754. class CServerRemoteTree;
  1755. typedef CServerRemoteTree *CServerRemoteTreePtr;
  1756. typedef MapBetween<__int64, __int64, CServerRemoteTreePtr, CServerRemoteTreePtr> UniqueTreeMapping;
  1757. struct CStoreInfo
  1758. {
  1759. unsigned edition;
  1760. unsigned crc;
  1761. StringAttr cache;
  1762. };
  1763. interface ICoalesce : extends IInterface
  1764. {
  1765. virtual void start() = 0;
  1766. virtual void stop() = 0;
  1767. };
  1768. //////////////////////
  1769. class CNodeSubscriberContainer;
  1770. interface INodeSubscriptionManager : extends ISubscriptionManager
  1771. {
  1772. virtual void associateSubscriber(CNodeSubscriberContainer &subscriber) = 0;
  1773. virtual void removeSubscriberAssociation(SubscriptionId id) = 0;
  1774. virtual void notifyDelete(CServerRemoteTree *node) = 0;
  1775. virtual void notify(CServerRemoteTree &node, PDState state) = 0;
  1776. virtual MemoryBuffer &collectSubscribers(MemoryBuffer &out) const = 0;
  1777. };
  1778. //////////////////////
  1779. enum LockStatus { LockFailed, LockHeld, LockTimedOut, LockSucceeded };
  1780. class CCovenSDSManager : public CSDSManagerBase, implements ISDSManagerServer, implements ISubscriptionManager, implements IExceptionHandler
  1781. {
  1782. public:
  1783. IMPLEMENT_IINTERFACE;
  1784. CCovenSDSManager(ICoven &_coven, IPropertyTree &config, const char *dataPath);
  1785. ~CCovenSDSManager();
  1786. void start();
  1787. void stop();
  1788. void restart(IException * e);
  1789. void loadStore(const char *store=NULL, const bool *abort=NULL);
  1790. void saveStore(const char *store=NULL, bool currentEdition=false);
  1791. bool unlock(__int64 treeId, ConnectionId connectionId, bool delayDelete=false);
  1792. void unlockAll(__int64 treeId);
  1793. void changeLockMode(CServerConnection &connection, unsigned newMode, unsigned timeout);
  1794. void clearSDSLocks();
  1795. void lock(CServerRemoteTree &tree, const char *xpath, ConnectionId connectionId, SessionId sessionId, unsigned mode, unsigned timeout, IUnlockCallback &callback);
  1796. CLock *queryLock(__int64 id) { return lockTable.find(&id); }
  1797. CSubscriberTable &querySubscriberTable() { return subscribers; }
  1798. IExternalHandler *queryExternalHandler(const char *handler) { if (!handler) return NULL; CExternalHandlerMapping *mapping = externalHandlers.find(handler); return mapping ? &mapping->query() : NULL; }
  1799. void handleNotify(CSubscriberContainerBase *subscriber, MemoryBuffer &notifyData);
  1800. void startNotification(IPropertyTree &changeTree, CPTStack &stack, CBranchChange &changes); // subscription notification
  1801. MemoryBuffer &collectUsageStats(MemoryBuffer &out);
  1802. MemoryBuffer &collectConnections(MemoryBuffer &out);
  1803. MemoryBuffer &collectSubscribers(MemoryBuffer &out);
  1804. void blockingSave(unsigned *writeTransactions=NULL);
  1805. bool queryStopped() { return server.queryStopped(); }
  1806. void handleNodeNotify(notifications n, CServerRemoteTree &tree); // server node notification
  1807. void deleteExternal(__int64 index);
  1808. void serializeExternal(__int64 index, IPropertyTree &owner, MemoryBuffer &mb, bool withValue);
  1809. void writeExternal(CServerRemoteTree &tree, bool direct=false, __int64 existing=0);
  1810. inline unsigned queryExternalThreshold() { return externalSizeThreshold; }
  1811. CServerConnection *queryConnection(ConnectionId id);
  1812. CServerConnection *getConnection(ConnectionId id);
  1813. inline CFitArray &queryAllNodes() { return allNodes; }
  1814. unsigned __int64 getNextExternal() { return nextExternal++; }
  1815. CServerConnection *createConnectionInstance(CRemoteTreeBase *root, SessionId sessionId, unsigned mode, unsigned timeout, const char *xpath, CRemoteTreeBase *&tree, ConnectionId connectionId, StringAttr *deltaPath, Owned<IPropertyTree> &deltaChange, Owned<CBranchChange> &branchChange, unsigned &additions);
  1816. void createConnection(SessionId sessionId, unsigned mode, unsigned timeout, const char *xpath, CServerRemoteTree *&tree, ConnectionId &connectionId, bool primary, Owned<LinkingCriticalBlock> &connectCritBlock);
  1817. void disconnect(ConnectionId connectionId, bool deleteRoot=false, Owned<CLCLockBlock> *lockBlock=NULL);
  1818. void registerTree(__int64 serverId, CServerRemoteTree &tree);
  1819. void unregisterTree(__int64 uniqId);
  1820. CServerRemoteTree *queryRegisteredTree(__int64 uniqId);
  1821. CServerRemoteTree *getRegisteredTree(__int64 uniqId);
  1822. CServerRemoteTree *queryRoot();
  1823. void saveDelta(const char *path, IPropertyTree &changeTree);
  1824. CSubscriberContainerList *getSubscribers(const char *xpath, CPTStack &stack);
  1825. void getExternalValue(__int64 index, MemoryBuffer &mb);
  1826. IPropertyTree *getXPathsSortLimitMatchTree(const char *baseXPath, const char *matchXPath, const char *sortby, bool caseinsensitive, bool ascending, unsigned from, unsigned limit);
  1827. void addNodeSubscriber(ISubscription *sub, SubscriptionId id);
  1828. void removeNodeSubscriber(SubscriptionId id);
  1829. void notifyNodeDelete(CServerRemoteTree &node);
  1830. void notifyNode(CServerRemoteTree &node, PDState state);
  1831. // ISDSConnectionManager
  1832. virtual CRemoteTreeBase *get(CRemoteConnection &connection, __int64 serverId);
  1833. virtual void getChildren(CRemoteTreeBase &parent, CRemoteConnection &connection, unsigned levels);
  1834. virtual void getChildrenFor(CRTArray &childLessList, CRemoteConnection &connection, unsigned levels);
  1835. virtual void ensureLocal(CRemoteConnection &connection, CRemoteTreeBase &_parent, IPropertyTree *serverMatchTree, IPTIteratorCodes flags=iptiter_null);
  1836. virtual IPropertyTreeIterator *getElements(CRemoteConnection &connection, const char *xpath);
  1837. virtual void commit(CRemoteConnection &connection, bool *disconnectDeleteRoot);
  1838. virtual void changeMode(CRemoteConnection &connection, unsigned mode, unsigned timeout, bool suppressReload);
  1839. virtual IPropertyTree *getXPaths(__int64 serverId, const char *xpath, bool getServerIds=false);
  1840. virtual IPropertyTreeIterator *getXPathsSortLimit(const char *baseXPath, const char *matchXPath, const char *sortby, bool caseinsensitive, bool ascending, unsigned from, unsigned limit);
  1841. virtual void getExternalValueFromServerId(__int64 serverId, MemoryBuffer &mb);
  1842. virtual bool unlock(__int64 connectionId, bool closeConn, StringBuffer &connectionInfo);
  1843. // ISDSManagerServer
  1844. virtual IRemoteConnections *connect(IMultipleConnector *mConnect, SessionId id, unsigned timeout);
  1845. virtual IRemoteConnection *connect(const char *xpath, SessionId id, unsigned mode, unsigned timeout);
  1846. virtual SubscriptionId subscribe(const char *xpath, ISDSSubscription &notify, bool sub=true, bool sendValue=false);
  1847. virtual SubscriptionId subscribeExact(const char *xpath, ISDSNodeSubscription &notify, bool sendValue=false);
  1848. virtual void unsubscribe(SubscriptionId id);
  1849. virtual void unsubscribeExact(SubscriptionId id);
  1850. virtual ILockInfoCollection *getLocks(const char *ipPattern, const char *xpathPattern);
  1851. virtual StringBuffer &getUsageStats(StringBuffer &out);
  1852. virtual StringBuffer &getConnections(StringBuffer &out);
  1853. virtual StringBuffer &getSubscribers(StringBuffer &out);
  1854. virtual StringBuffer &getExternalReport(StringBuffer &out);
  1855. virtual void installNotifyHandler(const char *handlerKey, ISDSNotifyHandler *handler);
  1856. virtual bool removeNotifyHandler(const char *handlerKey);
  1857. virtual IPropertyTree *lockStoreRead() const;
  1858. virtual void unlockStoreRead() const;
  1859. virtual unsigned countConnections();
  1860. virtual bool setSDSDebug(StringArray &params, StringBuffer &reply);
  1861. virtual unsigned countActiveLocks();
  1862. virtual unsigned queryExternalSizeThreshold() const { return externalSizeThreshold; }
  1863. virtual void setExternalSizeThreshold(unsigned _size) { externalSizeThreshold = _size; }
  1864. virtual bool queryRestartOnError() const { return restartOnError; }
  1865. virtual void setRestartOnError(bool _restart) { restartOnError = _restart; }
  1866. unsigned queryRequestsPending() const { return coven.probe(RANK_ALL,MPTAG_DALI_SDS_REQUEST,NULL); }
  1867. unsigned queryXactCount() const { return server.queryXactTimingStats().queryCount(); }
  1868. unsigned queryXactMeanTime() const { return server.queryXactTimingStats().queryMeanTime(); }
  1869. unsigned queryXactMaxTime() const { return server.queryXactTimingStats().queryMaxTime(); }
  1870. unsigned queryXactMinTime() const { return server.queryXactTimingStats().queryMinTime(); }
  1871. unsigned queryConnectMeanTime() const { return server.queryConnectTimingStats().queryMeanTime(); }
  1872. unsigned queryConnectMaxTime() const { return server.queryConnectTimingStats().queryMaxTime(); }
  1873. unsigned queryCommitMeanTime() const { return server.queryCommitTimingStats().queryMeanTime(); }
  1874. unsigned queryCommitMaxTime() const { return server.queryCommitTimingStats().queryMaxTime(); }
  1875. unsigned queryCommitMeanSize() const { return server.queryCommitTimingStats().queryMeanSize(); }
  1876. virtual void saveRequest();
  1877. virtual IPropertyTree &queryProperties() const;
  1878. virtual IPropertyTreeIterator *getElementsRaw(const char *xpath,INode *remotedali=NULL, unsigned timeout=MP_WAIT_FOREVER);
  1879. virtual void setConfigOpt(const char *opt, const char *value);
  1880. virtual unsigned queryCount(const char *xpath);
  1881. virtual bool updateEnvironment(IPropertyTree *newEnv, bool forceGroupUpdate, StringBuffer &response);
  1882. // ISubscriptionManager impl.
  1883. virtual void add(ISubscription *subs,SubscriptionId id);
  1884. virtual void remove(SubscriptionId id);
  1885. // IExceptionHandler
  1886. virtual bool fireException(IException *e);
  1887. public: // data
  1888. mutable ReadWriteLock dataRWLock;
  1889. CheckedCriticalSection connectCrit;
  1890. CheckedCriticalSection connDestructCrit;
  1891. CheckedCriticalSection cTableCrit;
  1892. CheckedCriticalSection sTableCrit;
  1893. CheckedCriticalSection lockCrit;
  1894. CheckedCriticalSection treeRegCrit;
  1895. Owned<Thread> unhandledThread;
  1896. unsigned writeTransactions;
  1897. bool ignoreExternals;
  1898. StringAttr dataPath;
  1899. Owned<IPropertyTree> properties;
  1900. private:
  1901. void validateBackup();
  1902. LockStatus establishLock(CLock &lock, __int64 treeId, ConnectionId connectionId, SessionId sessionId, unsigned mode, unsigned timeout, IUnlockCallback &lockCallback);
  1903. void _getChildren(CRemoteTreeBase &parent, CServerRemoteTree &serverParent, CRemoteConnection &connection, unsigned levels);
  1904. void matchServerTree(CClientRemoteTree *local, IPropertyTree &matchTree, bool allTail);
  1905. CSubscriberTable subscribers;
  1906. CSDSTransactionServer server;
  1907. ICoven &coven;
  1908. CServerRemoteTree *root;
  1909. CFitArray allNodes;
  1910. IPropertyTree &config;
  1911. MemoryBuffer incrementBuffer;
  1912. Owned<ICoalesce> coalesce;
  1913. unsigned __int64 nextExternal;
  1914. unsigned externalSizeThreshold;
  1915. CLockTable lockTable;
  1916. CNotifyHandlerTable nodeNotifyHandlers;
  1917. Owned<IThreadPool> scanNotifyPool, notifyPool;
  1918. CExternalHandlerTable externalHandlers;
  1919. CSubscriberNotifierTable subscriberNotificationTable;
  1920. Owned<CConnectionSubscriptionManager> connectionSubscriptionManager;
  1921. Owned<INodeSubscriptionManager> nodeSubscriptionManager;
  1922. bool restartOnError, externalEnvironment;
  1923. IStoreHelper *iStoreHelper;
  1924. bool doTimeComparison;
  1925. StringBuffer blockedDelta;
  1926. CBackupHandler backupHandler;
  1927. };
  1928. ISDSManagerServer &querySDSServer()
  1929. {
  1930. assertex(queryCoven().inCoven());
  1931. return *SDSManager;
  1932. }
  1933. /////////////////
  1934. void CConnectionSubscriptionManager::add(ISubscription *sub, SubscriptionId id)
  1935. {
  1936. MemoryBuffer mb;
  1937. mb.setBuffer(sub->queryData().length(), (void *)sub->queryData().get());
  1938. ConnectionId connId;
  1939. mb.read(connId);
  1940. Owned<CServerConnection> conn = SDSManager->getConnection(connId);
  1941. if (conn)
  1942. {
  1943. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  1944. conn->addSubscriber(* new CConnectionSubscriberContainer(sub, id));
  1945. subConnMap.setValue(id, connId);
  1946. }
  1947. // else assume connection has since been disconnected.
  1948. }
  1949. void CConnectionSubscriptionManager::remove(SubscriptionId id)
  1950. {
  1951. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  1952. ConnectionId *connId = subConnMap.getValue(id);
  1953. assertex(connId);
  1954. Owned<CServerConnection> conn = SDSManager->getConnection(*connId);
  1955. if (conn)
  1956. conn->removeSubscriber(id);
  1957. subConnMap.remove(id);
  1958. }
  1959. /////////////////
  1960. CDisableFetchChangeBlock::CDisableFetchChangeBlock(CRemoteConnection &_connection) : connection(_connection)
  1961. {
  1962. stateChanges = connection.queryStateChanges();
  1963. connection.setStateChanges(false);
  1964. lazyFetch = connection.setLazyFetch(false);
  1965. }
  1966. CDisableFetchChangeBlock::~CDisableFetchChangeBlock()
  1967. {
  1968. connection.setLazyFetch(lazyFetch);
  1969. connection.setStateChanges(stateChanges);
  1970. }
  1971. CDisableLazyFetchBlock::CDisableLazyFetchBlock(CRemoteConnection &_connection) : connection(_connection)
  1972. {
  1973. lazyFetch = connection.setLazyFetch(false);
  1974. }
  1975. CDisableLazyFetchBlock::~CDisableLazyFetchBlock()
  1976. {
  1977. connection.setLazyFetch(lazyFetch);
  1978. }
  1979. /////////////////
  1980. /// CPTStack impl.
  1981. bool CPTStack::_fill(IPropertyTree &current, const char *xpath, IPropertyTree &tail)
  1982. {
  1983. const char *nextSep = '/' == *xpath ? xpath+1 : xpath;
  1984. StringBuffer head;
  1985. const char *_nextSep = queryHead(nextSep, head);
  1986. if (!_nextSep)
  1987. {
  1988. Owned<IPropertyTreeIterator> iter = current.getElements(nextSep);
  1989. ForEach (*iter)
  1990. {
  1991. if (NotFound != iter->query().queryChildIndex(&tail))
  1992. {
  1993. append(*LINK((PTree *)&iter->query()));
  1994. append(*LINK((PTree *)&current));
  1995. return true;
  1996. }
  1997. }
  1998. }
  1999. else
  2000. {
  2001. Owned<IPropertyTreeIterator> iter = current.getElements(head.str());
  2002. ForEach (*iter)
  2003. {
  2004. if (&tail==&iter->query()) // Afaics, this should not be possible (so this test/block should really be removed)
  2005. {
  2006. ERRLOG("_fill() - tail (%s) found at intermediate level: %s", tail.queryName(), head.str());
  2007. append(*LINK((PTree *)&iter->query()));
  2008. append(*LINK((PTree *)&current));
  2009. return true;
  2010. }
  2011. else if (_fill(iter->query(), _nextSep, tail))
  2012. {
  2013. append(*LINK((PTree *)&current));
  2014. return true;
  2015. }
  2016. }
  2017. }
  2018. return false;
  2019. }
  2020. bool CPTStack::fill(IPropertyTree &root, const char *xpath, IPropertyTree &tail)
  2021. {
  2022. kill();
  2023. bool res = _fill(root, xpath, tail);
  2024. unsigned elems = ordinality();
  2025. if (elems<2) return res;
  2026. elems--;
  2027. unsigned to = 0;
  2028. while (to < elems)
  2029. swap(elems--, to++);
  2030. return res;
  2031. }
  2032. /////////////////////
  2033. StringBuffer &CPTStack::getAbsolutePath(StringBuffer &str)
  2034. {
  2035. str.append('/');
  2036. IPropertyTree *parent = &item(0);
  2037. if (ordinality()>1)
  2038. {
  2039. unsigned i = 1;
  2040. loop
  2041. {
  2042. IPropertyTree *child = &item(i);
  2043. str.append(child->queryName());
  2044. str.append('[');
  2045. aindex_t pos = parent->queryChildIndex(child);
  2046. #ifdef DEBUG_HPCC_11202
  2047. if (NotFound == pos)
  2048. {
  2049. ERRLOG("Invalid CPTStack detected");
  2050. ForEachItemIn(i, *this)
  2051. {
  2052. PTree &tree = item(i);
  2053. DBGLOG("PTree path item %d = %s", i, tree.queryName());
  2054. }
  2055. PrintStackReport();
  2056. }
  2057. #endif
  2058. str.append(pos+1);
  2059. str.append(']');
  2060. if (++i >= ordinality())
  2061. break;
  2062. str.append('/');
  2063. parent = child;
  2064. }
  2065. }
  2066. return str;
  2067. }
  2068. /////////////////////
  2069. CServerConnection::~CServerConnection()
  2070. {
  2071. Owned<LinkingCriticalBlock> checkedCritBlock;
  2072. if (!RTM_MODE(mode, RTM_INTERNAL))
  2073. {
  2074. ForEachItemIn(s, subscriptions)
  2075. SDSManager->remove(subscriptions.item(s).queryId());
  2076. checkedCritBlock.setown(new LinkingCriticalBlock(SDSManager->connDestructCrit, __FILE__, __LINE__));
  2077. }
  2078. ptreePath.kill();
  2079. root.clear();
  2080. }
  2081. void CServerConnection::aborted(SessionId id)
  2082. {
  2083. LOG(MCdebugInfo(100), unknownJob, "CServerConnection: connection aborted (%" I64F "x) sessId=%" I64F "x",connectionId, id);
  2084. #if 0 // JCSMORE - think this is ok, but concerned about deadlock, change later.
  2085. Owned<CLCLockBlock> lockBlock = new CLCWriteLockBlock(((CCovenSDSManager &)manager).dataRWLock, readWriteTimeout, __FILE__, __LINE__);
  2086. SDSManager->disconnect(connectionId, false);
  2087. #else
  2088. Owned<CLCLockBlock> lockBlock = new CLCReadLockBlock(((CCovenSDSManager &)manager).dataRWLock, readWriteTimeout, __FILE__, __LINE__);
  2089. SDSManager->disconnect(connectionId, false, &lockBlock);
  2090. #endif
  2091. }
  2092. ///////////////////
  2093. enum IncCmd { None, PropDelete, AttrDelete, PropChange, PropNew, PropExisting, ChildEndMarker, PropRename, AttrChange };
  2094. CRemoteTreeBase::CRemoteTreeBase(const char *name, IPTArrayValue *value, ChildMap *children)
  2095. : PTree(name, ipt_none, value, children)
  2096. {
  2097. serverId = 0;
  2098. }
  2099. CRemoteTreeBase::CRemoteTreeBase(MemoryBuffer &mb)
  2100. {
  2101. serverId = 0;
  2102. }
  2103. void CRemoteTreeBase::deserializeRT(MemoryBuffer &src)
  2104. {
  2105. deserializeSelfRT(src);
  2106. deserializeChildrenRT(src);
  2107. }
  2108. void CRemoteTreeBase::deserializeChildrenRT(MemoryBuffer &src)
  2109. {
  2110. StringAttr eName;
  2111. loop
  2112. {
  2113. size32_t pos = src.getPos();
  2114. src.read(eName);
  2115. if (!eName.length())
  2116. break;
  2117. src.reset(pos); // reset to re-read tree name
  2118. CRemoteTreeBase *child = (CRemoteTreeBase *) create(NULL);
  2119. child->deserializeRT(src);
  2120. addPropTree(eName, child);
  2121. }
  2122. }
  2123. void CRemoteTreeBase::deserializeSelfRT(MemoryBuffer &mb)
  2124. {
  2125. deserializeSelf(mb);
  2126. assertex(!isnocase());
  2127. __int64 _serverId;
  2128. mb.read(_serverId);
  2129. if (_serverId)
  2130. setServerId(_serverId); // ignore deserializing 0 serverId (indicated new)
  2131. }
  2132. void CRemoteTreeBase::setServerId(__int64 _serverId)
  2133. {
  2134. serverId = _serverId;
  2135. }
  2136. void CRemoteTreeBase::clearChildren()
  2137. {
  2138. if (children)
  2139. {
  2140. children->Release();
  2141. children=NULL;
  2142. }
  2143. }
  2144. CRemoteTreeBase *CRemoteTreeBase::createChild(int pos, const char *childName)
  2145. {
  2146. CRemoteTreeBase *child = (CRemoteTreeBase *) create(NULL);
  2147. if (-1 == pos)
  2148. child = (CRemoteTreeBase *) addPropTree(childName, child);
  2149. else
  2150. {
  2151. unsigned e = 0;
  2152. if (children)
  2153. {
  2154. PTree *match = (PTree *) children->query(childName);
  2155. if (match)
  2156. {
  2157. IPTArrayValue *value = match->queryValue();
  2158. e = value && value->isArray()?value->elements() : 1;
  2159. }
  2160. }
  2161. if ((unsigned)pos == e)
  2162. child = (CRemoteTreeBase *) addPropTree(childName, child);
  2163. else
  2164. {
  2165. StringBuffer childPos(childName);
  2166. childPos.append("[").append(pos+1).append("]");
  2167. child = (CRemoteTreeBase *) addPropTree(childPos.str(), child);
  2168. }
  2169. }
  2170. return child;
  2171. }
  2172. ///////////
  2173. static CheckedCriticalSection suppressedOrphanUnlockCrit; // to temporarily suppress unlockall
  2174. static bool suppressedOrphanUnlock=false;
  2175. #if defined(new)
  2176. #define __old_new new
  2177. #undef new
  2178. #endif
  2179. //Do not override the packing for this class - otherwise the fixed size allocator will allocate
  2180. //misaligned objects, which can cause problems on some architectures (especially for atomic operations)
  2181. class CServerRemoteTree : public CRemoteTreeBase
  2182. {
  2183. DECL_NAMEDCOUNT;
  2184. class COrphanHandler : public ChildMap
  2185. {
  2186. public:
  2187. COrphanHandler() : ChildMap() { }
  2188. ~COrphanHandler() { _releaseAll(); }
  2189. static void setOrphans(CServerRemoteTree &tree, bool tf)
  2190. {
  2191. if (tf)
  2192. IptFlagSet(tree.flags, ipt_ext5);
  2193. else
  2194. IptFlagClr(tree.flags, ipt_ext5);
  2195. IPTArrayValue *v = tree.queryValue();
  2196. if (v && v->isArray())
  2197. {
  2198. unsigned e;
  2199. for(e=0; e<v->elements(); e++)
  2200. setOrphans(*(CServerRemoteTree *)v->queryElement(e), tf);
  2201. }
  2202. if (tree.queryChildren())
  2203. {
  2204. if (SDSManager->queryStopped()) return;
  2205. SuperHashIteratorOf<IPropertyTree> iter(*tree.queryChildren());
  2206. ForEach (iter)
  2207. setOrphans((CServerRemoteTree &)iter.query(), tf);
  2208. }
  2209. }
  2210. virtual void onAdd(void *e) // ensure memory of constructed multi value elements are no longer orphaned.
  2211. {
  2212. ChildMap::onAdd(e);
  2213. CServerRemoteTree &tree = *((CServerRemoteTree *)(IPropertyTree *)e);
  2214. setOrphans(tree, false);
  2215. }
  2216. virtual void onRemove(void *e)
  2217. {
  2218. CServerRemoteTree &tree = *((CServerRemoteTree *)(IPropertyTree *)e);
  2219. bool dounlock;
  2220. {
  2221. CHECKEDCRITICALBLOCK(suppressedOrphanUnlockCrit, fakeCritTimeout);
  2222. dounlock = !suppressedOrphanUnlock;
  2223. }
  2224. if (dounlock) // element is moving, don't orphan or unlock
  2225. {
  2226. setOrphans(tree, true);
  2227. SDSManager->unlockAll(tree.queryServerId());
  2228. }
  2229. ChildMap::onRemove(e);
  2230. }
  2231. virtual bool replace(const char *key, IPropertyTree *tree) // provides different semantics, used if element being replaced is not to be treated as deleted.
  2232. {
  2233. CHECKEDCRITICALBLOCK(suppressedOrphanUnlockCrit, fakeCritTimeout);
  2234. BoolSetBlock bblock(suppressedOrphanUnlock);
  2235. bool ret = ChildMap::replace(key, tree);
  2236. return ret;
  2237. }
  2238. virtual bool set(const char *key, IPropertyTree *tree)
  2239. {
  2240. // NB: be careful if replacing, to remove element first, because may self-destruct if lastref in middle of SuperHashTable::replace, leaving tablecount wrong.
  2241. unsigned vs = getHashFromElement(tree);
  2242. const void *fp = getFindParam(tree);
  2243. IPropertyTree *et = (IPropertyTree *)SuperHashTable::find(vs, fp);
  2244. if (et)
  2245. removeExact(et);
  2246. return ChildMap::set(key, tree);
  2247. }
  2248. };
  2249. static void clearTree(CServerRemoteTree &tree)
  2250. {
  2251. Owned<IPropertyTreeIterator> iter = tree.getElements("*");
  2252. ForEach(*iter)
  2253. {
  2254. CServerRemoteTree &child = (CServerRemoteTree &)iter->query();
  2255. clearTree(child);
  2256. }
  2257. tree.clear();
  2258. }
  2259. public:
  2260. #ifdef _POOLED_SERVER_REMOTE_TREE
  2261. void * operator new(memsize_t sz)
  2262. {
  2263. #ifdef _DEBUG
  2264. assertex(sz==sizeof(CServerRemoteTree));
  2265. #endif
  2266. return CServerRemoteTree_Allocator->alloc();
  2267. }
  2268. void operator delete( void * p )
  2269. {
  2270. CServerRemoteTree_Allocator->dealloc(p);
  2271. }
  2272. #endif
  2273. CServerRemoteTree(MemoryBuffer &mb) : CRemoteTreeBase(mb) { init(); }
  2274. CServerRemoteTree(const char *name=NULL, IPTArrayValue *value=NULL, ChildMap *children=NULL)
  2275. : CRemoteTreeBase(name, value, children) { init(); }
  2276. ~CServerRemoteTree()
  2277. {
  2278. if (hasProp(NOTIFY_ATTR))
  2279. SDSManager->handleNodeNotify(notify_delete, *this);
  2280. __int64 index = getPropInt64(EXT_ATTR);
  2281. if (index)
  2282. {
  2283. try { SDSManager->deleteExternal(index); }
  2284. catch (IException *e)
  2285. {
  2286. LOG(MCoperatorWarning, unknownJob, e, StringBuffer("Deleting external reference for ").append(queryName()).str());
  2287. e->Release();
  2288. }
  2289. }
  2290. if (SDSManager->queryStopped()) return; // don't bother building up free list that will never be used hence (could get v. big/slow)
  2291. if (isSubscribed())
  2292. SDSManager->notifyNodeDelete(*this);
  2293. CHECKEDCRITICALBLOCK(SDSManager->treeRegCrit, fakeCritTimeout);
  2294. clear(); // NB: *should* already be cleared, when tree is removed via removeTree
  2295. }
  2296. virtual bool isEquivalent(IPropertyTree *tree) { return (NULL != QUERYINTERFACE(tree, CServerRemoteTree)); }
  2297. PDState processData(CServerConnection &connection, IPropertyTree &changeTree, MemoryBuffer &newIds);
  2298. void init()
  2299. {
  2300. INIT_NAMEDCOUNT;
  2301. assertex(!isnocase());
  2302. CHECKEDCRITICALBLOCK(SDSManager->treeRegCrit, fakeCritTimeout);
  2303. SDSManager->queryAllNodes().addElem(this);
  2304. }
  2305. void clear()
  2306. {
  2307. // always called inside SDSManager->treeRegCrit
  2308. if (serverId)
  2309. {
  2310. SDSManager->queryAllNodes().freeElem(serverId);
  2311. serverId = 0;
  2312. }
  2313. }
  2314. virtual bool removeTree(IPropertyTree *_child)
  2315. {
  2316. if (!_child)
  2317. return false;
  2318. dbgassertex(QUERYINTERFACE(_child, CServerRemoteTree));
  2319. Linked<CServerRemoteTree> child = static_cast<CServerRemoteTree *>(_child);
  2320. if (!CRemoteTreeBase::removeTree(child))
  2321. return false;
  2322. CHECKEDCRITICALBLOCK(SDSManager->treeRegCrit, fakeCritTimeout);
  2323. clearTree(*child);
  2324. return true;
  2325. }
  2326. virtual bool isOrphaned() const { return IptFlagTst(flags, ipt_ext5); }
  2327. virtual void setServerId(__int64 _serverId)
  2328. {
  2329. if (serverId && serverId != _serverId)
  2330. WARNLOG("Unexpected - client server id mismatch in %s, id=%" I64F "x", queryName(), _serverId);
  2331. CRemoteTreeBase::setServerId(_serverId);
  2332. }
  2333. virtual CSubscriberContainerList *getSubscribers(const char *xpath, CPTStack &stack)
  2334. {
  2335. return SDSManager->getSubscribers(xpath, stack);
  2336. }
  2337. IPropertyTree *create(const char *name=NULL, IPTArrayValue *value=NULL, ChildMap *children=NULL, bool existing=false)
  2338. {
  2339. return new CServerRemoteTree(name, value, children);
  2340. }
  2341. IPropertyTree *create(MemoryBuffer &mb)
  2342. {
  2343. return new CServerRemoteTree(mb);
  2344. }
  2345. virtual void createChildMap() { children = new COrphanHandler(); }
  2346. inline bool testExternalCandidate()
  2347. {
  2348. // maybe be other cases.
  2349. return (value && value->queryValueSize() >= SDSManager->queryExternalThreshold());
  2350. }
  2351. void serializeCutOffRT(MemoryBuffer &tgt, int cutoff=-1, int depth=0, bool extValues=true)
  2352. {
  2353. serializeSelfRT(tgt, extValues);
  2354. serializeCutOffChildrenRT(tgt, cutoff, depth, extValues);
  2355. }
  2356. void serializeCutOffChildrenRT(MemoryBuffer &tgt, int cutoff=-1, int depth=0, bool extValues=true)
  2357. {
  2358. #ifndef ALWAYSLAZY_NOTUSED
  2359. if (cutoff < 0)
  2360. {
  2361. if (FETCH_ENTIRE_COND == cutoff && getPropBool("@alwaysLazy"))
  2362. {
  2363. tgt.append("");
  2364. return; // i.e. truncate here, this already serialized, lazy fetch children
  2365. }
  2366. }
  2367. else if (getPropBool("@fetchEntire"))
  2368. cutoff = FETCH_ENTIRE_COND;
  2369. #else
  2370. if (cutoff >= 0 && getPropBool("@fetchEntire"))
  2371. cutoff = FETCH_ENTIRE_COND; // NB: can change all _COND references to FETCH_ENTIRE if not using alwaysFetch anymore
  2372. #endif
  2373. if (cutoff < 0 || depth<cutoff)
  2374. {
  2375. IPropertyTreeIterator *iter = getElements("*");
  2376. iter->first();
  2377. while (iter->isValid())
  2378. {
  2379. IPropertyTree *_child = &iter->query();
  2380. CServerRemoteTree *child = QUERYINTERFACE(_child, CServerRemoteTree); assertex(child);
  2381. child->serializeCutOffRT(tgt, cutoff, depth+1, extValues);
  2382. iter->next();
  2383. }
  2384. iter->Release();
  2385. }
  2386. tgt.append(""); // element terminator. i.e. blank child name.
  2387. }
  2388. void serializeSelfRT(MemoryBuffer &mb, bool extValues)
  2389. {
  2390. __int64 index = getPropInt64(EXT_ATTR);
  2391. if (index)
  2392. {
  2393. SDSManager->serializeExternal(index, *this, mb, extValues);
  2394. mb.append(serverId);
  2395. }
  2396. else
  2397. {
  2398. serializeSelf(mb);
  2399. mb.append(serverId);
  2400. }
  2401. byte STIInfo = 0;
  2402. if (children && children->count())
  2403. STIInfo += STI_HaveChildren;
  2404. if (index)
  2405. STIInfo += STI_External;
  2406. mb.append(STIInfo);
  2407. }
  2408. virtual void deserializeSelfRT(MemoryBuffer &src)
  2409. {
  2410. CRemoteTreeBase::deserializeSelfRT(src);
  2411. assertex(!isnocase());
  2412. byte STIInfo;
  2413. src.read(STIInfo);
  2414. }
  2415. virtual void removingElement(IPropertyTree *tree, unsigned pos)
  2416. {
  2417. COrphanHandler::setOrphans(*(CServerRemoteTree *)tree, true);
  2418. CRemoteTreeBase::removingElement(tree, pos);
  2419. }
  2420. virtual bool isCompressed(const char *xpath=NULL) const
  2421. {
  2422. if (isAttribute(xpath)) return false;
  2423. if (CRemoteTreeBase::isCompressed(xpath)) return true;
  2424. if (SDSManager->ignoreExternals) return false;
  2425. CServerRemoteTree *child = (CServerRemoteTree *)queryPropTree(xpath);
  2426. return child->hasProp(EXT_ATTR);
  2427. }
  2428. bool getProp(const char *xpath, StringBuffer &ret) const
  2429. {
  2430. if (xpath)
  2431. return CRemoteTreeBase::getProp(xpath, ret);
  2432. if (SDSManager->ignoreExternals)
  2433. return CRemoteTreeBase::getProp(xpath, ret);
  2434. CHECKEDCRITICALBLOCK(extCrit, fakeCritTimeout);
  2435. __int64 index = getPropInt64(EXT_ATTR);
  2436. if (!index)
  2437. return CRemoteTreeBase::getProp(xpath, ret);
  2438. MemoryBuffer mbv, mb;
  2439. SDSManager->getExternalValue(index, mbv);
  2440. CPTValue v(mbv);
  2441. v.getValue(mb, true);
  2442. unsigned len = mb.length();
  2443. char *mem = (char *)mb.detach();
  2444. mem[len-1] = '\0';
  2445. ret.setBuffer(len, mem, len-1);
  2446. return true;
  2447. }
  2448. void setSubscribed(bool tf)
  2449. {
  2450. if (tf)
  2451. IptFlagSet(flags, ipt_ext4);
  2452. else
  2453. IptFlagClr(flags, ipt_ext4);
  2454. }
  2455. inline bool isSubscribed() const
  2456. {
  2457. return IptFlagTst(flags, ipt_ext4);
  2458. }
  2459. private:
  2460. PDState processData(IPropertyTree &changeTree, Owned<CBranchChange> &parentBranchChange, MemoryBuffer &newIds);
  2461. PDState checkChange(IPropertyTree &tree, CBranchChange &parentBranchChange);
  2462. friend class COrphanHandler;
  2463. };
  2464. class CNodeSubscriberContainer : public CSubscriberContainerBase
  2465. {
  2466. StringAttr xpath;
  2467. bool sendValue;
  2468. ICopyArrayOf<CServerRemoteTree> nodes; // never linked, node must signal the unsubscription and removal of subscriber and these references
  2469. public:
  2470. CNodeSubscriberContainer(ISubscription *subscriber, SubscriptionId id, bool _sendValue, const char *_xpath)
  2471. : CSubscriberContainerBase(subscriber, id), sendValue(_sendValue), xpath(_xpath)
  2472. {
  2473. }
  2474. bool querySendValue() const { return sendValue; }
  2475. void add(CServerRemoteTree &node) { nodes.append(node); }
  2476. ICopyArrayOf<CServerRemoteTree> &queryNodes() { return nodes; }
  2477. MemoryBuffer &getInfo(MemoryBuffer &out) const
  2478. {
  2479. out.append(id).append(xpath).append(nodes.ordinality());
  2480. return out;
  2481. }
  2482. };
  2483. class CNodeSubscriptionManager : implements INodeSubscriptionManager, public CSimpleInterface
  2484. {
  2485. public:
  2486. class CNodeSubscriberContainerList : public CSimpleInterfaceOf<IInterface>
  2487. {
  2488. CServerRemoteTree *node;
  2489. ICopyArrayOf<CNodeSubscriberContainer> subscribers;
  2490. public:
  2491. CNodeSubscriberContainerList(CServerRemoteTree *_node) : node(_node)
  2492. {
  2493. }
  2494. const void *queryFindParam() const { return &node; }
  2495. ICopyArrayOf<CNodeSubscriberContainer> &querySubscribers() { return subscribers; }
  2496. void getSubscribers(IArrayOf<CNodeSubscriberContainer> &linkedSubscribers)
  2497. {
  2498. ForEachItemIn(s, subscribers)
  2499. linkedSubscribers.append(*LINK(&subscribers.item(s)));
  2500. }
  2501. void add(CNodeSubscriberContainer &subscriber) { subscribers.append(subscriber); }
  2502. };
  2503. CCovenSDSManager &owner;
  2504. OwningSimpleHashTableOf<CNodeSubscriberContainer, SubscriptionId> subscribersById;
  2505. OwningSimpleHashTableOf<CNodeSubscriberContainerList, CServerRemoteTree *> subscriberListByNode;
  2506. mutable CriticalSection subscriberListCrit;
  2507. void _notify(CServerRemoteTree *node, PDState state, IArrayOf<CNodeSubscriberContainer> &subscribers)
  2508. {
  2509. MemoryBuffer sendValueNotifyData;
  2510. int lastSendValue = -1;
  2511. while (subscribers.ordinality())
  2512. {
  2513. Owned<CNodeSubscriberContainer> subscriber = &subscribers.popGet();
  2514. if (subscriber->querySendValue())
  2515. {
  2516. if (1 != lastSendValue) // overkill unless many subscribers to same node
  2517. {
  2518. MemoryBuffer mb;
  2519. node->getPropBin(NULL, mb);
  2520. buildNotifyData(sendValueNotifyData.clear(), state, NULL, &mb);
  2521. lastSendValue = 1;
  2522. }
  2523. SDSManager->handleNotify(subscriber.getClear(), sendValueNotifyData);
  2524. }
  2525. else
  2526. {
  2527. if (0 != lastSendValue) // overkill unless many subscribers to same node
  2528. {
  2529. buildNotifyData(sendValueNotifyData.clear(), state, NULL, NULL);
  2530. lastSendValue = 0;
  2531. }
  2532. SDSManager->handleNotify(subscriber.getClear(), sendValueNotifyData);
  2533. }
  2534. }
  2535. }
  2536. void _notify(CServerRemoteTree *node, PDState state)
  2537. {
  2538. CNodeSubscriberContainerList *subscriberList = subscriberListByNode.find(node);
  2539. assertex(subscriberList);
  2540. IArrayOf<CNodeSubscriberContainer> subscribers;
  2541. subscriberList->getSubscribers(subscribers);
  2542. _notify(node, state, subscribers);
  2543. }
  2544. void _removeNode(CServerRemoteTree *node, SubscriptionId id)
  2545. {
  2546. CNodeSubscriberContainerList *subscriberList = subscriberListByNode.find(node);
  2547. assertex(subscriberList);
  2548. ICopyArrayOf<CNodeSubscriberContainer> &subscribers = subscriberList->querySubscribers();
  2549. ForEachItemInRev(s, subscribers)
  2550. {
  2551. CNodeSubscriberContainer &subscriber = subscribers.item(s);
  2552. if (0 == id) // remove all associated subscribers (node being deleted)
  2553. {
  2554. ICopyArrayOf<CServerRemoteTree> &nodes = subscriber.queryNodes();
  2555. verifyex(nodes.zap(*node));
  2556. subscribers.remove(s);
  2557. if (0 == nodes.ordinality()) // IOW this was the last node this subscriber was associated with
  2558. subscribersById.removeExact(&subscriber);
  2559. }
  2560. else if (subscriber.queryId() == id)
  2561. subscribers.remove(s);
  2562. }
  2563. if (0 == subscribers.ordinality())
  2564. {
  2565. node->setSubscribed(false);
  2566. subscriberListByNode.removeExact(subscriberList);
  2567. }
  2568. }
  2569. public:
  2570. IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
  2571. CNodeSubscriptionManager(CCovenSDSManager &_owner) : owner(_owner) { }
  2572. void notify(CServerRemoteTree &node, PDState state)
  2573. {
  2574. // shouldn't be here, unless node is in subscribers table
  2575. CriticalBlock b(subscriberListCrit);
  2576. _notify(&node, state);
  2577. }
  2578. void notifyDelete(CServerRemoteTree *node)
  2579. {
  2580. // shouldn't be here, unless node is in subscribers table
  2581. CriticalBlock b(subscriberListCrit);
  2582. /* Need to be careful not to release subscribers here (on this thread)
  2583. * 1) gather subscribers(linked)
  2584. * 2) remove nodes and lists, so no longer in use by SDS
  2585. * 3) Hand ownership over to notification mechanism
  2586. *
  2587. * Subscribers will be released when notification is done with them.
  2588. */
  2589. CNodeSubscriberContainerList *subscriberList = subscriberListByNode.find(node);
  2590. assertex(subscriberList);
  2591. IArrayOf<CNodeSubscriberContainer> linkedSubscribers;
  2592. subscriberList->getSubscribers(linkedSubscribers);
  2593. _removeNode(node, 0);
  2594. // NB: Notification will take ownership of subscribers being notified.
  2595. _notify(node, PDS_Deleted, linkedSubscribers);
  2596. }
  2597. // ISubscriptionManager impl.
  2598. virtual void add(ISubscription *sub, SubscriptionId id)
  2599. {
  2600. CHECKEDDALIREADLOCKBLOCK(owner.dataRWLock, readWriteTimeout);
  2601. CHECKEDCRITICALBLOCK(owner.treeRegCrit, fakeCritTimeout);
  2602. CriticalBlock b(subscriberListCrit);
  2603. /* calls back out to owner to scan for match, so that SDSManager can protect root/treereg.
  2604. * It calls back (associateSubscriber) in this class to add subscribers based on matches.
  2605. */
  2606. owner.addNodeSubscriber(sub, id);
  2607. }
  2608. virtual void remove(SubscriptionId id)
  2609. {
  2610. /* important to ensure have exclusive data lock on removal, ahead of subscriber lock
  2611. * as can get notifications whilst holding data lock, e.g. notifyDelete on node destruction.
  2612. */
  2613. CHECKEDDALIREADLOCKBLOCK(owner.dataRWLock, readWriteTimeout);
  2614. CHECKEDCRITICALBLOCK(owner.treeRegCrit, fakeCritTimeout);
  2615. CriticalBlock b(subscriberListCrit);
  2616. /* calls back out to owner to protect root/treereg.
  2617. * It calls back into removeSubscriberAssociation.
  2618. */
  2619. owner.removeNodeSubscriber(id);
  2620. }
  2621. virtual void removeSubscriberAssociation(SubscriptionId id) // Always called back from within remove() above.
  2622. {
  2623. CNodeSubscriberContainer *subscriber = subscribersById.find(id);
  2624. if (!subscriber)
  2625. return; // may not exist if removed already
  2626. ICopyArrayOf<CServerRemoteTree> &nodes = subscriber->queryNodes();
  2627. ForEachItemIn(n, nodes)
  2628. {
  2629. CServerRemoteTree &node = nodes.item(n);
  2630. _removeNode(&node, id);
  2631. }
  2632. verifyex(subscribersById.removeExact(subscriber));
  2633. }
  2634. void associateSubscriber(CNodeSubscriberContainer &subscriber) // Always called back from within add() above.
  2635. {
  2636. /* caller has established there are matches and added them to 'subscriber'
  2637. * add to HT's
  2638. */
  2639. verifyex(subscribersById.add(*LINK(&subscriber)));
  2640. ICopyArrayOf<CServerRemoteTree> &nodes = subscriber.queryNodes();
  2641. ForEachItemIn(n, nodes)
  2642. {
  2643. CServerRemoteTree *node = &nodes.item(n);
  2644. CNodeSubscriberContainerList *subscriberList = subscriberListByNode.find(node);
  2645. if (!subscriberList)
  2646. {
  2647. subscriberList = new CNodeSubscriberContainerList(node);
  2648. verifyex(subscriberListByNode.add(* subscriberList));
  2649. }
  2650. subscriberList->add(subscriber);
  2651. }
  2652. }
  2653. MemoryBuffer &collectSubscribers(MemoryBuffer &out) const
  2654. {
  2655. /* important to ensure have exclusive data lock on removal, ahead of subscriber lock
  2656. * as can get notifications whilst holding data lock, e.g. notifyDelete on node destruction.
  2657. */
  2658. CHECKEDDALIREADLOCKBLOCK(owner.dataRWLock, readWriteTimeout);
  2659. CHECKEDCRITICALBLOCK(owner.treeRegCrit, fakeCritTimeout);
  2660. CriticalBlock b(subscriberListCrit);
  2661. out.append(subscribersById.count());
  2662. SuperHashIteratorOf<CNodeSubscriberContainer> sdsNodeIter(subscribersById);
  2663. ForEach(sdsNodeIter)
  2664. sdsNodeIter.query().getInfo(out);
  2665. return out;
  2666. }
  2667. };
  2668. #if defined(_WIN32) && defined(__old_new)
  2669. #define new __old_new
  2670. #endif
  2671. void populateWithServerIds(IPropertyTree *matchParent, CRemoteTreeBase *parent)
  2672. {
  2673. matchParent->setPropInt64("@serverId", parent->queryServerId());
  2674. Owned<IPropertyTreeIterator> iter = matchParent->getElements("*");
  2675. ForEach (*iter)
  2676. {
  2677. IPropertyTree *matchChild = &iter->query();
  2678. StringBuffer path(matchChild->queryName());
  2679. path.append("[").append(matchChild->queryProp("@pos")).append("]");
  2680. CRemoteTreeBase *child = (CRemoteTreeBase *)parent->queryPropTree(path.str());
  2681. assertex(child);
  2682. populateWithServerIds(matchChild, child);
  2683. }
  2684. }
  2685. IPropertyTree *createServerTree(const char *tag=NULL)
  2686. {
  2687. return new CServerRemoteTree(tag);
  2688. }
  2689. // JCSMORE - these should be error conditions, for consistency with previous release not so for now.
  2690. #define consistencyCheck(TEXT, IDTREE, LOCALTREE, PATH, PARENTNAME, ID) \
  2691. if (!IDTREE) \
  2692. { \
  2693. StringBuffer s(TEXT": Consistency check failure, id'd tree not found: "); \
  2694. s.append(PATH).append(", id=").append(ID); \
  2695. LOG(MCoperatorWarning, unknownJob, s.str()); \
  2696. } \
  2697. else if (!LOCALTREE) \
  2698. { \
  2699. StringBuffer s(TEXT": Consistency check failure, positional property specification: "); \
  2700. s.append(PATH).append(", in client update not found in parent tree: ").append(PARENTNAME); \
  2701. LOG(MCoperatorWarning, unknownJob, s.str()); \
  2702. } \
  2703. else if (IDTREE != LOCALTREE) \
  2704. { \
  2705. StringBuffer s(TEXT": Consistency check failure, positional property specification does not match id'd tree, prop=");\
  2706. s.append(PATH); \
  2707. LOG(MCoperatorWarning, unknownJob, s.str()); \
  2708. }
  2709. PDState CServerRemoteTree::checkChange(IPropertyTree &changeTree, CBranchChange &parentBranchChange)
  2710. {
  2711. PDState res = PDS_None;
  2712. bool checkExternal = false;
  2713. Owned<IPropertyTreeIterator> iter = changeTree.getElements("*");
  2714. ICopyArrayOf<IPropertyTree> toremove;
  2715. ForEach(*iter)
  2716. {
  2717. IPropertyTree &e = iter->query();
  2718. const char *name = e.queryName();
  2719. switch (name[0])
  2720. {
  2721. case 'R':
  2722. {
  2723. IPropertyTree *idTree = SDSManager->queryRegisteredTree(e.getPropInt64("@id"));
  2724. if (!idTree)
  2725. throw MakeSDSException(SDSExcpt_OrphanedNode, "renaming %s to %s", e.queryProp("@from"), e.queryProp("@to"));
  2726. #ifdef SIBLING_MOVEMENT_CHECK
  2727. StringBuffer localTreePath(e.queryProp("@from"));
  2728. localTreePath.append('[').append(e.getPropInt("@pos")).append(']');
  2729. IPropertyTree *localTree = queryPropTree(localTreePath.str());
  2730. consistencyCheck("RENAME", idTree, localTree, localTreePath.str(), queryName(), e.getPropInt64("@id"));
  2731. #endif
  2732. int pos = findChild(idTree);
  2733. if (renameTree(idTree, e.queryProp("@to")))
  2734. {
  2735. e.setPropInt("@pos", pos+1);
  2736. mergePDState(res, PDS_Structure);
  2737. PDState _res = res;
  2738. mergePDState(_res, PDS_Renames);
  2739. Owned<CBranchChange> childChange = new CBranchChange(*(CRemoteTreeBase *)idTree);
  2740. childChange->noteChange(_res, _res);
  2741. childChange->Link();
  2742. parentBranchChange.addChildBranch(*childChange);
  2743. }
  2744. else
  2745. {
  2746. toremove.append(e);
  2747. continue;
  2748. }
  2749. break;
  2750. }
  2751. case 'D':
  2752. {
  2753. IPropertyTree *idTree = SDSManager->queryRegisteredTree(e.getPropInt64("@id"));
  2754. if (!idTree)
  2755. {
  2756. toremove.append(e);
  2757. continue;
  2758. }
  2759. #ifdef SIBLING_MOVEMENT_CHECK
  2760. StringBuffer localTreePath(e.queryProp("@name"));
  2761. localTreePath.append('[').append(e.getPropInt("@pos")).append(']');
  2762. IPropertyTree *localTree = queryPropTree(localTreePath.str());
  2763. consistencyCheck("DELETE", idTree, localTree, localTreePath.str(), queryName(), e.getPropInt64("@id"));
  2764. #endif
  2765. int pos = findChild(idTree);
  2766. if (NotFound == pos)
  2767. {
  2768. toremove.append(e);
  2769. continue;
  2770. }
  2771. e.setPropInt("@pos", pos+1);
  2772. Owned<CBranchChange> childChange = new CBranchChange(*(CRemoteTreeBase *)idTree);
  2773. if (!removeTree(idTree))
  2774. throw MakeSDSException(-1, "::checkChange - Failed to remove child(%s) from parent(%s) at %s(%d)", idTree->queryName(), queryName(), __FILE__, __LINE__);
  2775. mergePDState(res, PDS_Structure);
  2776. PDState _res = res;
  2777. mergePDState(_res, PDS_Deleted);
  2778. childChange->noteChange(_res, _res);
  2779. childChange->Link();
  2780. parentBranchChange.addChildBranch(*childChange);
  2781. break;
  2782. }
  2783. case 'A':
  2784. {
  2785. switch (name[1])
  2786. {
  2787. case 'D':
  2788. {
  2789. Owned<IAttributeIterator> iter = e.getAttributes();
  2790. ForEach(*iter)
  2791. {
  2792. if (removeAttr(iter->queryName()))
  2793. mergePDState(res, PDS_Data);
  2794. }
  2795. break;
  2796. }
  2797. case 'C':
  2798. {
  2799. Owned<IAttributeIterator> iter = e.getAttributes();
  2800. ForEach(*iter)
  2801. setProp(iter->queryName(), iter->queryValue());
  2802. mergePDState(res, PDS_Data);
  2803. break;
  2804. }
  2805. default:
  2806. throwUnexpected();
  2807. }
  2808. break;
  2809. }
  2810. case 'T':
  2811. break;
  2812. }
  2813. }
  2814. ForEachItemIn(tr, toremove)
  2815. changeTree.removeTree(&toremove.item(tr));
  2816. if (changeTree.getPropBool("@localValue"))
  2817. {
  2818. bool binary=changeTree.isBinary(NULL);
  2819. IPTArrayValue *v = ((PTree &)changeTree).queryValue();
  2820. setValue(v?new CPTValue(v->queryValueRawSize(), v->queryValueRaw(), binary, true, v->isCompressed()):NULL, binary);
  2821. if (changeTree.getPropBool("@new"))
  2822. mergePDState(res, PDS_New);
  2823. mergePDState(res, PDS_Data);
  2824. checkExternal = true;
  2825. }
  2826. else if (changeTree.getPropBool("@appendValue"))
  2827. {
  2828. bool binary=changeTree.isBinary(NULL);
  2829. if (binary != isBinary(NULL))
  2830. throw MakeSDSException(-1, "Error attempting to append binary and non-binary data together, in node: %s", queryName());
  2831. __int64 index = getPropInt64(EXT_ATTR);
  2832. MemoryBuffer mb;
  2833. if (index)
  2834. {
  2835. MemoryBuffer mbv;
  2836. SDSManager->getExternalValue(index, mbv);
  2837. CPTValue v(mbv);
  2838. v.getValue(mb, binary);
  2839. }
  2840. else
  2841. getPropBin(NULL, mb);
  2842. changeTree.getPropBin(NULL, mb);
  2843. if (binary)
  2844. setPropBin(NULL, mb.length(), mb.toByteArray());
  2845. else
  2846. {
  2847. mb.append('\0');
  2848. setProp(NULL, (const char *)mb.toByteArray());
  2849. }
  2850. mergePDState(res, PDS_Data);
  2851. checkExternal = true;
  2852. }
  2853. if (checkExternal)
  2854. {
  2855. __int64 index = getPropInt64(EXT_ATTR);
  2856. if (index)
  2857. {
  2858. bool r = false;
  2859. if (!testExternalCandidate())
  2860. {
  2861. SDSManager->deleteExternal(index); // i.e. no longer e.g. now less than threshold.
  2862. r = removeProp(EXT_ATTR);
  2863. }
  2864. else
  2865. SDSManager->writeExternal(*this, false, index);
  2866. if (r)
  2867. {
  2868. IPropertyTree *t = changeTree.queryPropTree(ATTRDELETE_TAG);
  2869. if (!t)
  2870. t = changeTree.addPropTree(ATTRDELETE_TAG, createPTree());
  2871. t->addProp(EXT_ATTR, "");
  2872. }
  2873. else
  2874. changeTree.setProp(NULL, (const char *)NULL);
  2875. }
  2876. else if (testExternalCandidate())
  2877. {
  2878. try
  2879. {
  2880. SDSManager->writeExternal(*this);
  2881. IPropertyTree *t = changeTree.queryPropTree(ATTRCHANGE_TAG);
  2882. if (!t)
  2883. t = changeTree.addPropTree(ATTRCHANGE_TAG, createPTree());
  2884. changeTree.setProp(NULL, (const char *)NULL);
  2885. t->setProp(EXT_ATTR, queryProp(EXT_ATTR));
  2886. }
  2887. catch (IException *)
  2888. {
  2889. setProp(NULL, NULL); // in the event of failure during externalization, lose the value
  2890. throw;
  2891. }
  2892. }
  2893. }
  2894. return res;
  2895. }
  2896. PDState CServerRemoteTree::processData(CServerConnection &connection, IPropertyTree &changeTree, MemoryBuffer &newIds)
  2897. {
  2898. Owned<CBranchChange> top;
  2899. PDState res = processData(changeTree, top, newIds);
  2900. changeTree.removeProp("@name");
  2901. if (res)
  2902. SDSManager->writeTransactions++;
  2903. // return asap from here, don't even wait for pool threads to queue, can take time.
  2904. if (res && !RTM_MODE(connection.queryMode(), RTM_INTERNAL))
  2905. {
  2906. CPTStack stack = connection.queryPTreePath();
  2907. if (connection.queryRoot() == (IPropertyTree *)SDSManager->queryRoot())
  2908. stack.pop();
  2909. connection.notify();
  2910. SDSManager->startNotification(changeTree, stack, *top);
  2911. }
  2912. return res;
  2913. }
  2914. PDState CServerRemoteTree::processData(IPropertyTree &changeTree, Owned<CBranchChange> &parentBranchChange, MemoryBuffer &newIds)
  2915. {
  2916. Owned<CBranchChange> branchChange = new CBranchChange(*this);
  2917. PDState localChange, res;
  2918. localChange = res = checkChange(changeTree, *branchChange);
  2919. Owned<IPropertyTreeIterator> iter = changeTree.getElements(RESERVED_CHANGE_NODE);
  2920. if (iter->first())
  2921. {
  2922. bool levelNotified = false;
  2923. do
  2924. {
  2925. IPropertyTree &childChanges = iter->query();
  2926. CServerRemoteTree *child;
  2927. StringAttr childName;
  2928. if (childChanges.getPropBool("@new"))
  2929. {
  2930. child = (CServerRemoteTree *)createChild(childChanges.getPropInt("@pos", -1), childChanges.queryProp("@name"));
  2931. newIds.append(child->queryServerId());
  2932. mergePDState(localChange, PDS_Added);
  2933. mergePDState(res, PDS_Added);
  2934. }
  2935. else
  2936. {
  2937. child = (CServerRemoteTree *) SDSManager->queryRegisteredTree(childChanges.getPropInt64("@id"));
  2938. #ifdef SIBLING_MOVEMENT_CHECK
  2939. StringBuffer localTreePath(childChanges.queryProp("@name"));
  2940. localTreePath.append('[').append(childChanges.getPropInt("@pos")).append(']');
  2941. CRemoteTreeBase *localTree = (CRemoteTreeBase *) queryPropTree(localTreePath.str());
  2942. consistencyCheck("PROP UPDATE", child, localTree, localTreePath.str(), queryName(), childChanges.getPropInt64("@id"))
  2943. #endif
  2944. if (NULL == child)
  2945. throw MakeSDSException(SDSExcpt_ClientCacheDirty, "child(%s) not found in parent(%s) at %s(%d)", childChanges.queryProp("@name"), queryName(), __FILE__, __LINE__);
  2946. int pos = findChild(child);
  2947. if (NotFound == pos)
  2948. throw MakeSDSException(SDSExcpt_ClientCacheDirty, "child(%s) not found in parent(%s) at %s(%d)", child->queryName(), queryName(), __FILE__, __LINE__);
  2949. childChanges.setPropInt("@pos", pos+1);
  2950. }
  2951. if (!levelNotified)
  2952. {
  2953. levelNotified = true;
  2954. branchChange->noteChange(localChange, res);
  2955. }
  2956. mergePDState(res, child->processData(childChanges, branchChange, newIds));
  2957. }
  2958. while (iter->next());
  2959. }
  2960. else
  2961. branchChange->noteChange(localChange, res);
  2962. if ((localChange != PDS_None) && isSubscribed())
  2963. SDSManager->notifyNode(*this, localChange);
  2964. if (!parentBranchChange.get())
  2965. parentBranchChange.setown(branchChange.getClear());
  2966. else
  2967. parentBranchChange->addChildBranch(*branchChange.getClear());
  2968. return res;
  2969. }
  2970. /////////////////
  2971. class CPendingLockBlock
  2972. {
  2973. CLock &lock;
  2974. public:
  2975. CPendingLockBlock(CLock &_lock);
  2976. ~CPendingLockBlock();
  2977. };
  2978. typedef Int64Array IdPath;
  2979. #define LOCKSESSCHECK (1000*60*5)
  2980. class CLock : implements IInterface, public CInterface
  2981. {
  2982. DECL_NAMEDCOUNT;
  2983. CLockTable &table;
  2984. unsigned sub, readLocks, holdLocks, pending, waiting;
  2985. IdPath idPath;
  2986. ConnectionInfoMap connectionInfo;
  2987. mutable CheckedCriticalSection crit;
  2988. Semaphore sem;
  2989. StringAttr xpath;
  2990. __int64 treeId;
  2991. bool exclusive;
  2992. Linked<CServerRemoteTree> parent, child;
  2993. #ifdef _DEBUG
  2994. DebugInfo debugInfo;
  2995. #endif
  2996. bool validateConnectionSessions()
  2997. {
  2998. PROGLOG("validateConnectionSessions");
  2999. bool ret = false;
  3000. try
  3001. {
  3002. IArrayOf<IMapping> entries;
  3003. {
  3004. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  3005. HashIterator iter(connectionInfo);
  3006. ForEach (iter)
  3007. entries.append(*LINK(&iter.query()));
  3008. }
  3009. ForEachItemIn(e, entries)
  3010. {
  3011. IMapping &imap = entries.item(e);
  3012. LockData *lD = connectionInfo.mapToValue(&imap);
  3013. Owned<INode> node = querySessionManager().getProcessSessionNode(lD->sessId);
  3014. if (node)
  3015. {
  3016. SessionId nodeSessId = querySessionManager().lookupProcessSession(node);
  3017. if (nodeSessId)
  3018. {
  3019. if (lD->sessId != nodeSessId)
  3020. {
  3021. StringBuffer out("Removing stale connection session [");
  3022. out.appendf("%" I64F "x], connectionId [%" I64F "x]", lD->sessId, * ((ConnectionId *) imap.getKey()));
  3023. out.append(" xpath [").append(xpath).append("]");
  3024. PROGLOG("%s", out.str());
  3025. querySessionManager().stopSession(lD->sessId, true);
  3026. ret = true;
  3027. }
  3028. else
  3029. {
  3030. StringBuffer nodeStr;
  3031. node->endpoint().getUrlStr(nodeStr);
  3032. PROGLOG("Validating connection to %s", nodeStr.str());
  3033. if (!queryWorldCommunicator().verifyConnection(node, LOCKSESSCHECK))
  3034. {
  3035. StringBuffer out("Terminating connection session to ");
  3036. out.append(nodeStr);
  3037. out.append(" [");
  3038. out.appendf("%" I64F "x], connectionId [%" I64F "x]", lD->sessId, * ((ConnectionId *) imap.getKey()));
  3039. out.append(" xpath [").append(xpath).append("]");
  3040. PROGLOG("%s", out.str());
  3041. queryCoven().disconnect(node);
  3042. ret = true;
  3043. }
  3044. }
  3045. }
  3046. }
  3047. }
  3048. }
  3049. catch (IException *e)
  3050. {
  3051. EXCLOG(e, "validateConnectionSessions");
  3052. e->Release();
  3053. }
  3054. PROGLOG("validateConnectionSessions done");
  3055. return ret;
  3056. }
  3057. LockStatus doLock(unsigned mode, unsigned timeout, ConnectionId id, SessionId sessionId, IUnlockCallback &callback, bool change=false)
  3058. {
  3059. if (INFINITE == timeout)
  3060. {
  3061. loop
  3062. {
  3063. if (!SDSManager->queryConnection(id))
  3064. return LockFailed;
  3065. LockStatus lockStatus = tryLock(mode, id, sessionId, change);
  3066. if (lockStatus == LockSucceeded || lockStatus == LockHeld)
  3067. return lockStatus;
  3068. else
  3069. {
  3070. bool timedout = false;
  3071. waiting++;
  3072. {
  3073. CHECKEDCRITICALUNBLOCK(crit, fakeCritTimeout);
  3074. callback.unblock();
  3075. timedout = !sem.wait(LOCKSESSCHECK);
  3076. callback.block();
  3077. }
  3078. if (timedout)
  3079. {
  3080. if (!sem.wait(0))
  3081. {
  3082. waiting--;
  3083. StringBuffer s("Infinite timeout lock still waiting: ");
  3084. toString(s);
  3085. PROGLOG("%s", s.str());
  3086. }
  3087. {
  3088. CHECKEDCRITICALUNBLOCK(crit, fakeCritTimeout);
  3089. callback.unblock();
  3090. validateConnectionSessions();
  3091. callback.block();
  3092. }
  3093. }
  3094. }
  3095. }
  3096. }
  3097. else
  3098. {
  3099. CTimeMon tm(timeout);
  3100. loop
  3101. {
  3102. if (!SDSManager->queryConnection(id))
  3103. return LockFailed;
  3104. LockStatus lockStatus = tryLock(mode, id, sessionId, change);
  3105. if (lockStatus == LockSucceeded || lockStatus == LockHeld)
  3106. return lockStatus;
  3107. else
  3108. {
  3109. bool timedout = false;
  3110. waiting++;
  3111. {
  3112. CHECKEDCRITICALUNBLOCK(crit, fakeCritTimeout);
  3113. callback.unblock();
  3114. unsigned remaining;
  3115. if (tm.timedout(&remaining) || !sem.wait(remaining>LOCKSESSCHECK?LOCKSESSCHECK:remaining))
  3116. timedout = true;
  3117. callback.block();
  3118. }
  3119. if (timedout) {
  3120. if (!sem.wait(0))
  3121. waiting--; //// only dec waiting if waiting wasn't signalled.
  3122. bool disconnects;
  3123. {
  3124. CHECKEDCRITICALUNBLOCK(crit, fakeCritTimeout);
  3125. callback.unblock();
  3126. disconnects = validateConnectionSessions();
  3127. callback.block();
  3128. }
  3129. if (tm.timedout())
  3130. {
  3131. if (disconnects) // if some sessions disconnected, one final try
  3132. {
  3133. if (!SDSManager->queryConnection(id))
  3134. return LockFailed;
  3135. lockStatus = tryLock(mode, id, sessionId, change);
  3136. if (lockStatus == LockSucceeded || lockStatus == LockHeld)
  3137. return lockStatus;
  3138. }
  3139. break;
  3140. }
  3141. }
  3142. // have to very careful here, have regained crit locks but have since timed out
  3143. // therefore before gaining crits after signal (this lock was unlocked)
  3144. // other thread can grab lock at this time, but this thread can't cause others to increase 'waiting' at this time.
  3145. // and not after crit locks regained.
  3146. if (tm.timedout())
  3147. break;
  3148. }
  3149. }
  3150. }
  3151. return LockTimedOut;
  3152. }
  3153. LockStatus tryLock(unsigned mode, ConnectionId id, SessionId sessId, bool changingMode=false)
  3154. {
  3155. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  3156. LockData *existingLd = NULL;
  3157. bool hadReadLock = false;
  3158. bool hadHoldLock = false;
  3159. if (changingMode)
  3160. {
  3161. existingLd = connectionInfo.getValue(id);
  3162. if (existingLd)
  3163. {
  3164. if ((existingLd->mode & RTM_LOCKBASIC_MASK) == (mode & RTM_LOCKBASIC_MASK))
  3165. return LockSucceeded; // nothing to do
  3166. // record and unlock existing state
  3167. switch (existingLd->mode & RTM_LOCKBASIC_MASK)
  3168. {
  3169. case (RTM_LOCK_HOLD+RTM_LOCK_READ):
  3170. holdLocks--;
  3171. hadHoldLock = true;
  3172. // fall into...
  3173. case RTM_LOCK_READ:
  3174. readLocks--;
  3175. hadReadLock = true;
  3176. break;
  3177. case (RTM_LOCK_WRITE+RTM_LOCK_READ):
  3178. case RTM_LOCK_WRITE:
  3179. exclusive = false;
  3180. // change will succeed
  3181. break;
  3182. case 0: // no locking
  3183. break;
  3184. default:
  3185. throwUnexpected();
  3186. }
  3187. }
  3188. else
  3189. changingMode = false; // nothing to restore in event of no change
  3190. }
  3191. switch (mode & RTM_LOCKBASIC_MASK)
  3192. {
  3193. case 0:
  3194. {
  3195. if (changingMode)
  3196. break;
  3197. return LockSucceeded;
  3198. }
  3199. case (RTM_LOCK_READ+RTM_LOCK_HOLD):
  3200. case RTM_LOCK_READ: // cannot fail if changingMode=true (exclusive will have been unlocked)
  3201. if (exclusive)
  3202. return LockFailed;
  3203. readLocks++;
  3204. if (mode & RTM_LOCK_HOLD)
  3205. holdLocks++;
  3206. break;
  3207. case (RTM_LOCK_WRITE+RTM_LOCK_READ):
  3208. case RTM_LOCK_WRITE:
  3209. if (exclusive || readLocks || holdLocks)
  3210. {
  3211. if (changingMode)
  3212. {
  3213. // only an unlocked read lock can fail and need restore here.
  3214. if (hadReadLock)
  3215. readLocks++;
  3216. if (hadHoldLock)
  3217. holdLocks++;
  3218. }
  3219. return holdLocks ? LockHeld : LockFailed;
  3220. }
  3221. exclusive = true;
  3222. #ifdef _DEBUG
  3223. debugInfo.ExclOwningThread = GetCurrentThreadId();
  3224. debugInfo.ExclOwningConnection = id;
  3225. debugInfo.ExclOwningSession = sessId;
  3226. #endif
  3227. break;
  3228. default:
  3229. if (changingMode)
  3230. {
  3231. // only an unlocked read lock can fail and need restore here.
  3232. if (hadReadLock)
  3233. readLocks++;
  3234. if (hadHoldLock)
  3235. holdLocks++;
  3236. }
  3237. throwUnexpected();
  3238. }
  3239. if (changingMode)
  3240. {
  3241. existingLd->mode = mode;
  3242. wakeWaiting();
  3243. }
  3244. else
  3245. {
  3246. if (RTM_LOCK_SUB & mode)
  3247. sub++;
  3248. LockData ld(mode, sessId, msTick());
  3249. connectionInfo.setValue(id, ld);
  3250. }
  3251. return LockSucceeded;
  3252. }
  3253. inline void wakeWaiting()
  3254. {
  3255. if (waiting)
  3256. {
  3257. sem.signal(waiting); // get blocked locks to recheck.
  3258. waiting=0;
  3259. }
  3260. }
  3261. public:
  3262. IMPLEMENT_IINTERFACE;
  3263. CLock(CLockTable &_table, __int64 _treeId, IdPath &_idPath, const char *_xpath, unsigned mode, ConnectionId id, SessionId sessId)
  3264. : table(_table), treeId(_treeId), xpath(_xpath), exclusive(false), sub(0), readLocks(0), holdLocks(0), waiting(0), pending(0)
  3265. {
  3266. INIT_NAMEDCOUNT;
  3267. verifyex(tryLock(mode, id, sessId)==LockSucceeded);
  3268. ForEachItemIn(i, _idPath)
  3269. idPath.append(_idPath.item(i));
  3270. }
  3271. ~CLock()
  3272. {
  3273. if (parent)
  3274. clearLastRef();
  3275. }
  3276. inline void clearLastRef();
  3277. bool querySub() { return (0 != sub); }
  3278. const void *queryFindParam() const
  3279. {
  3280. return (const void *) &treeId;
  3281. }
  3282. bool matchHead(IdPath &_idPath)
  3283. {
  3284. unsigned o = idPath.ordinality();
  3285. ForEachItemIn(i, _idPath)
  3286. {
  3287. if (i>=o) return false;
  3288. else if (idPath.item(i) != _idPath.item(i))
  3289. return false;
  3290. }
  3291. return true;
  3292. }
  3293. bool unlock(ConnectionId id, bool delayDelete=false)
  3294. {
  3295. bool ret = false;
  3296. CPendingLockBlock b(*this); // carefully placed, removePending can destroy this, therefore must be destroyed last
  3297. {
  3298. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  3299. LockData *ld = connectionInfo.getValue(id);
  3300. if (ld) // not necessarily any lock info for this connection
  3301. {
  3302. switch (ld->mode & RTM_LOCKBASIC_MASK)
  3303. {
  3304. case RTM_LOCK_READ+RTM_LOCK_HOLD:
  3305. assertex(holdLocks);
  3306. holdLocks--;
  3307. // fall into...
  3308. case RTM_LOCK_READ:
  3309. assertex(readLocks);
  3310. readLocks--;
  3311. break;
  3312. case (RTM_LOCK_WRITE+RTM_LOCK_READ):
  3313. case RTM_LOCK_WRITE:
  3314. assertex(exclusive && 0 == readLocks && 0 == holdLocks);
  3315. exclusive = false;
  3316. #ifdef _DEBUG
  3317. debugInfo.clearExclusive();
  3318. #endif
  3319. break;
  3320. case 0: // no locking
  3321. break;
  3322. default:
  3323. throwUnexpected();
  3324. }
  3325. if (RTM_LOCK_SUB & ld->mode)
  3326. sub--;
  3327. connectionInfo.remove(id);
  3328. if (parent && 0 == connectionInfo.count())
  3329. {
  3330. if (delayDelete)
  3331. {
  3332. parent.clear();
  3333. child.clear();
  3334. }
  3335. else
  3336. clearLastRef();
  3337. ret = true;
  3338. }
  3339. }
  3340. wakeWaiting();
  3341. }
  3342. return ret;
  3343. }
  3344. void unlockAll()
  3345. {
  3346. CPendingLockBlock b(*this); // carefully placed, removePending can destroy this.
  3347. { CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  3348. HashIterator iter(connectionInfo);
  3349. while (iter.first())
  3350. {
  3351. IMapping &map = iter.query();
  3352. ConnectionId id = *(ConnectionId *)map.getKey();
  3353. unlock(id);
  3354. }
  3355. }
  3356. }
  3357. inline void addPending()
  3358. {
  3359. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  3360. pending++;
  3361. }
  3362. inline void removePending()
  3363. {
  3364. Linked<CLock> destroy;
  3365. {
  3366. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  3367. pending--;
  3368. if (0 == (lockCount()+pending))
  3369. {
  3370. destroy.set(this);
  3371. table.removeExact(this);
  3372. }
  3373. }
  3374. }
  3375. LockStatus lock(unsigned mode, unsigned timeout, ConnectionId id, SessionId sessionId, IUnlockCallback &callback)
  3376. {
  3377. bool ret = false;
  3378. CPendingLockBlock b(*this); // carefully placed, removePending can destroy this, therefore must be destroyed last
  3379. {
  3380. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  3381. return doLock(mode, timeout, id, sessionId, callback);
  3382. }
  3383. return LockFailed;
  3384. }
  3385. void changeMode(ConnectionId id, SessionId sessionId, unsigned newMode, unsigned timeout, IUnlockCallback &callback)
  3386. {
  3387. CPendingLockBlock b(*this); // carefully placed, removePending can destroy this.
  3388. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  3389. LockStatus result = doLock(newMode, timeout, id, sessionId, callback, true);
  3390. if (result != LockSucceeded)
  3391. {
  3392. StringBuffer s;
  3393. switch (result)
  3394. {
  3395. case LockFailed:
  3396. throw MakeSDSException(SDSExcpt_ConnectionAbsent, "Lost connection performing changeMode on connection to : %s", xpath.get());
  3397. case LockTimedOut:
  3398. throw MakeSDSException(SDSExcpt_LockTimeout, "Lock timeout performing changeMode on connection to : %s, existing lock info: %s", xpath.get(), toString(s).str());
  3399. case LockHeld:
  3400. throw MakeSDSException(SDSExcpt_LockHeld, "Lock is held performing changeMode on connection to : %s, existing lock info: %s", xpath.get(), toString(s).str());
  3401. }
  3402. }
  3403. }
  3404. unsigned lockCount()
  3405. {
  3406. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  3407. return connectionInfo.count();
  3408. }
  3409. bool associated(ConnectionId connectionId)
  3410. {
  3411. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  3412. return NULL!=connectionInfo.getValue(connectionId);
  3413. }
  3414. const char *queryXPath() const { return xpath; }
  3415. ILockInfo *getLockInfo() const
  3416. {
  3417. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  3418. return createLockInfo(xpath, connectionInfo); // NB: doesn't resolve sessionId to Endpoint string at this point
  3419. }
  3420. void setDROLR(CServerRemoteTree *_parent, CServerRemoteTree *_child)
  3421. {
  3422. CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
  3423. if (parent)
  3424. {
  3425. assertex(parent.get() == _parent);
  3426. assertex(child.get() == _child);
  3427. return;
  3428. }
  3429. parent.set(_parent);
  3430. child.set(_child);
  3431. }
  3432. StringBuffer &toString(StringBuffer &out) const
  3433. {
  3434. Owned<ILockInfo> lockInfo = getLockInfo();
  3435. return lockInfo->toString(out, 0, true);
  3436. }
  3437. };
  3438. CPendingLockBlock::CPendingLockBlock(CLock &_lock) : lock(_lock)
  3439. {
  3440. lock.addPending();
  3441. }
  3442. CPendingLockBlock::~CPendingLockBlock()
  3443. {
  3444. lock.removePending();
  3445. }
  3446. template <> void CLockTable::onRemove(void *et)
  3447. {
  3448. ((CLock*)et)->Release();
  3449. }
  3450. ///////////////
  3451. CSDSTransactionServer::CSDSTransactionServer(CCovenSDSManager &_manager)
  3452. : Thread("SDS Manager, CSDSTransactionServer"), manager(_manager), CTransactionLogTracker(DAMP_SDSCMD_MAX)
  3453. {
  3454. stopped = true;
  3455. }
  3456. int CSDSTransactionServer::run()
  3457. {
  3458. ICoven &coven=queryCoven();
  3459. CMessageHandler<CSDSTransactionServer> handler("CSDSTransactionServer",this,&CSDSTransactionServer::processMessage, &manager, 100, TIMEOUT_ON_CLOSEDOWN);
  3460. stopped = false;
  3461. CMessageBuffer mb;
  3462. while (!stopped)
  3463. {
  3464. try
  3465. {
  3466. #ifdef TRACE_QWAITING
  3467. unsigned waiting = coven.probe(RANK_ALL,MPTAG_DALI_SDS_REQUEST,NULL);
  3468. static unsigned lastwaiting = 0;
  3469. static unsigned lasttick = 0;
  3470. if ((waiting>lastwaiting+25)||(waiting<lastwaiting/2)||
  3471. ((waiting>100)&&(msTick()-lasttick>1000))) {
  3472. DBGLOG("QPROBE: MPTAG_DALI_SDS_REQUEST has %d waiting",waiting);
  3473. lastwaiting = waiting;
  3474. lasttick = msTick();
  3475. }
  3476. #endif
  3477. mb.clear();
  3478. if (coven.recv(mb, RANK_ALL, MPTAG_DALI_SDS_REQUEST, NULL))
  3479. {
  3480. msgCount++;
  3481. try
  3482. {
  3483. SdsCommand action;
  3484. mb.read((int &)action);
  3485. action = (SdsCommand) (((unsigned)action) & ~DAMP_SDSCMD_LAZYEXT);
  3486. switch (action)
  3487. {
  3488. case DAMP_SDSCMD_CONNECT:
  3489. case DAMP_SDSCMD_MCONNECT:
  3490. case DAMP_SDSCMD_GETCHILDREN:
  3491. case DAMP_SDSCMD_GETCHILDREN2:
  3492. case DAMP_SDSCMD_GET:
  3493. case DAMP_SDSCMD_GET2:
  3494. case DAMP_SDSCMD_GETELEMENTS:
  3495. case DAMP_SDSCMD_DATA:
  3496. case DAMP_SDSCMD_CHANGEMODE:
  3497. case DAMP_SDSCMD_GETXPATHS:
  3498. case DAMP_SDSCMD_GETXPATHSPLUSIDS:
  3499. case DAMP_SDSCMD_GETEXTVALUE:
  3500. case DAMP_SDSCMD_GETELEMENTSRAW:
  3501. case DAMP_SDSCMD_GETCOUNT:
  3502. {
  3503. mb.reset();
  3504. handler.handleMessage(mb);
  3505. mb.clear(); // ^ has copied mb
  3506. break;
  3507. }
  3508. case DAMP_SDSCMD_GETSTORE:
  3509. {
  3510. TimingBlock xactTimingBlock(xactTimingStats);
  3511. CServerRemoteTree *root = manager.queryRoot();
  3512. mb.clear();
  3513. if (root)
  3514. {
  3515. mb.append(DAMP_SDSREPLY_OK);
  3516. root->serializeCutOffRT(mb);
  3517. }
  3518. else
  3519. mb.append(DAMP_SDSREPLY_EMPTY);
  3520. break;
  3521. }
  3522. #ifdef LEGACY_CLIENT_RESPONSE
  3523. // give a re
  3524. case DAMP_SDSCMD_VERSION:
  3525. {
  3526. TimingBlock xactTimingBlock(xactTimingStats);
  3527. mb.clear().append(DAMP_SDSREPLY_ERROR);
  3528. throw MakeStringException(-1, "Client too old to communicate with this dali");
  3529. }
  3530. #endif
  3531. case DAMP_SDSCMD_DIAGNOSTIC:
  3532. {
  3533. TimingBlock xactTimingBlock(xactTimingStats);
  3534. SdsDiagCommand cmd;
  3535. mb.read((int &)cmd);
  3536. switch (cmd)
  3537. {
  3538. case DIAG_CMD_LOCKINFO:
  3539. {
  3540. StringAttr ipPattern, xpathPattern;
  3541. mb.read(ipPattern).read(xpathPattern);
  3542. mb.clear().append(DAMP_SDSREPLY_OK);
  3543. Owned<ILockInfoCollection> lockInfoCollection = SDSManager->getLocks(ipPattern, xpathPattern);
  3544. lockInfoCollection->serialize(mb);
  3545. break;
  3546. }
  3547. case DIAG_CMD_STATS:
  3548. {
  3549. mb.clear().append(DAMP_SDSREPLY_OK);
  3550. SDSManager->collectUsageStats(mb);
  3551. break;
  3552. }
  3553. case DIAG_CMD_CONNECTIONS:
  3554. {
  3555. mb.clear().append(DAMP_SDSREPLY_OK);
  3556. SDSManager->collectConnections(mb);
  3557. break;
  3558. }
  3559. case DIAG_CMD_SUBSCRIBERS:
  3560. {
  3561. mb.clear().append(DAMP_SDSREPLY_OK);
  3562. SDSManager->collectSubscribers(mb);
  3563. break;
  3564. }
  3565. default:
  3566. assertex(false);
  3567. }
  3568. break;
  3569. }
  3570. case DAMP_SDSCMD_GETPROPS:
  3571. {
  3572. mb.clear().append(DAMP_SDSREPLY_OK);
  3573. manager.queryProperties().serialize(mb);
  3574. break;
  3575. }
  3576. case DAMP_SDSCMD_UPDTENV:
  3577. {
  3578. Owned<IPropertyTree> newEnv = createPTree(mb);
  3579. bool forceGroupUpdate;
  3580. mb.read(forceGroupUpdate);
  3581. StringBuffer response;
  3582. bool result = manager.updateEnvironment(newEnv, forceGroupUpdate, response);
  3583. mb.clear().append(DAMP_SDSREPLY_OK).append(result).append(response);
  3584. break;
  3585. }
  3586. default:
  3587. throw MakeSDSException(SDSExcpt_UnrecognisedCommand, "%d", action);
  3588. }
  3589. }
  3590. catch (IException *e)
  3591. {
  3592. mb.clear();
  3593. mb.append((int) DAMP_SDSREPLY_ERROR);
  3594. mb.append(e->errorCode());
  3595. StringBuffer s;
  3596. mb.append(e->errorMessage(s).str());
  3597. StringBuffer clientUrl("EXCEPTION in reply to client ");
  3598. mb.getSender().getUrlStr(clientUrl);
  3599. EXCLOG(e, clientUrl.str(), MSGCLS_warning);
  3600. e->Release();
  3601. }
  3602. if (mb.length())
  3603. {
  3604. try { coven.reply(mb); }
  3605. catch (IJSOCK_Exception *e)
  3606. {
  3607. LOG(MCwarning, unknownJob, e, "Failed to reply to client (CSDSTransactionServer thread)");
  3608. e->Release();
  3609. }
  3610. catch (IMP_Exception *e)
  3611. {
  3612. LOG(MCwarning, unknownJob, e, "Failed to reply to client (CSDSTransactionServer thread)");
  3613. e->Release();
  3614. }
  3615. }
  3616. }
  3617. else
  3618. stopped = true;
  3619. }
  3620. catch (IException *e)
  3621. {
  3622. StringBuffer s("Failure receiving message from client ");
  3623. mb.getSender().getUrlStr(s);
  3624. LOG(MCwarning, unknownJob, e, s.str());
  3625. e->Release();
  3626. }
  3627. }
  3628. return 0;
  3629. }
  3630. // backward compat.
  3631. bool checkOldFormat(CServerRemoteTree *parentServerTree, IPropertyTree *tree, MemoryBuffer &mb)
  3632. {
  3633. CPState state;
  3634. mb.read((int &)state);
  3635. bool change = false;
  3636. if (state)
  3637. {
  3638. if (CPS_Renames & state)
  3639. {
  3640. loop
  3641. {
  3642. __int64 id;
  3643. mb.read(id);
  3644. if (0 == id)
  3645. break;
  3646. StringAttr newName;
  3647. mb.read(newName);
  3648. IPropertyTree *child = SDSManager->queryRegisteredTree(id);
  3649. if (child)
  3650. {
  3651. assertex(parentServerTree);
  3652. int pos = parentServerTree->findChild(child);
  3653. if (NotFound == pos)
  3654. throw MakeSDSException(SDSExcpt_ClientCacheDirty, "::checkChange - child(%s) not found in parent(%s) at %s(%d)", child->queryName(), parentServerTree->queryName(), __FILE__, __LINE__);
  3655. IPropertyTree *t = createPTree();
  3656. t->setProp("@from", child->queryName());
  3657. t->setProp("@to", newName);
  3658. t->setPropInt64("@id", id);
  3659. #ifdef SIBLING_MOVEMENT_CHECK
  3660. t->setProp("@pos", pos);
  3661. #endif
  3662. tree->addPropTree(RENAME_TAG, t);
  3663. change = true;
  3664. }
  3665. }
  3666. }
  3667. if (CPS_Deletions & state)
  3668. {
  3669. loop
  3670. {
  3671. __int64 id;
  3672. mb.read(id);
  3673. if (0 == id)
  3674. break;
  3675. IPropertyTree *child = SDSManager->queryRegisteredTree(id);
  3676. if (child)
  3677. {
  3678. assertex(parentServerTree);
  3679. int pos = parentServerTree->findChild(child);
  3680. if (NotFound == pos)
  3681. continue;
  3682. IPropertyTree *t = createPTree();
  3683. t->setProp("@name", child->queryName());
  3684. t->setPropInt64("@id", id);
  3685. #ifdef SIBLING_MOVEMENT_CHECK
  3686. t->setPropInt("@pos", pos+1);
  3687. #endif
  3688. tree->addPropTree(DELETE_TAG, t);
  3689. change = true;
  3690. }
  3691. }
  3692. }
  3693. if (CPS_AttrDeletions & state)
  3694. {
  3695. unsigned count, c;
  3696. mb.read(count);
  3697. if (count)
  3698. {
  3699. IPropertyTree *ct = tree->queryPropTree(ATTRCHANGE_TAG);
  3700. IPropertyTree *t = tree->addPropTree(ATTRDELETE_TAG, createPTree());
  3701. for (c=0; c<count; c++)
  3702. {
  3703. StringAttr attr;
  3704. mb.read(attr);
  3705. if (ct) ct->removeProp(attr);
  3706. t->addProp(attr, "");
  3707. }
  3708. change = true;
  3709. }
  3710. }
  3711. if (CPS_Changed & state)
  3712. {
  3713. Owned<PTree> clientTree = new LocalPTree();
  3714. clientTree->deserializeSelf(mb);
  3715. __int64 serverId;
  3716. mb.read(serverId);
  3717. byte STIInfo;
  3718. mb.read(STIInfo);
  3719. tree->setPropBool("@localValue", true);
  3720. if (clientTree->queryValue())
  3721. {
  3722. bool binary = clientTree->isBinary(NULL);
  3723. IPTArrayValue *v = ((PTree *)clientTree)->detachValue();
  3724. ((PTree *)tree)->setValue(v, binary);
  3725. }
  3726. else
  3727. ((PTree *)tree)->setValue(new CPTValue(0, NULL, false, true, false), false);
  3728. Owned<IAttributeIterator> attrs = clientTree->getAttributes();
  3729. IPropertyTree *t = createPTree();
  3730. if (attrs->first())
  3731. {
  3732. do
  3733. {
  3734. t->setProp(attrs->queryName(), clientTree->queryProp(attrs->queryName()));
  3735. }
  3736. while (attrs->next());
  3737. tree->addPropTree(ATTRCHANGE_TAG, t);
  3738. }
  3739. change = true;
  3740. }
  3741. }
  3742. return change;
  3743. }
  3744. bool translateOldFormat(CServerRemoteTree *parentServerTree, IPropertyTree *parentTree, MemoryBuffer &mb)
  3745. {
  3746. bool change = checkOldFormat(parentServerTree, parentTree, mb);
  3747. bool hasChildren;
  3748. mb.read(hasChildren);
  3749. if (hasChildren)
  3750. {
  3751. loop
  3752. {
  3753. __int64 id;
  3754. int pos = -1;
  3755. mb.read(id);
  3756. if (NoMoreChildrenMarker == id)
  3757. break;
  3758. mb.read(pos);
  3759. CServerRemoteTree *serverTree = NULL;
  3760. Owned<IPropertyTree> tree = createPTree(RESERVED_CHANGE_NODE);
  3761. if (0 == id)
  3762. {
  3763. StringAttr childName;
  3764. mb.read(childName);
  3765. tree->setPropBool("@new", true);
  3766. tree->setProp("@name", childName);
  3767. if (-1 != pos)
  3768. tree->setPropInt("@pos", pos+1);
  3769. }
  3770. else
  3771. {
  3772. assertex(parentServerTree);
  3773. serverTree = (CServerRemoteTree *) SDSManager->queryRegisteredTree(id);
  3774. assertex(serverTree);
  3775. pos = parentServerTree->findChild(serverTree);
  3776. if (NotFound == pos)
  3777. throw MakeSDSException(SDSExcpt_ClientCacheDirty, "child(%s) not found in parent(%s) at %s(%d)", serverTree->queryName(), parentServerTree->queryName(), __FILE__, __LINE__);
  3778. tree->setProp("@name", serverTree->queryName());
  3779. tree->setPropInt64("@id", id);
  3780. tree->setPropInt("@pos", pos+1);
  3781. }
  3782. if (translateOldFormat(serverTree, tree, mb))
  3783. {
  3784. parentTree->addPropTree(tree->queryName(), LINK(tree));
  3785. change = true;
  3786. }
  3787. }
  3788. }
  3789. return change;
  3790. }
  3791. ///
  3792. void CSDSTransactionServer::processMessage(CMessageBuffer &mb)
  3793. {
  3794. TimingBlock xactTimingBlock(xactTimingStats);
  3795. ICoven &coven = queryCoven();
  3796. StringAttr xpath;
  3797. ConnectionId connectionId;
  3798. SessionId id;
  3799. unsigned mode;
  3800. unsigned timeout;
  3801. SdsCommand action = (SdsCommand)-1;
  3802. try
  3803. {
  3804. mb.read((int &)action);
  3805. bool getExt = 0 == (action & DAMP_SDSCMD_LAZYEXT);
  3806. action = (SdsCommand) (((unsigned)action) & ~DAMP_SDSCMD_LAZYEXT);
  3807. TransactionLog transactionLog(*this, action, mb.getSender()); // only active if queryTransactionLogging()==true
  3808. switch (action)
  3809. {
  3810. case DAMP_SDSCMD_CONNECT:
  3811. {
  3812. TimingBlock connectTimingBlock(connectTimingStats);
  3813. Owned<CLCLockBlock> lockBlock;
  3814. unsigned startPos = mb.getPos();
  3815. mb.read(id);
  3816. mb.read(mode);
  3817. mb.read(timeout);
  3818. mb.read(xpath);
  3819. if (queryTransactionLogging())
  3820. transactionLog.log("xpath='%s' mode=%d", xpath.get(), (unsigned)mode);
  3821. Owned<LinkingCriticalBlock> connectCritBlock = new LinkingCriticalBlock(manager.connectCrit, __FILE__, __LINE__);
  3822. if (RTM_CREATE == (mode & RTM_CREATE_MASK) || RTM_CREATE_QUERY == (mode & RTM_CREATE_MASK))
  3823. lockBlock.setown(new CLCWriteLockBlock(manager.dataRWLock, readWriteTimeout, __FILE__, __LINE__));
  3824. else
  3825. lockBlock.setown(new CLCReadLockBlock(manager.dataRWLock, readWriteTimeout, __FILE__, __LINE__));
  3826. if (queryTransactionLogging())
  3827. transactionLog.markExtra();
  3828. connectionId = 0;
  3829. CServerRemoteTree *_tree;
  3830. Owned<CServerRemoteTree> tree;
  3831. manager.createConnection(id, mode, timeout, xpath, _tree, connectionId, true, connectCritBlock);
  3832. if (connectionId)
  3833. tree.setown(_tree);
  3834. connectCritBlock.clear();
  3835. if (connectionId)
  3836. {
  3837. if (0 == id)
  3838. {
  3839. StringBuffer str("Dali client passing sessionid=0 to connect (xpath=");
  3840. str.append(xpath).append(", mode=").append(mode).append(", connectionId=").appendf("%" I64F "x", connectionId).append(")");
  3841. WARNLOG("%s", str.str());
  3842. }
  3843. mb.clear();
  3844. mb.append((int)DAMP_SDSREPLY_OK);
  3845. mb.append(connectionId);
  3846. tree->serializeCutOffRT(mb, RTM_SUB & mode?FETCH_ENTIRE:tree->getPropBool("@fetchEntire")?FETCH_ENTIRE_COND : 0, 0, getExt);
  3847. }
  3848. else
  3849. {
  3850. mb.clear();
  3851. mb.append((int)DAMP_SDSREPLY_EMPTY);
  3852. }
  3853. break;
  3854. }
  3855. case DAMP_SDSCMD_MCONNECT:
  3856. {
  3857. TimingBlock connectTimingBlock(connectTimingStats);
  3858. Owned<CLCLockBlock> lockBlock;
  3859. if (queryTransactionLogging())
  3860. transactionLog.log();
  3861. unsigned startPos = mb.getPos();
  3862. mb.read(id);
  3863. mb.read(timeout);
  3864. Owned<IMultipleConnector> mConnect = deserializeIMultipleConnector(mb);
  3865. mb.clear();
  3866. lockBlock.setown(new CLCReadLockBlock(manager.dataRWLock, readWriteTimeout, __FILE__, __LINE__));
  3867. try
  3868. {
  3869. Owned<CRemoteConnections> remoteConnections = new CRemoteConnections;
  3870. unsigned c;
  3871. for (c=0; c<mConnect->queryConnections(); c++)
  3872. {
  3873. StringAttr xpath;
  3874. unsigned mode;
  3875. mConnect->getConnectionDetails(c, xpath, mode);
  3876. if (queryTransactionLogging())
  3877. transactionLog.extra(", xpath='%s', mode=%d", xpath.get(), mode);
  3878. connectionId = 0;
  3879. CServerRemoteTree *_tree;
  3880. Owned<CServerRemoteTree> tree;
  3881. Owned<LinkingCriticalBlock> connectCritBlock = new LinkingCriticalBlock(manager.connectCrit, __FILE__, __LINE__);
  3882. manager.createConnection(id, mode, timeout, xpath, _tree, connectionId, true, connectCritBlock);
  3883. if (connectionId)
  3884. tree.setown(_tree);
  3885. connectCritBlock.clear();
  3886. if (connectionId)
  3887. {
  3888. if (0 == id)
  3889. {
  3890. StringBuffer str("Dali client passing sessionid=0 to multi connect (xpath=");
  3891. str.append(xpath).append(", mode=").append(mode).append(", connectionId=").appendf("%" I64F "x", connectionId).append(")");
  3892. WARNLOG("%s", str.str());
  3893. }
  3894. CRemoteConnection *conn = new CRemoteConnection(*SDSManager, connectionId, xpath, id, mode, timeout);
  3895. assertex(conn);
  3896. remoteConnections->add(conn);
  3897. mb.append((int)DAMP_SDSREPLY_OK);
  3898. mb.append(connectionId);
  3899. tree->serializeCutOffRT(mb, RTM_SUB & mode?FETCH_ENTIRE:tree->getPropBool("@fetchEntire")?FETCH_ENTIRE_COND : 0, 0, getExt);
  3900. }
  3901. else
  3902. {
  3903. mb.append((int)DAMP_SDSREPLY_EMPTY);
  3904. }
  3905. }
  3906. // success detach establish connections from holder (which would otherwise disconnect them)
  3907. remoteConnections->detachConnections();
  3908. }
  3909. catch (IException *e)
  3910. {
  3911. StringBuffer s("Failed to establish locks to multiple paths: ");
  3912. getMConnectString(mConnect, s);
  3913. LOG(MCwarning, unknownJob, e, s.str());
  3914. throw;
  3915. }
  3916. catch (DALI_CATCHALL)
  3917. {
  3918. StringBuffer s("(Unknown exception); Failed to establish locks to multiple paths: ");
  3919. getMConnectString(mConnect, s);
  3920. throw;
  3921. }
  3922. break;
  3923. }
  3924. case DAMP_SDSCMD_GET:
  3925. case DAMP_SDSCMD_GET2:
  3926. {
  3927. mb.read(connectionId);
  3928. if (queryTransactionLogging())
  3929. transactionLog.log();
  3930. __int64 serverId;
  3931. mb.read(serverId);
  3932. CHECKEDDALIREADLOCKBLOCK(manager.dataRWLock, readWriteTimeout);
  3933. CHECKEDCRITICALBLOCK(SDSManager->treeRegCrit, fakeCritTimeout);
  3934. Owned<CServerRemoteTree> tree = manager.getRegisteredTree(serverId);
  3935. if (queryTransactionLogging())
  3936. {
  3937. CServerConnection *conn = manager.queryConnection(connectionId);
  3938. transactionLog.extra(", xpath='%s', node=%s", conn?conn->queryXPath():"???", tree?tree->queryName():"???");
  3939. }
  3940. mb.clear();
  3941. if (!tree)
  3942. {
  3943. if (DAMP_SDSCMD_GET2 == action)
  3944. mb.append((int)DAMP_SDSREPLY_EMPTY);
  3945. else
  3946. {
  3947. CServerConnection *connection = manager.queryConnection(connectionId);
  3948. StringBuffer s;
  3949. if (connection)
  3950. {
  3951. s.append("path=").append(connection->queryXPath());
  3952. s.append(", mode=").append(connection->queryMode());
  3953. }
  3954. else
  3955. s.append("Missing connection!");
  3956. throw MakeSDSException(SDSExcpt_UnknownTreeId, "get: treeId = (%d), connection={ %s }", (unsigned)serverId, s.str());
  3957. }
  3958. }
  3959. else
  3960. {
  3961. mb.append((int)DAMP_SDSREPLY_OK);
  3962. tree->serializeCutOffRT(mb, tree->getPropBool("@fetchEntire")?-1 : 0, 0, getExt);
  3963. }
  3964. break;
  3965. }
  3966. case DAMP_SDSCMD_GETCHILDREN:
  3967. case DAMP_SDSCMD_GETCHILDREN2:
  3968. {
  3969. mb.read(connectionId);
  3970. if (queryTransactionLogging())
  3971. {
  3972. CServerConnection *conn = manager.queryConnection(connectionId);
  3973. transactionLog.log("%s",conn?conn->queryXPath():"???");
  3974. }
  3975. __int64 serverId;
  3976. CHECKEDDALIREADLOCKBLOCK(manager.dataRWLock, readWriteTimeout);
  3977. CHECKEDCRITICALBLOCK(SDSManager->treeRegCrit, fakeCritTimeout);
  3978. CMessageBuffer replyMb;
  3979. replyMb.init(mb.getSender(), mb.getTag(), mb.getReplyTag());
  3980. replyMb.append((int)DAMP_SDSREPLY_OK);
  3981. bool first = true, empty = false;
  3982. loop
  3983. {
  3984. mb.read(serverId);
  3985. if (!serverId) break;
  3986. if (!first && empty) replyMb.clear();
  3987. unsigned levels;
  3988. mb.read(levels);
  3989. Owned<CServerRemoteTree> parent = manager.getRegisteredTree(serverId);
  3990. if (!parent)
  3991. {
  3992. if (DAMP_SDSCMD_GETCHILDREN2 == action)
  3993. replyMb.append(false);
  3994. else
  3995. {
  3996. if (first) // if only one, can acheive without serialization change.
  3997. {
  3998. empty = true;
  3999. replyMb.clear().append((int)DAMP_SDSREPLY_EMPTY);
  4000. }
  4001. else
  4002. {
  4003. CServerConnection *connection = manager.queryConnection(connectionId);
  4004. StringBuffer s;
  4005. if (connection)
  4006. {
  4007. s.append("path=").append(connection->queryXPath());
  4008. s.append(", mode=").append(connection->queryMode());
  4009. }
  4010. else
  4011. s.append("Missing connection!");
  4012. throw MakeSDSException(SDSExcpt_UnknownTreeId, "GETCHILDREN: Failed to locate parent (%d), connection={ %s }", (unsigned)serverId, s.str());
  4013. }
  4014. }
  4015. }
  4016. else
  4017. {
  4018. if (DAMP_SDSCMD_GETCHILDREN2 == action)
  4019. replyMb.append(true);
  4020. parent->serializeCutOffChildrenRT(replyMb, 0==levels ? (unsigned)-1 : levels, 0, getExt);
  4021. if (queryTransactionLogging())
  4022. transactionLog.extra(", node=%s",parent->queryName());
  4023. }
  4024. first = false;
  4025. }
  4026. mb.clear();
  4027. mb.transferFrom(replyMb);
  4028. break;
  4029. }
  4030. case DAMP_SDSCMD_GETELEMENTS:
  4031. {
  4032. mb.read(connectionId);
  4033. if (queryTransactionLogging())
  4034. {
  4035. CServerConnection *conn = manager.queryConnection(connectionId);
  4036. transactionLog.log("%s",conn?conn->queryXPath():"???");
  4037. }
  4038. CHECKEDDALIREADLOCKBLOCK(manager.dataRWLock, readWriteTimeout);
  4039. CHECKEDCRITICALBLOCK(SDSManager->treeRegCrit, fakeCritTimeout);
  4040. CServerConnection *connection = manager.queryConnection(connectionId);
  4041. if (!connection)
  4042. throw MakeSDSException(SDSExcpt_ConnectionAbsent, " [getElements]");
  4043. StringAttr xpath;
  4044. mb.read(xpath);
  4045. if (queryTransactionLogging())
  4046. transactionLog.extra(", xpath='%s'", xpath.get());
  4047. Owned<IPropertyTreeIterator> iter = connection->queryRoot()->getElements(xpath);
  4048. ICopyArrayOf<CServerRemoteTree> arr;
  4049. ForEach (*iter) arr.append((CServerRemoteTree &)iter->query());
  4050. CMessageBuffer replyMb;
  4051. replyMb.init(mb.getSender(), mb.getTag(), mb.getReplyTag());
  4052. replyMb.append((int)DAMP_SDSREPLY_OK);
  4053. replyMb.append(arr.ordinality());
  4054. ForEachItemIn(i, arr)
  4055. arr.item(i).serializeSelfRT(replyMb, getExt);
  4056. mb.clear();
  4057. mb.transferFrom(replyMb);
  4058. break;
  4059. }
  4060. case DAMP_SDSCMD_DATA:
  4061. {
  4062. TimingSizeBlock commitTimingBlock(commitTimingStats);
  4063. CheckTime block0("DAMP_SDSCMD_DATA total");
  4064. unsigned inputStart = mb.getPos();
  4065. mb.read(connectionId);
  4066. byte disconnect; // kludge, high bit to indicate new client format. (for backward compat.)
  4067. bool deleteRoot;
  4068. mb.read(disconnect);
  4069. bool oldFormat = (0 == (0x80 & disconnect));
  4070. disconnect &= ~0x80;
  4071. if (1 == disconnect)
  4072. mb.read(deleteRoot);
  4073. bool data = mb.length() != mb.getPos();
  4074. if (queryTransactionLogging())
  4075. {
  4076. CServerConnection *conn = manager.queryConnection(connectionId);
  4077. transactionLog.log("disconnect=%s, data=%s, deleteRoot=%s", disconnect?"true":"false", data?"true":"false", deleteRoot?"true":"false");
  4078. }
  4079. Owned<CLCLockBlock> lockBlock;
  4080. {
  4081. CheckTime block1("DAMP_SDSCMD_DATA.1");
  4082. if (data || deleteRoot)
  4083. lockBlock.setown(new CLCWriteLockBlock(manager.dataRWLock, readWriteTimeout, __FILE__, __LINE__));
  4084. else
  4085. lockBlock.setown(new CLCReadLockBlock(manager.dataRWLock, readWriteTimeout, __FILE__, __LINE__));
  4086. }
  4087. unsigned dataStart = mb.getPos();
  4088. commitTimingBlock.recordSize(mb.length() - dataStart);
  4089. CServerConnection *connection = manager.queryConnection(connectionId);
  4090. if (!connection)
  4091. throw MakeSDSException(SDSExcpt_ConnectionAbsent, " [commit]");
  4092. try
  4093. {
  4094. if (queryTransactionLogging())
  4095. transactionLog.extra(", xpath='%s'", connection->queryXPath());
  4096. CServerRemoteTree *tree = data ? (CServerRemoteTree *)connection->queryRoot() : (CServerRemoteTree *)connection->queryRootUnvalidated();
  4097. MemoryBuffer newIds;
  4098. Owned<IPropertyTree> changeTree;
  4099. if (data)
  4100. {
  4101. if (oldFormat)
  4102. {
  4103. Owned<IPropertyTree> t = createPTree(RESERVED_CHANGE_NODE);
  4104. t->setProp("@name", tree->queryName());
  4105. if (translateOldFormat(tree, t, mb))
  4106. changeTree.setown(LINK(t));
  4107. }
  4108. else
  4109. changeTree.setown(createPTree(mb));
  4110. }
  4111. if (changeTree && tree->processData(*connection, *changeTree, newIds))
  4112. { // something commited, if RTM_Create was used need to remember this.
  4113. CheckTime block6("DAMP_SDSCMD_DATA.6");
  4114. StringBuffer path;
  4115. connection->queryPTreePath().getAbsolutePath(path);
  4116. manager.saveDelta(path.str(), *changeTree);
  4117. }
  4118. mb.clear();
  4119. mb.append((int)DAMP_SDSREPLY_OK);
  4120. mb.append(newIds); // JCSMORE not particularly efficient change later
  4121. if (block0.slow())
  4122. {
  4123. block0.appendMsg(", xpath=").append(connection->queryXPath());
  4124. block0.appendMsg(", block size = ").append(mb.length());
  4125. }
  4126. }
  4127. catch (IException *)
  4128. {
  4129. if (disconnect)
  4130. manager.disconnect(connectionId, deleteRoot, (data || deleteRoot)?NULL:&lockBlock);
  4131. throw;
  4132. }
  4133. if (disconnect)
  4134. manager.disconnect(connectionId, deleteRoot, (data || deleteRoot)?NULL:&lockBlock);
  4135. break;
  4136. }
  4137. case DAMP_SDSCMD_CHANGEMODE:
  4138. {
  4139. mb.read(connectionId);
  4140. if (queryTransactionLogging())
  4141. transactionLog.log();
  4142. CHECKEDDALIWRITELOCKBLOCK(manager.dataRWLock, readWriteTimeout);
  4143. Linked<CServerConnection> connection = manager.queryConnection(connectionId);
  4144. if (!connection)
  4145. throw MakeSDSException(SDSExcpt_ConnectionAbsent, " [changeMode]");
  4146. CServerRemoteTree *tree = (CServerRemoteTree *) connection->queryRoot();
  4147. assertex(tree);
  4148. if (queryTransactionLogging())
  4149. transactionLog.extra(", xpath='%s'", connection->queryXPath());
  4150. unsigned newMode;
  4151. unsigned timeout;
  4152. mb.read(newMode);
  4153. mb.read(timeout);
  4154. mb.clear();
  4155. manager.changeLockMode(*connection, newMode, timeout);
  4156. if (!manager.queryConnection(connectionId))
  4157. {
  4158. manager.unlock(tree->queryServerId(), connectionId);
  4159. throw MakeSDSException(SDSExcpt_AbortDuringConnection, " during changeMode");
  4160. }
  4161. mb.append((int) DAMP_SDSREPLY_OK);
  4162. break;
  4163. }
  4164. case DAMP_SDSCMD_GETXPATHS:
  4165. case DAMP_SDSCMD_GETXPATHSPLUSIDS:
  4166. {
  4167. __int64 serverId;
  4168. mb.read(serverId);
  4169. mb.read(xpath);
  4170. if (queryTransactionLogging())
  4171. transactionLog.log("xpath='%s'", xpath.get());
  4172. mb.clear();
  4173. CHECKEDDALIREADLOCKBLOCK(manager.dataRWLock, readWriteTimeout);
  4174. Owned<IPropertyTree> matchTree = SDSManager->getXPaths(serverId, xpath, DAMP_SDSCMD_GETXPATHSPLUSIDS==action);
  4175. if (matchTree)
  4176. {
  4177. mb.append((int) DAMP_SDSREPLY_OK);
  4178. matchTree->serialize(mb);
  4179. }
  4180. else
  4181. mb.append((int) DAMP_SDSREPLY_EMPTY);
  4182. break;
  4183. }
  4184. case DAMP_SDSCMD_GETXPATHSCRITERIA:
  4185. {
  4186. StringAttr matchXPath, sortBy;
  4187. bool caseinsensitive, ascending;
  4188. unsigned from, limit;
  4189. mb.read(xpath);
  4190. mb.read(matchXPath);
  4191. mb.read(sortBy);
  4192. mb.read(caseinsensitive);
  4193. mb.read(ascending);
  4194. mb.read(from);
  4195. mb.read(limit);
  4196. if (queryTransactionLogging())
  4197. {
  4198. transactionLog.log("xpath='%s',matchXPath='%s',sortBy='%s',acscending=%s,from=%d,limit=%d",
  4199. xpath.get(), matchXPath.get(), sortBy.get(),
  4200. ascending?"true":"false", from, limit);
  4201. }
  4202. mb.clear();
  4203. CHECKEDDALIREADLOCKBLOCK(manager.dataRWLock, readWriteTimeout);
  4204. Owned<IPropertyTree> matchTree = SDSManager->getXPathsSortLimitMatchTree(xpath, matchXPath, sortBy, caseinsensitive, ascending, from, limit);
  4205. if (matchTree)
  4206. {
  4207. mb.append((int) DAMP_SDSREPLY_OK);
  4208. matchTree->serialize(mb);
  4209. }
  4210. else
  4211. mb.append((int) DAMP_SDSREPLY_EMPTY);
  4212. break;
  4213. }
  4214. case DAMP_SDSCMD_GETEXTVALUE:
  4215. {
  4216. __int64 serverId;
  4217. mb.read(serverId);
  4218. mb.clear().append((int) DAMP_SDSREPLY_OK);
  4219. if (queryTransactionLogging())
  4220. {
  4221. Owned<CServerRemoteTree> idTree = (CServerRemoteTree *) SDSManager->getRegisteredTree(serverId);
  4222. transactionLog.log("%s", idTree?idTree->queryName():"???");
  4223. }
  4224. SDSManager->getExternalValueFromServerId(serverId, mb);
  4225. break;
  4226. }
  4227. case DAMP_SDSCMD_GETELEMENTSRAW:
  4228. {
  4229. CHECKEDDALIREADLOCKBLOCK(manager.dataRWLock, readWriteTimeout);
  4230. StringAttr _xpath;
  4231. mb.read(_xpath);
  4232. if (queryTransactionLogging())
  4233. transactionLog.log("%s", xpath.get());
  4234. CMessageBuffer replyMb;
  4235. replyMb.init(mb.getSender(), mb.getTag(), mb.getReplyTag());
  4236. replyMb.append((int)DAMP_SDSREPLY_OK);
  4237. unsigned pos = replyMb.length();
  4238. unsigned count = 0;
  4239. replyMb.append(count);
  4240. const char *xpath = _xpath.get();
  4241. if ('/' == *xpath) ++xpath;
  4242. Owned<IPropertyTreeIterator> iter = manager.queryRoot()->getElements(xpath);
  4243. ForEach (*iter)
  4244. {
  4245. ++count;
  4246. IPropertyTree &e = iter->query();
  4247. e.serialize(replyMb);
  4248. }
  4249. replyMb.writeDirect(pos,sizeof(count),&count);
  4250. mb.clear();
  4251. mb.transferFrom(replyMb);
  4252. break;
  4253. }
  4254. case DAMP_SDSCMD_GETCOUNT:
  4255. {
  4256. mb.read(xpath);
  4257. if (queryTransactionLogging())
  4258. transactionLog.log("xpath='%s'", xpath.get());
  4259. CHECKEDDALIREADLOCKBLOCK(manager.dataRWLock, readWriteTimeout);
  4260. mb.clear();
  4261. mb.append((int)DAMP_SDSREPLY_OK);
  4262. mb.append(manager.queryCount(xpath));
  4263. break;
  4264. }
  4265. default:
  4266. throwUnexpected();
  4267. }
  4268. }
  4269. catch (IException *e)
  4270. {
  4271. mb.clear();
  4272. mb.append((int) DAMP_SDSREPLY_ERROR);
  4273. StringBuffer s;
  4274. e->errorMessage(s);
  4275. // NB: wanted to do this in a catch (IPTreeException *) block, but did catch,
  4276. // in spite of being able to query with dynamic cast
  4277. // something to do with rethrow, if I changed to catch early as IPT then it would catch here correctly.
  4278. if (QUERYINTERFACE(e, IPTreeException))
  4279. {
  4280. s.append(" in xpath '").append(xpath).append("'");
  4281. e->Release();
  4282. e = MakeSDSException(SDSExcpt_IPTError, "%s", s.str());
  4283. }
  4284. mb.append(e->errorCode());
  4285. mb.append(e->errorMessage(s.clear()));
  4286. StringBuffer clientUrl("EXCEPTION in reply to client ");
  4287. mb.getSender().getUrlStr(clientUrl);
  4288. EXCLOG(e, clientUrl.str(), MSGCLS_warning);
  4289. e->Release();
  4290. }
  4291. catch (DALI_CATCHALL)
  4292. {
  4293. Owned<IException> e = MakeSDSException(-1, "Unknown server exception processing client action: %d", action);
  4294. mb.clear();
  4295. mb.append((int) DAMP_SDSREPLY_ERROR);
  4296. StringBuffer s;
  4297. mb.append(e->errorCode());
  4298. mb.append(e->errorMessage(s).str());
  4299. StringBuffer clientUrl("EXCEPTION in reply to client ");
  4300. mb.getSender().getUrlStr(clientUrl);
  4301. LOG(MCoperatorError, unknownJob, e);
  4302. }
  4303. try {
  4304. CheckTime block10("DAMP_REQUEST reply");
  4305. coven.reply(mb);
  4306. }
  4307. catch (IMP_Exception *e)
  4308. {
  4309. LOG(MCwarning, unknownJob, e, "Failed to reply to client (processMessage)");
  4310. e->Release();
  4311. }
  4312. catch (IException *e)
  4313. {
  4314. // attempt error reply, on failed initial reply, reply error *might* have been message sensitive, e.g. OOM.
  4315. mb.clear();
  4316. mb.append((int) DAMP_SDSREPLY_ERROR);
  4317. mb.append(e->errorCode());
  4318. StringBuffer s;
  4319. mb.append(e->errorMessage(s).str());
  4320. StringBuffer clientUrl("EXCEPTION in reply to client ");
  4321. mb.getSender().getUrlStr(clientUrl);
  4322. EXCLOG(e, clientUrl.str(), MSGCLS_warning);
  4323. e->Release();
  4324. try
  4325. {
  4326. coven.reply(mb);
  4327. LOG(MCdebugInfo(100), unknownJob, "Failed to reply, but succeeded sending initial reply error to client");
  4328. }
  4329. catch (IException *e)
  4330. {
  4331. LOG(MCwarning, unknownJob, e, "Failed to reply and failed to send reply error to client");
  4332. e->Release();
  4333. }
  4334. }
  4335. }
  4336. void CSDSTransactionServer::stop()
  4337. {
  4338. if (!stopped) {
  4339. stopped = true;
  4340. queryCoven().cancel(RANK_ALL, MPTAG_DALI_SDS_REQUEST);
  4341. }
  4342. PROGLOG("clearing remaining sds locks");
  4343. manager.clearSDSLocks();
  4344. PROGLOG("waiting for transaction server to stop");
  4345. join();
  4346. }
  4347. /////////////////
  4348. // CServerConnection impl.
  4349. IPropertyTree *CServerConnection::queryRoot()
  4350. {
  4351. if (((CServerRemoteTree *)root.get())->isOrphaned())
  4352. throw MakeSDSException(SDSExcpt_OrphanedNode, "%s", queryXPath());
  4353. return root;
  4354. }
  4355. ////////////////
  4356. void CLock::clearLastRef()
  4357. {
  4358. if (parent)
  4359. {
  4360. CPendingLockBlock b(*this); // carefully placed, removePending can destroy this.
  4361. parent->removeTree(child);
  4362. parent.clear();
  4363. child.clear();
  4364. }
  4365. }
  4366. ////////////////
  4367. class ConnectionIdHashTable : public SuperHashTableOf<ConnectionId, ConnectionId>
  4368. {
  4369. public:
  4370. ~ConnectionIdHashTable() { _releaseAll(); }
  4371. IMPLEMENT_SUPERHASHTABLEOF_REF_FIND(ConnectionId, ConnectionId);
  4372. virtual void onAdd(void *et) { }
  4373. virtual void onRemove(void *et) { delete (ConnectionId *)et; }
  4374. virtual unsigned getHashFromElement(const void *et) const
  4375. {
  4376. return hashc((const unsigned char *) et, sizeof(ConnectionId), 0);
  4377. }
  4378. virtual unsigned getHashFromFindParam(const void *fp) const
  4379. {
  4380. return hashc((const unsigned char *) fp, sizeof(ConnectionId), 0);
  4381. }
  4382. virtual const void *getFindParam(const void *et) const
  4383. {
  4384. return et;
  4385. }
  4386. virtual bool matchesFindParam(const void *et, const void *fp, unsigned) const
  4387. {
  4388. return *(ConnectionId *)et == *(ConnectionId *)fp;
  4389. }
  4390. };
  4391. static bool retryRename(const char *from, const char *to, unsigned maxAttempts, unsigned delay)
  4392. {
  4393. unsigned attempts=maxAttempts;
  4394. loop
  4395. {
  4396. OwnedIFile iFile = createIFile(from);
  4397. try
  4398. {
  4399. iFile->rename(to);
  4400. break;
  4401. }
  4402. catch (IException *e)
  4403. {
  4404. StringBuffer errTxt("Failed to rename: ");
  4405. EXCLOG(e, errTxt.append(from).append(" to ").append(to).append(", retrying...").str());
  4406. e->Release();
  4407. }
  4408. if (attempts && 0 == --attempts)
  4409. break;
  4410. MilliSleep(delay);
  4411. }
  4412. return (attempts>0);
  4413. }
  4414. #ifdef NODELETE
  4415. static bool retryCopy(const char *from, const char *to, unsigned maxAttempts, unsigned delay)
  4416. {
  4417. unsigned attempts=maxAttempts;
  4418. loop
  4419. {
  4420. StringBuffer _from;
  4421. StringBuffer fname;
  4422. splitFilename(from, &_from, &_from, &fname, &fname);
  4423. _from.append('_').append(fname);
  4424. OwnedIFile iFile = createIFile(from);
  4425. try
  4426. {
  4427. iFile->rename(_from.str());
  4428. copyFile(to, _from.str());
  4429. break;
  4430. }
  4431. catch (IException *e)
  4432. {
  4433. EXCLOG(e, NULL);
  4434. e->Release();
  4435. }
  4436. if (attempts && 0 == --attempts)
  4437. break;
  4438. DBGLOG("Failed to copy: %s to %s, retrying...", from, to);
  4439. MilliSleep(delay);
  4440. }
  4441. return (attempts>0);
  4442. }
  4443. #endif
  4444. inline unsigned nextEditionN(unsigned e, unsigned i=1)
  4445. {
  4446. return e+i;
  4447. }
  4448. inline unsigned prevEditionN(unsigned e, unsigned i=1)
  4449. {
  4450. return e-i;
  4451. }
  4452. void removeDaliFile(const char *path, const char *base, unsigned e)
  4453. {
  4454. StringBuffer filename(path);
  4455. constructStoreName(base, e, filename);
  4456. OwnedIFile iFile = createIFile(filename.str());
  4457. try
  4458. {
  4459. iFile->remove();
  4460. }
  4461. catch (IException *e)
  4462. {
  4463. EXCLOG(e, NULL);
  4464. e->Release();
  4465. }
  4466. }
  4467. // Ensure internally used branches are present
  4468. void initializeInternals(IPropertyTree *root)
  4469. {
  4470. ensurePTree(root, "Files");
  4471. ensurePTree(root, "Queues");
  4472. ensurePTree(root, "Groups");
  4473. ensurePTree(root, "Status");
  4474. ensurePTree(root, "WorkUnits");
  4475. ensurePTree(root, "JobQueues");
  4476. ensurePTree(root, "Environment");
  4477. ensurePTree(root, "Locks");
  4478. ensurePTree(root, "DFU");
  4479. ensurePTree(root, "DFU/RECOVERY");
  4480. ensurePTree(root, "DFU/WorkUnits");
  4481. ensurePTree(root, "Files/Relationships");
  4482. root->removeProp("Status/Servers");
  4483. root->addPropTree("Status/Servers",createPTree());
  4484. }
  4485. IPropertyTree *loadStore(const char *storeFilename, IPTreeMaker *iMaker, unsigned crcValidation, bool logErrorsOnly=false, const bool *abort=NULL)
  4486. {
  4487. CHECKEDCRITICALBLOCK(loadStoreCrit, fakeCritTimeout);
  4488. CHECKEDCRITICALBLOCK(saveStoreCrit, fakeCritTimeout);
  4489. Owned<IPropertyTree> root;
  4490. try
  4491. {
  4492. OwnedIFile iFileStore = createIFile(storeFilename);
  4493. OwnedIFileIO iFileIOStore = iFileStore->open(IFOread);
  4494. if (!iFileIOStore)
  4495. throw MakeSDSException(SDSExcpt_OpenStoreFailed, "%s", storeFilename);
  4496. Owned<IFileIOStream> fstream = createIOStream(iFileIOStore);
  4497. Owned<ICrcIOStream> crcPipeStream = createCrcPipeStream(fstream);
  4498. Owned<IIOStream> ios = createBufferedIOStream(crcPipeStream);
  4499. root.setown((CServerRemoteTree *) createPTree(*ios, ipt_none, ptr_ignoreWhiteSpace, iMaker));
  4500. ios.clear();
  4501. unsigned crc = crcPipeStream->queryCrc();
  4502. if (crcValidation && crc != crcValidation)
  4503. LOG(MCoperatorWarning, unknownJob, "Error processing store %s - CRC ERROR (file size=%" I64F "d, validation crc=%x, calculated crc=%x)", storeFilename, iFileIOStore->size(), crcValidation, crc); // not fatal yet (maybe later)
  4504. }
  4505. catch (IException *e)
  4506. {
  4507. if (!abort)
  4508. {
  4509. StringBuffer s("Exception - loading store file : ");
  4510. s.appendf("%s", storeFilename);
  4511. LOG(MCoperatorError, unknownJob, e, s.str());
  4512. }
  4513. if (SDSExcpt_OpenStoreFailed != e->errorCode())
  4514. if (!logErrorsOnly)
  4515. throw;
  4516. e->Release();
  4517. }
  4518. catch (DALI_CATCHALL)
  4519. {
  4520. IException *e = MakeStringException(0, "Unknown exception - loading store file : %s", storeFilename);
  4521. LOG(MCdisaster, unknownJob, e, "");
  4522. if (!logErrorsOnly)
  4523. throw;
  4524. e->Release();
  4525. }
  4526. return LINK(root);
  4527. }
  4528. // Not really coalescing, blocking transations and saving store (which will delete pending transactions).
  4529. class CLightCoalesceThread : implements ICoalesce, public CInterface
  4530. {
  4531. bool stopped, within24;
  4532. Semaphore sem;
  4533. unsigned writeTransactionsNow, lastSaveWriteTransactions, lastWarning;
  4534. unsigned idlePeriod, minimumTimeBetweenSaves, idleRate;
  4535. Linked<IPropertyTree> config;
  4536. Owned<IJlibDateTime> quietStartTime, quietEndTime;
  4537. CheckedCriticalSection crit;
  4538. IStoreHelper *iStoreHelper;
  4539. class CThreaded : public Thread
  4540. {
  4541. CLightCoalesceThread *coalesce;
  4542. public:
  4543. CThreaded() : Thread("CLightCoalesceThread") { coalesce = NULL; }
  4544. void init(CLightCoalesceThread *_coalesce) { coalesce = _coalesce; start(); }
  4545. virtual int run() { coalesce->main(); return 1; }
  4546. } threaded;
  4547. public:
  4548. IMPLEMENT_IINTERFACE;
  4549. CLightCoalesceThread(IPropertyTree &_config, IStoreHelper *_iStoreHelper) : config(&_config), iStoreHelper(_iStoreHelper)
  4550. {
  4551. stopped = false;
  4552. idlePeriod = config->getPropInt("@lCIdlePeriod", DEFAULT_LCIDLE_PERIOD)*1000;
  4553. minimumTimeBetweenSaves = config->getPropInt("@lCMinTime", DEFAULT_LCMIN_TIME)*1000;
  4554. idleRate = config->getPropInt("@lCIdleRate", DEFAULT_LCIDLE_RATE);
  4555. char const *quietStartTimeStr = config->queryProp("@lCQuietStartTime");
  4556. if (quietStartTimeStr)
  4557. {
  4558. if (*quietStartTimeStr)
  4559. {
  4560. quietStartTime.setown(createDateTime());
  4561. quietStartTime->setLocalTimeString(quietStartTimeStr);
  4562. quietStartTime->setGmtDate(1970, 1, 1);
  4563. }
  4564. else
  4565. quietStartTimeStr = NULL;
  4566. }
  4567. char const *quietEndTimeStr = config->queryProp("@lCQuietEndTime");
  4568. if (quietStartTimeStr && !quietEndTimeStr)
  4569. {
  4570. WARNLOG("Start time for quiet period specified without end time, ignoring times");
  4571. quietStartTime.clear();
  4572. }
  4573. else if (quietEndTimeStr && *quietEndTimeStr)
  4574. {
  4575. if (!quietStartTimeStr)
  4576. WARNLOG("End time for quiet period specified without start time, ignoring times");
  4577. else
  4578. {
  4579. quietEndTime.setown(createDateTime());
  4580. quietEndTime->setLocalTimeString(quietEndTimeStr);
  4581. quietEndTime->setGmtDate(1970, 1, 1);
  4582. within24 = quietStartTime->compare(*quietEndTime) <= 0;
  4583. }
  4584. }
  4585. }
  4586. ~CLightCoalesceThread()
  4587. {
  4588. stop();
  4589. }
  4590. void main()
  4591. {
  4592. unsigned t = 0;
  4593. lastSaveWriteTransactions = SDSManager->writeTransactions;
  4594. lastWarning = 0;
  4595. unsigned lastEdition = iStoreHelper->queryCurrentEdition();
  4596. while (!stopped)
  4597. {
  4598. unsigned writeTransactionsNow = SDSManager->writeTransactions;
  4599. if (!sem.wait(idlePeriod))
  4600. {
  4601. if (writeTransactionsNow != lastSaveWriteTransactions)
  4602. {
  4603. if (quietStartTime)
  4604. {
  4605. Owned<IJlibDateTime> nowTime = createDateTimeNow();
  4606. nowTime->setGmtDate(1970, 1, 1);
  4607. if (within24)
  4608. {
  4609. if (!(nowTime->compare(*quietStartTime) >= 0 && nowTime->compare(*quietEndTime) <= 0))
  4610. continue; // if outside quiet period within 0-24
  4611. }
  4612. else if (nowTime->compare(*quietEndTime) > 0 && nowTime->compare(*quietStartTime) < 0)
  4613. continue; // if inside period excluded by quiet period
  4614. }
  4615. if (lastEdition == iStoreHelper->queryCurrentEdition()) // if not then something else has saved (e.g. probably sasha)
  4616. {
  4617. unsigned transactions = SDSManager->writeTransactions-writeTransactionsNow; // don't care about rollover.
  4618. if (0 == transactions ||
  4619. (0 != idleRate && idlePeriod>=60000 && (transactions/(idlePeriod/60000))<=idleRate))
  4620. {
  4621. StringBuffer filename;
  4622. iStoreHelper->getPrimaryLocation(filename);
  4623. iStoreHelper->getCurrentStoreFilename(filename);
  4624. OwnedIFile iFile = createIFile(filename);
  4625. CDateTime createTime, nowTime;
  4626. nowTime.setNow();
  4627. int diff = 0;
  4628. try
  4629. {
  4630. if (iFile->getTime(&createTime, NULL, NULL))
  4631. diff = ((int)nowTime.getSimple()-(int)createTime.getSimple())*1000;
  4632. }
  4633. catch (IException *e)
  4634. {
  4635. StringBuffer errMsg("failed to get createtime for : ");
  4636. errMsg.append(filename);
  4637. EXCLOG(e, errMsg.str());
  4638. e->Release();
  4639. }
  4640. int period;
  4641. if (diff<=0 || diff > (int)minimumTimeBetweenSaves) // <0 - createTime>nowTime, assume time skew and allow save.
  4642. {
  4643. period = minimumTimeBetweenSaves-idlePeriod;
  4644. if (0 > period) period = 0;
  4645. {
  4646. CHECKEDCRITICALBLOCK(saveStoreCrit, fakeCritTimeout);
  4647. SDSManager->blockingSave(&lastSaveWriteTransactions);
  4648. lastEdition = iStoreHelper->queryCurrentEdition();
  4649. }
  4650. t = lastWarning = 0;
  4651. }
  4652. else
  4653. period = minimumTimeBetweenSaves-diff;
  4654. sem.wait(period);
  4655. }
  4656. else
  4657. {
  4658. t += idlePeriod/1000;
  4659. if (t/3600 >= STORENOTSAVE_WARNING_PERIOD && ((t-lastWarning)/3600>(STORENOTSAVE_WARNING_PERIOD/2)))
  4660. {
  4661. WARNLOG("Store has not been saved for %d hours", t/3600);
  4662. lastWarning = t;
  4663. }
  4664. }
  4665. }
  4666. else
  4667. {
  4668. t = lastWarning = 0;
  4669. lastEdition = iStoreHelper->queryCurrentEdition();
  4670. }
  4671. }
  4672. }
  4673. }
  4674. }
  4675. // implements ICoalsce
  4676. virtual void start()
  4677. {
  4678. threaded.init(this);
  4679. }
  4680. virtual void stop()
  4681. {
  4682. if (!stopped)
  4683. {
  4684. stopped = true;
  4685. sem.signal();
  4686. threaded.join();
  4687. }
  4688. }
  4689. };
  4690. /////////////////
  4691. class CUnlockCallback : implements IUnlockCallback
  4692. { // NB: unblock() always called 1st, then block()
  4693. StringAttr xpath;
  4694. ConnectionId connectionId;
  4695. CServerRemoteTree &tree;
  4696. bool lockedForWrite, unlocked;
  4697. public:
  4698. CUnlockCallback(const char *_xpath, ConnectionId _connectionId, CServerRemoteTree &_tree) : xpath(_xpath), connectionId(_connectionId), tree(_tree), lockedForWrite(false), unlocked(false) { }
  4699. void block()
  4700. {
  4701. assertex(unlocked);
  4702. unsigned got = msTick();
  4703. if (lockedForWrite)
  4704. CHECKEDWRITELOCKENTER(SDSManager->dataRWLock, readWriteTimeout);
  4705. else
  4706. CHECKEDREADLOCKENTER(SDSManager->dataRWLock, readWriteTimeout);
  4707. CHECKEDCRITENTER(SDSManager->lockCrit, fakeCritTimeout);
  4708. unlocked = false;
  4709. unsigned e=msTick()-got;
  4710. if (e>readWriteSlowTracing)
  4711. {
  4712. StringBuffer s("TIME: CUnlockCallback(write=");
  4713. s.append(lockedForWrite).append(",xpath=").append(xpath).append(", connectionId=").appendf("%" I64F "x", connectionId).append(") took ").append(e);
  4714. DBGLOG("%s", s.str());
  4715. if (readWriteStackTracing)
  4716. PrintStackReport();
  4717. }
  4718. if (tree.isOrphaned())
  4719. throw MakeSDSException(SDSExcpt_OrphanedNode, "Whilst completing lock to %s", xpath.get());
  4720. }
  4721. void unblock()
  4722. {
  4723. unlocked = true;
  4724. lockedForWrite = SDSManager->dataRWLock.queryWriteLocked();
  4725. CHECKEDCRITLEAVE(SDSManager->lockCrit);
  4726. if (lockedForWrite)
  4727. SDSManager->dataRWLock.unlockWrite();
  4728. else
  4729. SDSManager->dataRWLock.unlockRead();
  4730. }
  4731. };
  4732. class CStoreHelper : implements IStoreHelper, public CInterface
  4733. {
  4734. StringAttr storeName, location, remoteBackupLocation;
  4735. CStoreInfo storeInfo, deltaInfo;
  4736. unsigned configFlags;
  4737. const bool *abort;
  4738. unsigned delay, keepStores;
  4739. SessionId mySessId;
  4740. void clearStoreInfo(const char *base, const char *location, unsigned edition, CStoreInfo *storeInfo=NULL)
  4741. {
  4742. StringBuffer wcard;
  4743. wcard.append(base).append(".*");
  4744. StringBuffer path, filename;
  4745. filename.append(base).append('.').append(edition);
  4746. if (location) path.append(location);
  4747. path.append(filename);
  4748. if (!storeInfo || !storeInfo->cache)
  4749. {
  4750. Owned<IDirectoryIterator> dIter = createDirectoryIterator(location, wcard.str());
  4751. ForEach (*dIter)
  4752. {
  4753. loop
  4754. {
  4755. try { dIter->query().remove(); break; }
  4756. catch (IException *e)
  4757. {
  4758. e->Release();
  4759. if (abort && *abort)
  4760. return;
  4761. MilliSleep(delay);
  4762. }
  4763. }
  4764. }
  4765. }
  4766. else
  4767. {
  4768. if (0 != stricmp(filename.str(), storeInfo->cache))
  4769. {
  4770. StringBuffer path(location);
  4771. path.append(storeInfo->cache);
  4772. OwnedIFile iFile = createIFile(path.str());
  4773. loop
  4774. {
  4775. try { iFile->remove(); break; }
  4776. catch (IException *e)
  4777. {
  4778. e->Release();
  4779. if (abort && *abort)
  4780. return;
  4781. MilliSleep(delay);
  4782. }
  4783. }
  4784. }
  4785. storeInfo->cache.clear();
  4786. }
  4787. }
  4788. void writeStoreInfo(const char *base, const char *location, unsigned edition, unsigned *crc, CStoreInfo *storeInfo=NULL)
  4789. {
  4790. StringBuffer path, filename;
  4791. filename.append(base).append('.').append(edition);
  4792. if (location) path.append(location);
  4793. path.append(filename);
  4794. OwnedIFile iFile = createIFile(path.str());
  4795. OwnedIFileIO iFileIO = iFile->open(IFOcreate);
  4796. if (crc)
  4797. iFileIO->write(0, sizeof(unsigned), crc);
  4798. if (storeInfo)
  4799. storeInfo->cache.set(filename.str());
  4800. }
  4801. void updateStoreInfo(const char *base, const char *location, unsigned edition, unsigned *crc, CStoreInfo *storeInfo=NULL)
  4802. {
  4803. clearStoreInfo(base, location, edition, storeInfo);
  4804. writeStoreInfo(base, location, edition, crc, storeInfo);
  4805. }
  4806. void refreshInfo(CStoreInfo &info, const char *base)
  4807. {
  4808. OwnedIFile found;
  4809. OwnedIFileIO iFileIO;
  4810. if (info.cache.length()) // avoid directory lookup if poss.
  4811. {
  4812. StringBuffer path(location);
  4813. OwnedIFile iFile = createIFile(path.append(info.cache).str());
  4814. if (iFile->exists())
  4815. {
  4816. found.set(iFile);
  4817. try { iFileIO.setown(found->open(IFOread)); }
  4818. catch (IException *e)
  4819. {
  4820. e->Release();
  4821. }
  4822. }
  4823. }
  4824. if (!iFileIO)
  4825. {
  4826. StringBuffer wcard;
  4827. wcard.append(base).append(".*");
  4828. Owned<IDirectoryIterator> dIter = createDirectoryIterator(location, wcard.str());
  4829. unsigned totalDelays = 0;
  4830. loop
  4831. {
  4832. if (dIter->first())
  4833. {
  4834. const char *name = dIter->query().queryFilename();
  4835. StringBuffer base, ext, fname;
  4836. splitFilename(name, NULL, NULL, &base, &ext);
  4837. fname.append(base).append(ext);
  4838. info.cache.set(fname.str());
  4839. found.set(&dIter->query());
  4840. try { iFileIO.setown(found->open(IFOread)); }
  4841. catch (IException *e)
  4842. {
  4843. e->Release();
  4844. }
  4845. if (iFileIO)
  4846. break;
  4847. }
  4848. totalDelays++;
  4849. if (totalDelays >= MAXDELAYS)
  4850. throw MakeSDSException(SDSExcpt_StoreInfoMissing, "store.<edition> file appears to be missing");
  4851. if (abort && *abort)
  4852. return;
  4853. MilliSleep(delay);
  4854. }
  4855. }
  4856. assertex(iFileIO);
  4857. StringBuffer tail, ext, fname;
  4858. splitFilename(found->queryFilename(), NULL, NULL, &tail, &ext);
  4859. fname.append(tail).append(ext);
  4860. const char *name = fname.str();
  4861. const char *editionBegin = name+strlen(base)+1;
  4862. info.edition = atoi(editionBegin);
  4863. if (iFileIO->size())
  4864. iFileIO->read(0, sizeof(unsigned), &info.crc);
  4865. else
  4866. info.crc = 0;
  4867. }
  4868. void refreshStoreInfo() { refreshInfo(storeInfo, "store"); }
  4869. void refreshDeltaInfo() { refreshInfo(deltaInfo, "store"); }
  4870. void checkInfo(const char *base, CStoreInfo &info)
  4871. {
  4872. StringBuffer wcard;
  4873. wcard.append(base).append(".*");
  4874. Owned<IDirectoryIterator> dIter = createDirectoryIterator(location, wcard.str());
  4875. if (!dIter->first())
  4876. updateStoreInfo(base, location, 0, NULL, &info);
  4877. else if (dIter->next())
  4878. throw MakeStringException(0, "Multiple store.X files - only one corresponding to latest dalisds<X>.xml should exist");
  4879. }
  4880. void renameDelta(unsigned oldEdition, unsigned newEdition, const char *path)
  4881. {
  4882. StringBuffer deltaName(path);
  4883. constructStoreName(DELTANAME, oldEdition, deltaName);
  4884. OwnedIFile oldDelta = createIFile(deltaName.str());
  4885. if (oldDelta->exists())
  4886. {
  4887. deltaName.clear();
  4888. constructStoreName(DELTANAME, newEdition, deltaName);
  4889. oldDelta->rename(deltaName.str());
  4890. }
  4891. }
  4892. struct CheckDeltaBlock
  4893. {
  4894. CheckDeltaBlock(CStoreHelper &_storeHelper) : storeHelper(_storeHelper)
  4895. {
  4896. bool created = false;
  4897. StringBuffer deltaIPStr(storeHelper.location);
  4898. OwnedIFile deltaIPIFile = createIFile(deltaIPStr.append(DELTAINPROGRESS).str());
  4899. activeDetachIPStr.append(DETACHINPROGRESS);
  4900. inactiveDetachIPStr.append(storeHelper.location).append('_').append(DETACHINPROGRESS);
  4901. detachIPIFile.setown(createIFile(inactiveDetachIPStr.str()));
  4902. OwnedIFileIO detachIPIO = detachIPIFile->open(IFOcreate);
  4903. detachIPIO->write(0, sizeof(storeHelper.mySessId), &storeHelper.mySessId);
  4904. detachIPIO.clear();
  4905. detachIPIFile->rename(activeDetachIPStr.str());
  4906. // check often do not wait any longer than necessary
  4907. unsigned d=0;
  4908. while (deltaIPIFile->exists())
  4909. {
  4910. if (0 == d++ % 50)
  4911. PROGLOG("Waiting for a saveDelta in progress");
  4912. MilliSleep(100);
  4913. }
  4914. }
  4915. ~CheckDeltaBlock() noexcept(false)
  4916. {
  4917. if (detachIPIFile)
  4918. {
  4919. unsigned a=0;
  4920. loop
  4921. {
  4922. try { detachIPIFile->remove(); break; }
  4923. catch (IException *e)
  4924. {
  4925. EXCLOG(e, "removing detach file marker");
  4926. if (a++ > 10) throw;
  4927. e->Release();
  4928. }
  4929. MilliSleep(500);
  4930. }
  4931. }
  4932. }
  4933. private:
  4934. CStoreHelper &storeHelper;
  4935. OwnedIFile detachIPIFile;
  4936. StringBuffer activeDetachIPStr, inactiveDetachIPStr;
  4937. };
  4938. public:
  4939. IMPLEMENT_IINTERFACE;
  4940. CStoreHelper(const char *_storeName, const char *_location, const char *_remoteBackupLocation, unsigned _configFlags, unsigned _keepStores, unsigned _delay, const bool *_abort) : storeName(_storeName), location(_location), remoteBackupLocation(_remoteBackupLocation), configFlags(_configFlags), keepStores(_keepStores), delay(_delay), abort(_abort)
  4941. {
  4942. mySessId = daliClientActive()?myProcessSession():0;
  4943. if (!keepStores) keepStores = DEFAULT_KEEP_LASTN_STORES;
  4944. checkInfo("store", storeInfo);
  4945. checkInfo("store", deltaInfo);
  4946. if (0 == (SH_External & configFlags))
  4947. {
  4948. refreshStoreInfo();
  4949. unsigned edition = storeInfo.edition;
  4950. Owned<IDirectoryIterator> di = createDirectoryIterator(location, "dali*.xml");
  4951. ForEach (*di)
  4952. {
  4953. StringBuffer fname;
  4954. di->getName(fname);
  4955. if ('_' != fname.charAt(7)) // Unhelpful naming convention to differentiate store files from externals!
  4956. {
  4957. if (0 == memicmp("inc", fname.str()+4, 3) || 0 == memicmp("sds", fname.str()+4, 3))
  4958. {
  4959. const char *num = fname.str()+7;
  4960. const char *dot = (const char *)strchr(num, '.');
  4961. unsigned fileEdition = atoi_l(num, dot-num);
  4962. int d = (int)fileEdition-(int)edition;
  4963. if (edition != fileEdition && (d>=1 || d<(-(int)keepStores)))
  4964. {
  4965. IFile &file = di->query();
  4966. CDateTime dt;
  4967. dt.setNow();
  4968. StringBuffer newName(file.queryFilename());
  4969. newName.append('.');
  4970. unsigned i=newName.length();
  4971. dt.getString(newName); // base on date, incase any old copies.
  4972. for (;i<newName.length();i++)
  4973. if (newName.charAt(i)==':')
  4974. newName.setCharAt(i,'_');
  4975. newName.append(".unused");
  4976. PROGLOG("Detected spurious data file : '%s' - renaming to %s", file.queryFilename(), newName.str());
  4977. try
  4978. {
  4979. file.rename(newName.str());
  4980. }
  4981. catch (IException *e) // safe to ignore these (should e.g. files be in use).
  4982. {
  4983. EXCLOG(e, NULL);
  4984. e->Release();
  4985. }
  4986. }
  4987. }
  4988. }
  4989. }
  4990. StringBuffer dst(location);
  4991. addPathSepChar(dst);
  4992. dst.append(DEBUG_DIR);
  4993. addPathSepChar(dst);
  4994. OwnedIFile dFile = createIFile(dst.str());
  4995. Owned<IDirectoryIterator> dIter = dFile->directoryFiles();
  4996. ForEach(*dIter)
  4997. dIter->query().remove();
  4998. }
  4999. }
  5000. virtual StringBuffer &getDetachedDeltaName(StringBuffer &detachName)
  5001. {
  5002. refreshDeltaInfo();
  5003. constructStoreName(DELTADETACHED, deltaInfo.edition, detachName);
  5004. return detachName;
  5005. }
  5006. virtual bool loadDelta(const char *filename, IFile *iFile, IPropertyTree *root)
  5007. {
  5008. Owned<IFileIO> iFileIO = iFile->open(IFOread);
  5009. if (!iFileIO) // no delta to load
  5010. return true;
  5011. MemoryBuffer tmp;
  5012. char *ptr = (char *) tmp.reserveTruncate(strlen(deltaHeader));
  5013. unsigned embeddedCrc = 0;
  5014. offset_t pos = 0;
  5015. bool hasCrcHeader = false; // check really only needed for deltas proceeding CRC header
  5016. if (strlen(deltaHeader) == iFileIO->read(0, strlen(deltaHeader), ptr))
  5017. {
  5018. if (0 == memicmp(deltaHeader, ptr, 5))
  5019. {
  5020. pos = deltaHeaderSizeStart;
  5021. hasCrcHeader = true;
  5022. embeddedCrc = (unsigned)atoi64_l(ptr+deltaHeaderCrcOff, 10);
  5023. if (0 == memicmp(deltaHeader+deltaHeaderSizeStart, ptr+deltaHeaderSizeStart, 6)) // has <SIZE> too
  5024. {
  5025. pos = strlen(deltaHeader);
  5026. offset_t lastGood;
  5027. if (sscanf(ptr+deltaHeaderSizeOff, "%" I64F "X", &lastGood))
  5028. {
  5029. offset_t fSize = iFileIO->size();
  5030. if (fSize > lastGood)
  5031. {
  5032. offset_t diff = fSize - lastGood;
  5033. LOG(MCoperatorError, unknownJob, "Delta file '%s', has %" I64F "d bytes of trailing data (possible power loss during save?), file size: %" I64F "d, last committed size: %" I64F "d", filename, diff, fSize, lastGood);
  5034. LOG(MCoperatorError, unknownJob, "Resetting delta file '%s' to size: %" I64F "d", filename, lastGood);
  5035. iFileIO->close();
  5036. backup(filename);
  5037. iFileIO.setown(iFile->open(IFOreadwrite));
  5038. iFileIO->setSize(lastGood);
  5039. iFileIO->close();
  5040. iFileIO.setown(iFile->open(IFOread));
  5041. }
  5042. }
  5043. }
  5044. }
  5045. }
  5046. OwnedIFileIOStream iFileIOStream = createIOStream(iFileIO);
  5047. iFileIOStream->seek(pos, IFSbegin);
  5048. Owned<ICrcIOStream> crcPipeStream = createCrcPipeStream(iFileIOStream); // crc *rest* of stream
  5049. Owned<IIOStream> ios = createBufferedIOStream(crcPipeStream);
  5050. bool noErrors;
  5051. Owned<IException> deltaE;
  5052. noErrors = applyXmlDeltas(*root, *ios, 0 == (SH_RecoverFromIncErrors & configFlags));
  5053. if (noErrors && hasCrcHeader)
  5054. {
  5055. unsigned crc = crcPipeStream->queryCrc();
  5056. if (embeddedCrc != crc)
  5057. {
  5058. noErrors = false;
  5059. StringBuffer s;
  5060. LOG(MCoperatorWarning, unknownJob, "%s", s.append("Delta '").append(filename).append("' crc mismatch").str());
  5061. }
  5062. }
  5063. return noErrors;
  5064. }
  5065. virtual bool loadDeltas(IPropertyTree *root, bool *errors)
  5066. {
  5067. bool res = false;
  5068. if (errors) *errors = false;
  5069. StringBuffer deltaFilename(location);
  5070. constructStoreName(DELTANAME, storeInfo.edition, deltaFilename);
  5071. StringBuffer detachPath(location);
  5072. OwnedIFile detachedDeltaIFile = createIFile(getDetachedDeltaName(detachPath).str());
  5073. bool detached = detachedDeltaIFile->exists();
  5074. OwnedIFile deltaIFile = createIFile(deltaFilename.str());
  5075. loop
  5076. {
  5077. StringAttr filename;
  5078. IFile *iFile;
  5079. if (detached)
  5080. {
  5081. iFile = detachedDeltaIFile;
  5082. filename.set(iFile->queryFilename());
  5083. }
  5084. else
  5085. {
  5086. iFile = deltaIFile;
  5087. filename.set(iFile->queryFilename());
  5088. if (!iFile->exists())
  5089. break;
  5090. }
  5091. PROGLOG("Loading delta: %s", filename.get());
  5092. bool noError;
  5093. Owned<IException> deltaE;
  5094. try { noError = loadDelta(filename, iFile, root); }
  5095. catch (IException *e) { deltaE.setown(e); noError = false; }
  5096. if (!noError)
  5097. {
  5098. backup(filename);
  5099. if (errors) *errors = true;
  5100. if (deltaE)
  5101. throw LINK(deltaE);
  5102. }
  5103. res = true;
  5104. if (detached)
  5105. detached = false;
  5106. else
  5107. break;
  5108. }
  5109. return res;
  5110. }
  5111. virtual bool detachCurrentDelta()
  5112. {
  5113. StringBuffer deltaFilename(location);
  5114. getCurrentDeltaFilename(deltaFilename);
  5115. refreshStoreInfo();
  5116. bool res = false;
  5117. try
  5118. {
  5119. CheckDeltaBlock cD(*this);
  5120. OwnedIFile deltaIFile = createIFile(deltaFilename.str());
  5121. if (deltaIFile->exists())
  5122. {
  5123. #ifdef NODELETE
  5124. StringBuffer detachPath(location);
  5125. getDetachedDeltaName(detachPath);
  5126. if (retryCopy(deltaFilename.str(), detachPath.str(), 5, delay))
  5127. #else
  5128. StringBuffer detachName;
  5129. getDetachedDeltaName(detachName);
  5130. if (retryRename(deltaFilename.str(), detachName.str(), 5, delay))
  5131. #endif
  5132. res = true;
  5133. }
  5134. if (remoteBackupLocation.length())
  5135. {
  5136. deltaFilename.clear().append(remoteBackupLocation);
  5137. getCurrentDeltaFilename(deltaFilename);
  5138. OwnedIFile iFile = createIFile(deltaFilename);
  5139. if (iFile->exists())
  5140. {
  5141. StringBuffer detachName;
  5142. getDetachedDeltaName(detachName);
  5143. iFile->rename(detachName.str());
  5144. }
  5145. }
  5146. }
  5147. catch (IException *e)
  5148. {
  5149. LOG(MCoperatorError, unknownJob, e, "detachCurrentDelta");
  5150. e->Release();
  5151. }
  5152. return res;
  5153. }
  5154. virtual void saveStore(IPropertyTree *root, unsigned *_newEdition, bool currentEdition=false)
  5155. {
  5156. LOG(MCdebugInfo(100), unknownJob, "Saving store");
  5157. refreshStoreInfo();
  5158. unsigned edition = storeInfo.edition;
  5159. unsigned newEdition = currentEdition?edition:nextEditionN(edition);
  5160. bool done = false;
  5161. try
  5162. {
  5163. unsigned crc = 0;
  5164. StringBuffer tmpStoreName;
  5165. OwnedIFileIO iFileIOTmpStore = createUniqueFile(location, TMPSAVENAME, NULL, tmpStoreName);
  5166. OwnedIFile iFileTmpStore = createIFile(tmpStoreName);
  5167. try
  5168. {
  5169. OwnedIFileIOStream fstream = createIOStream(iFileIOTmpStore);
  5170. Owned<ICrcIOStream> crcPipeStream = createCrcPipeStream(fstream);
  5171. Owned<IIOStream> ios = createBufferedIOStream(crcPipeStream);
  5172. #ifdef _DEBUG
  5173. toXML(root, *ios); // formatted (default)
  5174. #else
  5175. toXML(root, *ios, 0, 0);
  5176. #endif
  5177. ios.clear();
  5178. fstream.clear();
  5179. crc = crcPipeStream->queryCrc();
  5180. crcPipeStream.clear();
  5181. iFileIOTmpStore.clear();
  5182. }
  5183. catch (IException *e)
  5184. {
  5185. LOG(MCoperatorError, unknownJob, e, "Exception(1) - Error saving store file");
  5186. iFileIOTmpStore.clear();
  5187. iFileTmpStore->remove();
  5188. throw;
  5189. }
  5190. StringBuffer newStoreName;
  5191. constructStoreName(storeName, newEdition, newStoreName);
  5192. StringBuffer newStoreNamePath(location);
  5193. newStoreNamePath.append(newStoreName);
  5194. refreshStoreInfo();
  5195. if (storeInfo.edition != edition)
  5196. {
  5197. WARNLOG("Another process has updated the edition whilst saving the store: %s", newStoreNamePath.str());
  5198. iFileTmpStore->remove();
  5199. return;
  5200. }
  5201. try
  5202. {
  5203. OwnedIFile newStoreIFile = createIFile(newStoreNamePath.str());
  5204. newStoreIFile->remove();
  5205. iFileTmpStore->rename(newStoreName.str());
  5206. }
  5207. catch (IException *e)
  5208. {
  5209. StringBuffer errMsg;
  5210. EXCLOG(e, errMsg.append("Failed to rename new store to : ").append(newStoreNamePath).append(". Has already been created by another process?").str());
  5211. e->Release();
  5212. iFileTmpStore->remove();
  5213. return;
  5214. }
  5215. if (0 != (SH_CheckNewDelta & configFlags))
  5216. {
  5217. CheckDeltaBlock cD(*this);
  5218. try { renameDelta(edition, newEdition, location); }
  5219. catch (IException *e)
  5220. {
  5221. StringBuffer s("Exception(2) - Error saving store file");
  5222. LOG(MCoperatorError, unknownJob, e, s.str());
  5223. e->Release();
  5224. return;
  5225. }
  5226. if (remoteBackupLocation.length())
  5227. {
  5228. try { renameDelta(edition, newEdition, remoteBackupLocation); }
  5229. catch (IException *e)
  5230. {
  5231. LOG(MCoperatorError, unknownJob, e, "Failure handling backup");
  5232. e->Release();
  5233. }
  5234. }
  5235. clearStoreInfo("store", location, 0, NULL);
  5236. writeStoreInfo("store", location, newEdition, &crc, &storeInfo);
  5237. }
  5238. else
  5239. {
  5240. clearStoreInfo("store", location, 0, NULL);
  5241. writeStoreInfo("store", location, newEdition, &crc, &storeInfo);
  5242. }
  5243. try
  5244. {
  5245. if (remoteBackupLocation.length())
  5246. {
  5247. PROGLOG("Copying store to backup location");
  5248. StringBuffer rL(remoteBackupLocation);
  5249. constructStoreName(storeName, newEdition, rL);
  5250. copyFile(rL.str(), newStoreNamePath.str());
  5251. clearStoreInfo("store", remoteBackupLocation, 0, NULL);
  5252. writeStoreInfo("store", remoteBackupLocation, newEdition, &crc, &storeInfo);
  5253. PROGLOG("Copy done");
  5254. }
  5255. }
  5256. catch (IException *e)
  5257. {
  5258. StringBuffer s;
  5259. LOG(MCoperatorError, unknownJob, e, s.append("Failure to backup dali to remote location: ").append(remoteBackupLocation));
  5260. e->Release();
  5261. }
  5262. if (_newEdition)
  5263. *_newEdition = newEdition;
  5264. done = true;
  5265. LOG(MCdebugInfo(100), unknownJob, "Store saved");
  5266. }
  5267. catch (IException *e)
  5268. {
  5269. StringBuffer s("Exception(3) - Error saving store file");
  5270. LOG(MCoperatorError, unknownJob, e, s.str());
  5271. e->Release();
  5272. }
  5273. if (done)
  5274. {
  5275. #ifndef NODELETE
  5276. unsigned toDeleteEdition = prevEditionN(edition, keepStores+(currentEdition?1:0));
  5277. StringBuffer filename(location);
  5278. constructStoreName(storeName, toDeleteEdition, filename);
  5279. OwnedIFile iFile = createIFile(filename.str());
  5280. if (iFile->exists())
  5281. PROGLOG("Deleting old store: %s", filename.str());
  5282. removeDaliFile(location, storeName, toDeleteEdition);
  5283. removeDaliFile(location, DELTANAME, toDeleteEdition);
  5284. removeDaliFile(location, DELTADETACHED, toDeleteEdition);
  5285. if (remoteBackupLocation)
  5286. {
  5287. removeDaliFile(remoteBackupLocation, storeName, toDeleteEdition);
  5288. removeDaliFile(remoteBackupLocation, DELTANAME, toDeleteEdition);
  5289. removeDaliFile(remoteBackupLocation, DELTADETACHED, toDeleteEdition);
  5290. }
  5291. #endif
  5292. }
  5293. }
  5294. virtual unsigned queryCurrentEdition()
  5295. {
  5296. refreshStoreInfo();
  5297. return storeInfo.edition;
  5298. }
  5299. virtual StringBuffer &getCurrentStoreFilename(StringBuffer &res, unsigned *crc=NULL)
  5300. {
  5301. refreshStoreInfo();
  5302. constructStoreName(storeName, storeInfo.edition, res);
  5303. if (crc)
  5304. * crc = storeInfo.crc;
  5305. return res;
  5306. }
  5307. virtual StringBuffer &getCurrentDeltaFilename(StringBuffer &res, unsigned *crc=NULL)
  5308. {
  5309. refreshDeltaInfo();
  5310. constructStoreName(DELTANAME, deltaInfo.edition, res);
  5311. if (crc)
  5312. * crc = deltaInfo.crc; // TBD, combine into store.<edition>.<store_crc>.<delta_crc>
  5313. return res;
  5314. }
  5315. virtual StringBuffer &getCurrentStoreInfoFilename(StringBuffer &res)
  5316. {
  5317. refreshStoreInfo();
  5318. res.append(storeInfo.cache);
  5319. return res;
  5320. }
  5321. virtual void backup(const char *filename)
  5322. {
  5323. try
  5324. {
  5325. unsigned crc = getFileCRC(filename);
  5326. StringBuffer dst(location);
  5327. if (dst.length())
  5328. addPathSepChar(dst);
  5329. dst.append(DEBUG_DIR);
  5330. addPathSepChar(dst);
  5331. recursiveCreateDirectoryForFile(dst.str());
  5332. OwnedIFile dFile = createIFile(dst.str());
  5333. Owned<IDirectoryIterator> dIter = dFile->directoryFiles();
  5334. unsigned debugFiles = 0;
  5335. ForEach (*dIter) debugFiles++;
  5336. if (debugFiles >= 10) return;
  5337. StringBuffer fname(filename);
  5338. getFileNameOnly(fname);
  5339. dst.append(fname.str()).append('.').append(crc);
  5340. OwnedIFile backupIFile = createIFile(dst.str());
  5341. if (!backupIFile->exists()) // a copy could already have been made
  5342. {
  5343. PROGLOG("Backing up: %s", filename);
  5344. OwnedIFile iFile = createIFile(filename);
  5345. copyFile(backupIFile, iFile);
  5346. PROGLOG("Backup made: %s", dst.str());
  5347. }
  5348. }
  5349. catch (IException *e)
  5350. {
  5351. StringBuffer tmp;
  5352. EXCLOG(e, tmp.append("Failed to take backup of: ").append(filename).str());
  5353. e->Release();
  5354. }
  5355. }
  5356. virtual StringBuffer &getPrimaryLocation(StringBuffer &_location)
  5357. {
  5358. _location.append(location);
  5359. return _location;
  5360. }
  5361. virtual StringBuffer &getBackupLocation(StringBuffer &backupLocation)
  5362. {
  5363. backupLocation.append(remoteBackupLocation);
  5364. return backupLocation;
  5365. }
  5366. friend struct CheckDeltaBlock;
  5367. };
  5368. IStoreHelper *createStoreHelper(const char *storeName, const char *location, const char *remoteBackupLocation, unsigned configFlags, unsigned keepStores, unsigned delay, const bool *abort)
  5369. {
  5370. if (!storeName) storeName = "dalisds";
  5371. return new CStoreHelper(storeName, location, remoteBackupLocation, configFlags, keepStores, delay, abort);
  5372. }
  5373. ///////////////
  5374. #ifdef _MSC_VER
  5375. #pragma warning (push)
  5376. #pragma warning (disable : 4355) // 'this' : used in base member initializer list
  5377. #endif
  5378. CCovenSDSManager::CCovenSDSManager(ICoven &_coven, IPropertyTree &_config, const char *_dataPath)
  5379. : coven(_coven), config(_config), server(*this), dataPath(_dataPath), backupHandler(_config)
  5380. {
  5381. config.Link();
  5382. restartOnError = config.getPropBool("@restartOnUnhandled");
  5383. root = NULL;
  5384. writeTransactions=0;
  5385. externalEnvironment = false;
  5386. ignoreExternals=false;
  5387. unsigned initNodeTableSize = queryCoven().getInitSDSNodes();
  5388. allNodes.ensure(initNodeTableSize?initNodeTableSize:INIT_NODETABLE_SIZE);
  5389. externalSizeThreshold = config.getPropInt("@externalSizeThreshold", DEFAULT_EXTERNAL_SIZE_THRESHOLD);
  5390. remoteBackupLocation.set(config.queryProp("@remoteBackupLocation"));
  5391. nextExternal = 1;
  5392. if (0 == coven.getServerRank())
  5393. {
  5394. if (coven.size() > 1)
  5395. {
  5396. unsigned s;
  5397. for (s=1; s<coven.size(); s++)
  5398. {
  5399. CMessageBuffer mb;
  5400. __int64 dummy=0;
  5401. mb.append(dummy);
  5402. coven.sendRecv(mb, s, MPTAG_DALI_SDS_REQUEST);
  5403. bool success;
  5404. mb.read(success);
  5405. assertex(success);
  5406. }
  5407. }
  5408. }
  5409. else
  5410. {
  5411. CMessageBuffer mb;
  5412. #ifdef TRACE_QWAITING
  5413. unsigned waiting = coven.probe(0,MPTAG_DALI_SDS_REQUEST,NULL);
  5414. if ((waiting!=0)&&(waiting%10==0))
  5415. DBGLOG("QPROBE: MPTAG_DALI_SDS_REQUEST.2 has %d waiting",waiting);
  5416. #endif
  5417. if (coven.recv(mb, 0, MPTAG_DALI_SDS_REQUEST, NULL))
  5418. {
  5419. __int64 dummy;
  5420. mb.read(dummy);
  5421. mb.clear().append(true); // denote success
  5422. coven.reply(mb);
  5423. }
  5424. else
  5425. assertex(false);
  5426. }
  5427. registerSubscriptionManager(SDS_PUBLISHER, this);
  5428. connectionSubscriptionManager.setown(new CConnectionSubscriptionManager());
  5429. nodeSubscriptionManager.setown(new CNodeSubscriptionManager(*this));
  5430. registerSubscriptionManager(SDSCONN_PUBLISHER, connectionSubscriptionManager.get());
  5431. registerSubscriptionManager(SDSNODE_PUBLISHER, nodeSubscriptionManager);
  5432. // add external handlers
  5433. Owned<CXMLFileExternal> xmlExternalHandler = new CXMLFileExternal(dataPath, backupHandler);
  5434. externalHandlers.replace(* new CExternalHandlerMapping(EF_XML, *xmlExternalHandler));
  5435. Owned<CLegacyBinaryFileExternal> legacyBinaryExternalHandler = new CLegacyBinaryFileExternal(dataPath, backupHandler);
  5436. externalHandlers.replace(* new CExternalHandlerMapping(EF_LegacyBinaryValue, *legacyBinaryExternalHandler));
  5437. Owned<CBinaryFileExternal> binaryExternalHandler = new CBinaryFileExternal(dataPath, backupHandler);
  5438. externalHandlers.replace(* new CExternalHandlerMapping(EF_BinaryValue, *binaryExternalHandler));
  5439. properties.setown(createPTree("Properties"));
  5440. IPropertyTree *clientProps = properties->setPropTree("Client", config.hasProp("Client") ? config.getPropTree("Client") : createPTree());
  5441. clientProps->setPropBool("@serverIterAvailable", true);
  5442. clientProps->setPropBool("@useAppendOpt", true);
  5443. clientProps->setPropBool("@serverGetIdsAvailable", true);
  5444. IPropertyTree *throttle = clientProps->setPropTree("Throttle", createPTree());
  5445. throttle->setPropInt("@limit", CLIENT_THROTTLE_LIMIT);
  5446. throttle->setPropInt("@delay", CLIENT_THROTTLE_DELAY);
  5447. // NB: dataPath is assumed to be local
  5448. RemoteFilename rfn;
  5449. if (dataPath.length())
  5450. rfn.setLocalPath(dataPath);
  5451. else
  5452. {
  5453. char cwd[1024];
  5454. if (!GetCurrentDirectory(1024, cwd)) {
  5455. ERRLOG("CCovenSDSManager: Current directory path too big, setting local path to null");
  5456. cwd[0] = 0;
  5457. }
  5458. rfn.setLocalPath(cwd);
  5459. }
  5460. unsigned keepLastN = config.getPropInt("@keepStores", DEFAULT_KEEP_LASTN_STORES);
  5461. StringBuffer path;
  5462. rfn.getRemotePath(path);
  5463. properties->setProp("@dataPathUrl", path.str());
  5464. properties->setPropInt("@keepStores", keepLastN);
  5465. if (remoteBackupLocation.length())
  5466. {
  5467. properties->setProp("@backupPathUrl", remoteBackupLocation.get());
  5468. backupHandler.init(remoteBackupLocation, config.getPropBool("@asyncBackup", true));
  5469. }
  5470. const char *storeName = config.queryProp("@store");
  5471. if (!storeName) storeName = "dalisds";
  5472. #if 1 // legacy
  5473. StringBuffer tail, ext;
  5474. splitFilename(storeName, NULL, NULL, &tail, &ext);
  5475. if (0 == stricmp(".xml", ext.str()))
  5476. {
  5477. config.setProp("@store", tail.str());
  5478. storeName = tail.str();
  5479. }
  5480. #endif
  5481. StringBuffer tmp(dataPath);
  5482. OwnedIFile inProgressIFile = createIFile(tmp.append(DELTAINPROGRESS).str());
  5483. inProgressIFile->remove();
  5484. OwnedIFile detachIPIFile = createIFile(tmp.append(DETACHINPROGRESS).str());
  5485. detachIPIFile->remove();
  5486. unsigned configFlags = config.getPropBool("@recoverFromIncErrors", true) ? SH_RecoverFromIncErrors : 0;
  5487. configFlags |= config.getPropBool("@backupErrorFiles", true) ? SH_BackupErrorFiles : 0;
  5488. iStoreHelper = createStoreHelper(storeName, dataPath, remoteBackupLocation, configFlags, keepLastN, 100, &server.queryStopped());
  5489. doTimeComparison = false;
  5490. if (config.getPropBool("@lightweightCoalesce", true))
  5491. coalesce.setown(new CLightCoalesceThread(config, iStoreHelper));
  5492. }
  5493. #ifdef _MSC_VER
  5494. #pragma warning (pop)
  5495. #endif
  5496. CCovenSDSManager::~CCovenSDSManager()
  5497. {
  5498. backupHandler.stop();
  5499. if (unhandledThread) unhandledThread->join();
  5500. if (coalesce) coalesce->stop();
  5501. scanNotifyPool.clear();
  5502. notifyPool.clear();
  5503. connections.kill();
  5504. ::Release(iStoreHelper);
  5505. if (!config.getPropBool("@leakStore", true)) // intentional default leak of time consuming deconstruction of tree
  5506. ::Release(root);
  5507. else
  5508. enableMemLeakChecking(false);
  5509. config.Release();
  5510. }
  5511. bool compareFiles(IFile *file1, IFile *file2, bool compareTimes=true)
  5512. {
  5513. if (file1->exists())
  5514. {
  5515. if (file2->exists())
  5516. {
  5517. if (file1->size() == file2->size())
  5518. {
  5519. if (!compareTimes) return true;
  5520. CDateTime modifiedTimeBackup;
  5521. file1->getTime(NULL, &modifiedTimeBackup, NULL);
  5522. CDateTime modifiedTime;
  5523. file2->getTime(NULL, &modifiedTime, NULL);
  5524. if (0 == modifiedTimeBackup.compare(modifiedTime, false))
  5525. return true;
  5526. }
  5527. }
  5528. }
  5529. else
  5530. return !file2->exists();
  5531. return false;
  5532. }
  5533. void CCovenSDSManager::validateBackup()
  5534. {
  5535. // check consistency of store info file.
  5536. StringBuffer storeInfoFilename(dataPath);
  5537. iStoreHelper->getCurrentStoreInfoFilename(storeInfoFilename);
  5538. OwnedIFile infoIFile = createIFile(storeInfoFilename.str());
  5539. if (infoIFile->exists())
  5540. {
  5541. StringBuffer rL(remoteBackupLocation);
  5542. iStoreHelper->getCurrentStoreInfoFilename(rL);
  5543. copyFile(rL.str(), storeInfoFilename.str());
  5544. }
  5545. // check consistency of delta
  5546. StringBuffer deltaFilename(dataPath);
  5547. iStoreHelper->getCurrentDeltaFilename(deltaFilename);
  5548. OwnedIFile iFileDelta = createIFile(deltaFilename.str());
  5549. deltaFilename.clear().append(remoteBackupLocation);
  5550. iStoreHelper->getCurrentDeltaFilename(deltaFilename);
  5551. OwnedIFile iFileDeltaBackup = createIFile(deltaFilename.str());
  5552. if (!compareFiles(iFileDeltaBackup, iFileDelta, false))
  5553. WARNLOG("Delta file backup doesn't exist or differs, filename=%s", deltaFilename.str());
  5554. // ensure there's a copy of the primary store present at startup.
  5555. StringBuffer storeFilename(dataPath);
  5556. iStoreHelper->getCurrentStoreFilename(storeFilename);
  5557. OwnedIFile iFileStore = createIFile(storeFilename.str());
  5558. storeFilename.clear().append(remoteBackupLocation);
  5559. iStoreHelper->getCurrentStoreFilename(storeFilename);
  5560. OwnedIFile iFileBackupStore = createIFile(storeFilename.str());
  5561. if (!compareFiles(iFileBackupStore, iFileStore))
  5562. WARNLOG("Store backup file doesn't exist or differs, filename=%s", storeFilename.str());
  5563. }
  5564. static int uint64compare(unsigned __int64 const *i1, unsigned __int64 const *i2)
  5565. {
  5566. if (*i1==*i2) return 0;
  5567. if (*i1<*i2) return -1;
  5568. return 1;
  5569. }
  5570. class CLegacyFmtItem : public CInterface
  5571. {
  5572. public:
  5573. CLegacyFmtItem(const char *_name, const char *_ext, unsigned __int64 _num) : name(_name), ext(_ext), num(_num) { }
  5574. StringAttr name, ext;
  5575. unsigned __int64 num;
  5576. };
  5577. static int extNcompareFunc(CInterface * const *_itm1, CInterface * const *_itm2)
  5578. {
  5579. CLegacyFmtItem *itm1 = (CLegacyFmtItem *)*_itm1;
  5580. CLegacyFmtItem *itm2 = (CLegacyFmtItem *)*_itm2;
  5581. if (itm1->num==itm2->num) return 0;
  5582. if (itm1->num<itm2->num) return -1;
  5583. return 1;
  5584. }
  5585. void CCovenSDSManager::loadStore(const char *storeName, const bool *abort)
  5586. {
  5587. if (root) root->Release();
  5588. class CNodeCreate : implements IPTreeNodeCreator, public CInterface
  5589. {
  5590. public:
  5591. IMPLEMENT_IINTERFACE;
  5592. virtual IPropertyTree *create(const char *tag) { return createServerTree(tag); }
  5593. } nodeCreator;
  5594. class CSDSTreeMaker : public CPTreeMaker
  5595. {
  5596. public:
  5597. CSDSTreeMaker(IPTreeNodeCreator *nodeCreator) : CPTreeMaker(ipt_none, nodeCreator) { }
  5598. virtual void endNode(const char *tag, unsigned length, const void *value, bool binary, offset_t endOffset)
  5599. {
  5600. IPropertyTree *node = queryCurrentNode();
  5601. CPTreeMaker::endNode(tag, length, value, binary, endOffset);
  5602. if (((CServerRemoteTree *)node)->testExternalCandidate())
  5603. convertQueue.append(*(CServerRemoteTree *)node);
  5604. }
  5605. ICopyArrayOf<CServerRemoteTree> convertQueue;
  5606. } treeMaker(&nodeCreator);
  5607. Owned<IPropertyTree> oldEnvironment;
  5608. try
  5609. {
  5610. bool saveNeeded = false;
  5611. if (!storeName)
  5612. storeName = config.queryProp("@store");
  5613. unsigned crc = 0;
  5614. StringBuffer storeFilename(dataPath);
  5615. iStoreHelper->getCurrentStoreFilename(storeFilename, &crc);
  5616. LOG(MCdebugInfo(100), unknownJob, "loading store %d, storedCrc=%x", iStoreHelper->queryCurrentEdition(), crc);
  5617. root = (CServerRemoteTree *)::loadStore(storeFilename.str(), &treeMaker, crc, false, abort);
  5618. if (!root)
  5619. {
  5620. StringBuffer s(storeName);
  5621. LOG(MCdebugInfo(100), unknownJob, "Store %d does not exist, creating new store", iStoreHelper->queryCurrentEdition());
  5622. root = new CServerRemoteTree("SDS");
  5623. }
  5624. bool errors;
  5625. Owned<IException> deltaE;
  5626. try { iStoreHelper->loadDeltas(root, &errors); }
  5627. catch (IException *e) { deltaE.setown(e); errors = true; }
  5628. if (errors && config.getPropBool("@backupErrorFiles", true))
  5629. {
  5630. iStoreHelper->backup(storeFilename.str());
  5631. if (deltaE.get())
  5632. throw LINK(deltaE);
  5633. }
  5634. LOG(MCdebugInfo(100), unknownJob, "store loaded");
  5635. const char *environment = config.queryProp("@environment");
  5636. if (environment && *environment)
  5637. {
  5638. LOG(MCdebugInfo(100), unknownJob, "loading external Environment from: %s", environment);
  5639. Owned<IFile> envFile = createIFile(environment);
  5640. if (!envFile->exists())
  5641. throw MakeStringException(0, "'%s' does not exist", environment);
  5642. OwnedIFileIO iFileIO = envFile->open(IFOread);
  5643. if (!iFileIO)
  5644. throw MakeStringException(0, "Failed to open '%s'", environment);
  5645. Owned<IPropertyTree> envTree = createPTreeFromXMLFile(environment);
  5646. if (0 != stricmp("Environment", envTree->queryName()))
  5647. throw MakeStringException(0, "External environment file '%s', has '%s' as root, expecting a 'Environment' xml node.", environment, envTree->queryName());
  5648. Owned <IMPServer> thisDali = getMPServer();
  5649. assertex(thisDali);
  5650. IPropertyTree *thisDaliInfo = findDaliProcess(envTree, thisDali->queryMyNode()->endpoint());
  5651. assertex(thisDaliInfo);
  5652. const char *daliName = thisDaliInfo->queryProp("@name");
  5653. if (daliName)
  5654. {
  5655. VStringBuffer xpath("Software/DaliServerPlugin[@daliServers='%s']", daliName);
  5656. Owned<IPropertyTreeIterator> plugins = envTree->getElements(xpath);
  5657. ForEach(*plugins)
  5658. {
  5659. Owned<IPluggableFactory> factory = loadPlugin(&plugins->query());
  5660. assertex (factory);
  5661. if (!factory->initializeStore())
  5662. throw MakeStringException(0, "Failed to initialize plugin store '%s'", plugins->query().queryProp("@type"));
  5663. }
  5664. }
  5665. oldEnvironment.setown(root->getPropTree("Environment"));
  5666. root->removeTree(oldEnvironment);
  5667. root->addPropTree("Environment", envTree.getClear());
  5668. externalEnvironment = true;
  5669. }
  5670. UInt64Array refExts;
  5671. PROGLOG("Scanning store for external references");
  5672. Owned<IPropertyTreeIterator> rootIter = root->getElements("//*");
  5673. ForEach (*rootIter)
  5674. {
  5675. IPropertyTree &tree = rootIter->query();
  5676. __int64 index = tree.getPropInt64(EXT_ATTR);
  5677. if (index)
  5678. refExts.append(index);
  5679. }
  5680. PROGLOG("External reference count = %d", refExts.ordinality());
  5681. refExts.sort(uint64compare);
  5682. if (refExts.ordinality())
  5683. nextExternal = refExts.tos()+1; // JCSMORE could keep array and fill gaps
  5684. // build list of primary, backup and legacy type external files
  5685. CIArrayOf<CLegacyFmtItem> legacyFmts;
  5686. UInt64Array primaryExts, backupExts;
  5687. unsigned l = strlen(EXTERNAL_NAME_PREFIX);
  5688. bool primary = true;
  5689. Owned<IDirectoryIterator> di = createDirectoryIterator(dataPath);
  5690. loop
  5691. {
  5692. try
  5693. {
  5694. ForEach(*di)
  5695. {
  5696. StringBuffer fname;
  5697. di->getName(fname);
  5698. if (fname.length() > l && 0 == memicmp(EXTERNAL_NAME_PREFIX, fname.str(), l))
  5699. {
  5700. StringBuffer name, ext;
  5701. splitFilename(fname, NULL, NULL, &name, &ext);
  5702. if (ext.length())
  5703. {
  5704. ext.remove(0, 1); // delete '.'
  5705. if (0 != stricmp(EF_BinaryValue, ext.str()))
  5706. {
  5707. unsigned __int64 num = atoi64(name.str()+l);
  5708. legacyFmts.append(* new CLegacyFmtItem(name.str(), ext.str(), num));
  5709. }
  5710. #ifndef _WIN32 // win32 files are case insensitive, so no point in this step.
  5711. StringBuffer lfName(fname);
  5712. lfName.toLowerCase();
  5713. if (0 != strcmp(fname.str(), lfName.str()))
  5714. di->query().rename(lfName.str());
  5715. #endif
  5716. unsigned __int64 extN = atoi64(name.str()+l);
  5717. if (primary)
  5718. primaryExts.append(extN);
  5719. else
  5720. backupExts.append(extN);
  5721. }
  5722. }
  5723. }
  5724. }
  5725. catch (IException *e)
  5726. {
  5727. if (primary)
  5728. throw;
  5729. EXCLOG(e, NULL);
  5730. e->Release();
  5731. }
  5732. if (!primary || 0 == remoteBackupLocation.length())
  5733. break;
  5734. di.setown(createDirectoryIterator(remoteBackupLocation));
  5735. primary = false;
  5736. }
  5737. primaryExts.sort(uint64compare);
  5738. backupExts.sort(uint64compare);
  5739. // Compare list of primary/backup externals against referenced externals and issue warnings
  5740. // Copy over any if reference and present either only in primary or backup
  5741. IExternalHandler *extHandler = queryExternalHandler(EF_BinaryValue);
  5742. primary = true;
  5743. UInt64Array missingPrimarys;
  5744. loop
  5745. {
  5746. UInt64Array &fileExts = primary ? primaryExts : backupExts;
  5747. unsigned __int64 refN = refExts.ordinality() ? refExts.item(0) : (unsigned __int64)-1;
  5748. unsigned __int64 fileN = fileExts.ordinality() ? fileExts.item(0) : (unsigned __int64)-1;
  5749. unsigned fileP=0;
  5750. unsigned refP=0;
  5751. unsigned missP=0;
  5752. while (fileN != ((unsigned __int64)-1) || refN != ((unsigned __int64)-1))
  5753. {
  5754. while (fileN == refN)
  5755. {
  5756. if (!primary)
  5757. {
  5758. bool found = false;
  5759. while (missP < missingPrimarys.ordinality())
  5760. {
  5761. unsigned __int64 missN = missingPrimarys.item(missP);
  5762. if (refN == missN)
  5763. {
  5764. ++missP;
  5765. found = true;
  5766. break;
  5767. }
  5768. if (missN > refN)
  5769. break;
  5770. ++missP;
  5771. }
  5772. // i.e. found in backup, but was listed missing in primary
  5773. if (found)
  5774. {
  5775. StringBuffer fname;
  5776. StringBuffer name(EXTERNAL_NAME_PREFIX);
  5777. name.append(refN);
  5778. extHandler->getFilename(fname, name.str());
  5779. PROGLOG("Copying missing primary external from backup: %s", fname.str());
  5780. StringBuffer backupFname(remoteBackupLocation);
  5781. extHandler->getName(backupFname, name.str());
  5782. try
  5783. {
  5784. copyFile(fname.str(), backupFname.str());
  5785. }
  5786. catch (IException *e)
  5787. {
  5788. EXCLOG(e, NULL);
  5789. e->Release();
  5790. }
  5791. }
  5792. }
  5793. ++refP;
  5794. ++fileP;
  5795. refN = (refP == refExts.ordinality()) ? (unsigned __int64)-1 : refExts.item(refP);
  5796. fileN = (fileP == fileExts.ordinality()) ? (unsigned __int64)-1 : fileExts.item(fileP);
  5797. if (fileN == ((unsigned __int64)-1) || refN == ((unsigned __int64)-1))
  5798. break;
  5799. }
  5800. while (fileN < refN)
  5801. {
  5802. StringBuffer fname;
  5803. StringBuffer name(EXTERNAL_NAME_PREFIX);
  5804. name.append(fileN);
  5805. extHandler->getName(fname, name.str());
  5806. WARNLOG("Unreferenced %s external %s file", primary?"primary":"backup", fname.str());
  5807. ++fileP;
  5808. if (fileP == fileExts.ordinality())
  5809. {
  5810. fileN = (unsigned __int64)-1;
  5811. break;
  5812. }
  5813. fileN = fileExts.item(fileP);
  5814. }
  5815. while (refN < fileN)
  5816. {
  5817. StringBuffer fname;
  5818. StringBuffer name(EXTERNAL_NAME_PREFIX);
  5819. name.append(refN);
  5820. extHandler->getName(fname, name.str());
  5821. WARNLOG("External %s file reference %s missing", primary?"primary":"backup", fname.str());
  5822. if (primary)
  5823. missingPrimarys.append(refN);
  5824. else
  5825. {
  5826. bool found = false;
  5827. while (missP < missingPrimarys.ordinality())
  5828. {
  5829. unsigned __int64 missN = missingPrimarys.item(missP);
  5830. if (refN == missN)
  5831. {
  5832. ++missP;
  5833. found = true;
  5834. break;
  5835. }
  5836. if (missN > refN)
  5837. break;
  5838. ++missP;
  5839. }
  5840. if (!found)
  5841. {
  5842. // i.e. not missing in primary, but missing in backup
  5843. StringBuffer fname;
  5844. StringBuffer name(EXTERNAL_NAME_PREFIX);
  5845. name.append(refN);
  5846. extHandler->getFilename(fname, name.str());
  5847. PROGLOG("Copying missing backup external from primary: %s", fname.str());
  5848. StringBuffer backupFname(remoteBackupLocation);
  5849. extHandler->getName(backupFname, name.str());
  5850. try
  5851. {
  5852. copyFile(backupFname.str(), fname.str());
  5853. }
  5854. catch (IException *e)
  5855. {
  5856. EXCLOG(e, NULL);
  5857. e->Release();
  5858. }
  5859. }
  5860. }
  5861. ++refP;
  5862. if (refP == refExts.ordinality())
  5863. {
  5864. refN = (unsigned __int64)-1;
  5865. break;
  5866. }
  5867. refN = refExts.item(refP);
  5868. }
  5869. }
  5870. if (!primary || 0 == remoteBackupLocation.length())
  5871. break;
  5872. if (missingPrimarys.ordinality())
  5873. missingPrimarys.sort(uint64compare);
  5874. primary = false;
  5875. }
  5876. // check any marked with legacy formats
  5877. if (legacyFmts.ordinality() && refExts.ordinality())
  5878. {
  5879. legacyFmts.sort(extNcompareFunc);
  5880. unsigned refP = 0;
  5881. unsigned __int64 refN = refExts.item(refP++);
  5882. bool done = false;
  5883. ForEachItemIn(l, legacyFmts)
  5884. {
  5885. CLegacyFmtItem &itm = legacyFmts.item(l);
  5886. while (refN<itm.num)
  5887. {
  5888. if (refP == refExts.ordinality())
  5889. {
  5890. done = true;
  5891. break;
  5892. }
  5893. refN = refExts.item(refP++);
  5894. }
  5895. if (done)
  5896. break;
  5897. if (refN == itm.num)
  5898. {
  5899. IExternalHandler *extHandler = queryExternalHandler(itm.ext);
  5900. if (!extHandler)
  5901. {
  5902. WARNLOG("Unknown external extension, external=%s, extension=%s", itm.name.get(), itm.ext.get());
  5903. continue;
  5904. }
  5905. StringBuffer fname;
  5906. extHandler->getFilename(fname, itm.name);
  5907. PROGLOG("Converting legacy external type(%s) to new, file=%s", itm.ext.get(), fname.str());
  5908. MemoryBuffer mb;
  5909. Owned<IPropertyTree> tree = createPTree("tmp");
  5910. extHandler->read(itm.name, *tree, mb, true);
  5911. mb.append(""); // no children
  5912. tree.setown(createPTree(mb));
  5913. IExternalHandler *extHandlerNew = queryExternalHandler(EF_BinaryValue);
  5914. extHandlerNew->write(itm.name, *tree);
  5915. extHandler->remove(itm.name);
  5916. }
  5917. }
  5918. }
  5919. root->removeProp("External"); // remove legacy /External if present
  5920. unsigned items = treeMaker.convertQueue.ordinality();
  5921. if (items)
  5922. {
  5923. LOG(MCdebugInfo(100), unknownJob, "Converting %d items larger than threshold size %d, to external definitions", items, externalSizeThreshold);
  5924. ForEachItemIn(i, treeMaker.convertQueue)
  5925. SDSManager->writeExternal(treeMaker.convertQueue.item(i), true);
  5926. saveNeeded = true;
  5927. }
  5928. if (saveNeeded)
  5929. {
  5930. LOG(MCdebugInfo(100), unknownJob, "Saving converted store");
  5931. SDSManager->saveStore();
  5932. }
  5933. }
  5934. catch (IException *e)
  5935. {
  5936. LOG(MCoperatorError, unknownJob, e, "Exception - Failed to load main store");
  5937. throw;
  5938. }
  5939. catch (DALI_CATCHALL)
  5940. {
  5941. LOG(MCoperatorError, unknownJob, "Unknown exception - Failed to load main store");
  5942. throw;
  5943. }
  5944. if (!root)
  5945. {
  5946. root = (CServerRemoteTree *) createServerTree();
  5947. root->setName("root");
  5948. }
  5949. if (remoteBackupLocation.length())
  5950. {
  5951. try { validateBackup(); }
  5952. catch (IException *e) { LOG(MCoperatorError, unknownJob, e, "Validating backup"); e->Release(); }
  5953. StringBuffer deltaFilename(dataPath);
  5954. iStoreHelper->getCurrentDeltaFilename(deltaFilename);
  5955. OwnedIFile iFileDelta = createIFile(deltaFilename.str());
  5956. deltaFilename.clear().append(remoteBackupLocation);
  5957. iStoreHelper->getCurrentDeltaFilename(deltaFilename);
  5958. OwnedIFile iFileDeltaBackup = createIFile(deltaFilename.str());
  5959. CDateTime modifiedTime;
  5960. if (!iFileDelta->exists() && !iFileDeltaBackup->exists())
  5961. doTimeComparison = true;
  5962. else if (iFileDelta->getTime(NULL, &modifiedTime, NULL))
  5963. {
  5964. if (iFileDeltaBackup->setTime(NULL, &modifiedTime, NULL))
  5965. doTimeComparison = true;
  5966. }
  5967. if (!doTimeComparison)
  5968. LOG(MCoperatorWarning, unknownJob, "Unable to use time comparison when comparing delta backup file");
  5969. }
  5970. Owned<IRemoteConnection> conn = connect("/", 0, RTM_INTERNAL, INFINITE);
  5971. initializeInternals(conn->queryRoot());
  5972. conn.clear();
  5973. bool forceGroupUpdate = config.getPropBool("DFS/@forceGroupUpdate");
  5974. StringBuffer response;
  5975. initClusterGroups(forceGroupUpdate, response, oldEnvironment);
  5976. if (response.length())
  5977. PROGLOG("DFS group initialization : %s", response.str()); // should this be a syslog?
  5978. }
  5979. void CCovenSDSManager::saveStore(const char *storeName, bool currentEdition)
  5980. {
  5981. struct CIgnore
  5982. {
  5983. CIgnore() { SDSManager->ignoreExternals=true; }
  5984. ~CIgnore() { SDSManager->ignoreExternals=false; }
  5985. } ignore;
  5986. iStoreHelper->saveStore(root, NULL, currentEdition);
  5987. unsigned initNodeTableSize = allNodes.maxElements()+OVERFLOWSIZE;
  5988. queryCoven().setInitSDSNodes(initNodeTableSize>INIT_NODETABLE_SIZE?initNodeTableSize:INIT_NODETABLE_SIZE);
  5989. }
  5990. CServerRemoteTree *CCovenSDSManager::queryRegisteredTree(__int64 uniqId)
  5991. {
  5992. CHECKEDCRITICALBLOCK(treeRegCrit, fakeCritTimeout);
  5993. return (CServerRemoteTree *)allNodes.queryElem(uniqId);
  5994. }
  5995. CServerRemoteTree *CCovenSDSManager::getRegisteredTree(__int64 uniqId)
  5996. {
  5997. CHECKEDCRITICALBLOCK(treeRegCrit, fakeCritTimeout);
  5998. return (CServerRemoteTree *)allNodes.getElem(uniqId);
  5999. }
  6000. CServerRemoteTree *CCovenSDSManager::queryRoot()
  6001. {
  6002. return root;
  6003. }
  6004. StringBuffer &transformToAbsolute(StringBuffer &result, const char *xpath, unsigned index)
  6005. {
  6006. const char *end = xpath+strlen(xpath);
  6007. const char *p = end;
  6008. const char *q = NULL;
  6009. loop
  6010. {
  6011. if (p == xpath)
  6012. {
  6013. p = end;
  6014. break;
  6015. }
  6016. --p;
  6017. if ('/' == *p)
  6018. {
  6019. p = end;
  6020. break;
  6021. }
  6022. else if ('[' == *p)
  6023. {
  6024. q = p;
  6025. break;
  6026. }
  6027. }
  6028. if (q)
  6029. result.append(p-xpath, xpath);
  6030. else
  6031. result.append(xpath);
  6032. result.append('[');
  6033. result.append(index);
  6034. result.append(']');
  6035. return result;
  6036. }
  6037. void cleanChangeTree(IPropertyTree &tree)
  6038. {
  6039. tree.removeProp("@id");
  6040. Owned<IPropertyTreeIterator> iter = tree.getElements(RENAME_TAG);
  6041. ForEach (*iter)
  6042. iter->query().removeProp("@id");
  6043. iter.setown(tree.getElements(DELETE_TAG));
  6044. ForEach (*iter)
  6045. iter->query().removeProp("@id");
  6046. iter.setown(tree.getElements(RESERVED_CHANGE_NODE));
  6047. ForEach (*iter)
  6048. cleanChangeTree(iter->query());
  6049. }
  6050. void CCovenSDSManager::saveDelta(const char *path, IPropertyTree &changeTree)
  6051. {
  6052. CHECKEDCRITICALBLOCK(saveIncCrit, fakeCritTimeout);
  6053. // translate changeTree to inc format (e.g. remove id's)
  6054. if (externalEnvironment)
  6055. {
  6056. // don't save any changed to /Environment if external
  6057. if (0 == strncmp("/Environment", path, strlen("/Environment")))
  6058. {
  6059. WARNLOG("Attempt to change read-only Dali environment, path = %s", path);
  6060. return;
  6061. }
  6062. if (0 == strcmp("/", path) && changeTree.hasProp("*[@name=\"Environment\"]"))
  6063. {
  6064. WARNLOG("Attempt to change read-only Dali environment, path = %s", path);
  6065. return;
  6066. }
  6067. }
  6068. cleanChangeTree(changeTree);
  6069. // write out with header details (e.g. path)
  6070. Owned<IPropertyTree> header = createPTree("Header");
  6071. header->setProp("@path", path);
  6072. IPropertyTree *delta = header->addPropTree("Delta", createPTree());
  6073. delta->addPropTree(changeTree.queryName(), LINK(&changeTree));
  6074. StringBuffer fname(dataPath);
  6075. OwnedIFile deltaIPIFile = createIFile(fname.append(DELTAINPROGRESS).str());
  6076. OwnedIFileIO deltaIPIFileIO = deltaIPIFile->open(IFOcreate);
  6077. deltaIPIFileIO.clear();
  6078. struct RemoveDIPBlock
  6079. {
  6080. IFile &iFile;
  6081. bool done;
  6082. void doit() { done = true; iFile.remove(); }
  6083. RemoveDIPBlock(IFile &_iFile) : iFile(_iFile), done(false) { }
  6084. ~RemoveDIPBlock () { if (!done) doit(); }
  6085. } removeDIP(*deltaIPIFile);
  6086. StringBuffer detachIPStr(dataPath);
  6087. OwnedIFile detachIPIFile = createIFile(detachIPStr.append(DETACHINPROGRESS).str());
  6088. if (detachIPIFile->exists()) // very small window where this can happen.
  6089. {
  6090. // implies other operation about to access current delta
  6091. // CHECK session is really alive, otherwise it has been orphaned, so remove it.
  6092. try
  6093. {
  6094. SessionId sessId = 0;
  6095. OwnedIFileIO detachIPIO = detachIPIFile->open(IFOread);
  6096. if (detachIPIO)
  6097. {
  6098. size_t s = detachIPIO->read(0, sizeof(sessId), &sessId);
  6099. detachIPIO.clear();
  6100. if (sizeof(sessId) == s)
  6101. {
  6102. // double check session is really alive
  6103. if (querySessionManager().sessionStopped(sessId, 0))
  6104. detachIPIFile->remove();
  6105. else
  6106. {
  6107. // *cannot block* because other op (sasha) accessing remote dali files, can access dali.
  6108. removeDIP.doit();
  6109. PROGLOG("blocked");
  6110. toXML(header, blockedDelta);
  6111. return;
  6112. }
  6113. }
  6114. }
  6115. }
  6116. catch (IException *e) { EXCLOG(e, NULL); e->Release(); }
  6117. }
  6118. try
  6119. {
  6120. StringBuffer deltaFilename(dataPath);
  6121. iStoreHelper->getCurrentDeltaFilename(deltaFilename);
  6122. toXML(header, blockedDelta);
  6123. OwnedIFile iFile = createIFile(deltaFilename.str());
  6124. bool first = !iFile->exists() || 0 == iFile->size();
  6125. writeDelta(blockedDelta, *iFile);
  6126. if (remoteBackupLocation.length())
  6127. backupHandler.addDelta(blockedDelta, iStoreHelper->queryCurrentEdition(), first);
  6128. else
  6129. {
  6130. if (blockedDelta.length() > 0x100000)
  6131. blockedDelta.kill();
  6132. else
  6133. blockedDelta.clear();
  6134. }
  6135. }
  6136. catch (IException *e)
  6137. {
  6138. LOG(MCoperatorError, unknownJob, e, "saveDelta");
  6139. e->Release();
  6140. }
  6141. }
  6142. CSubscriberContainerList *CCovenSDSManager::getSubscribers(const char *xpath, CPTStack &stack)
  6143. {
  6144. return subscribers.getQualifiedList(xpath, stack);
  6145. }
  6146. inline void serverToClientTree(CServerRemoteTree &src, CClientRemoteTree &dst)
  6147. {
  6148. if (src.getPropInt64(EXT_ATTR))
  6149. {
  6150. MemoryBuffer mb;
  6151. src.serializeSelfRT(mb, false);
  6152. dst.deserializeSelfRT(mb);
  6153. }
  6154. else
  6155. dst.clone(src, true, false);
  6156. dst.setServerId(src.queryServerId());
  6157. if (src.hasChildren()) dst.addServerTreeInfo(STI_HaveChildren);
  6158. }
  6159. class CMultipleConnector : implements IMultipleConnector, public CInterface
  6160. {
  6161. StringArray xpaths;
  6162. UnsignedArray modes;
  6163. public:
  6164. IMPLEMENT_IINTERFACE;
  6165. CMultipleConnector() { }
  6166. CMultipleConnector(MemoryBuffer &src)
  6167. {
  6168. unsigned c;
  6169. src.read(c);
  6170. xpaths.ensure(c);
  6171. modes.ensure(c);
  6172. while (c--)
  6173. {
  6174. StringAttr xpath;
  6175. unsigned mode;
  6176. src.read(xpath);
  6177. src.read(mode);
  6178. xpaths.append(xpath);
  6179. modes.append(mode);
  6180. }
  6181. }
  6182. // IMultipleConnector impl.
  6183. virtual void addConnection(const char *xpath, unsigned mode)
  6184. {
  6185. if (RTM_CREATE == (mode & RTM_CREATE_MASK) || RTM_CREATE_QUERY == (mode & RTM_CREATE_MASK))
  6186. throw MakeSDSException(SDSExcpt_BadMode, "multiple connections do not support creation modes");
  6187. xpaths.append(xpath);
  6188. modes.append(mode);
  6189. }
  6190. virtual unsigned queryConnections() { return xpaths.ordinality(); }
  6191. virtual void getConnectionDetails(unsigned which, StringAttr &xpath, unsigned &mode)
  6192. {
  6193. xpath.set(xpaths.item(which));
  6194. mode = modes.item(which);
  6195. }
  6196. virtual void serialize(MemoryBuffer &dst)
  6197. {
  6198. unsigned c=xpaths.ordinality();
  6199. dst.append(c);
  6200. while (c--)
  6201. dst.append(xpaths.item(c)).append(modes.item(c));
  6202. }
  6203. };
  6204. IMultipleConnector *deserializeIMultipleConnector(MemoryBuffer &src)
  6205. {
  6206. return new CMultipleConnector(src);
  6207. }
  6208. IMultipleConnector *createIMultipleConnector()
  6209. {
  6210. return new CMultipleConnector();
  6211. }
  6212. StringBuffer &getMConnectString(IMultipleConnector *mConnect, StringBuffer &s)
  6213. {
  6214. unsigned c;
  6215. for (c=0; c<mConnect->queryConnections(); c++)
  6216. {
  6217. StringAttr xpath;
  6218. unsigned mode;
  6219. mConnect->getConnectionDetails(c, xpath, mode);
  6220. s.append("xpath=\"").append(xpath).append("\" [mode=").append(mode).append("]");
  6221. if (c != mConnect->queryConnections()-1)
  6222. s.append(", ");
  6223. }
  6224. return s;
  6225. }
  6226. // ISDSManager impl.
  6227. IRemoteConnections *CCovenSDSManager::connect(IMultipleConnector *mConnect, SessionId id, unsigned timeout)
  6228. {
  6229. Owned<CLCLockBlock> lockBlock;
  6230. lockBlock.setown(new CLCReadLockBlock(dataRWLock, readWriteTimeout, __FILE__, __LINE__));
  6231. Owned<CRemoteConnections> remoteConnections = new CRemoteConnections;
  6232. unsigned c;
  6233. for (c=0; c<mConnect->queryConnections(); c++)
  6234. {
  6235. StringAttr xpath;
  6236. unsigned mode;
  6237. mConnect->getConnectionDetails(c, xpath, mode);
  6238. // connect can return NULL.
  6239. remoteConnections->add(connect(xpath, id, mode, timeout));
  6240. }
  6241. return LINK(remoteConnections);
  6242. }
  6243. IRemoteConnection *CCovenSDSManager::connect(const char *xpath, SessionId id, unsigned mode, unsigned timeout)
  6244. {
  6245. Owned<CLCLockBlock> lockBlock;
  6246. Owned<LinkingCriticalBlock> connectCritBlock;
  6247. if (!RTM_MODE(mode, RTM_INTERNAL))
  6248. {
  6249. connectCritBlock.setown(new LinkingCriticalBlock(connectCrit, __FILE__, __LINE__));
  6250. if (RTM_CREATE == (mode & RTM_CREATE_MASK) || RTM_CREATE_QUERY == (mode & RTM_CREATE_MASK))
  6251. lockBlock.setown(new CLCWriteLockBlock(dataRWLock, readWriteTimeout, __FILE__, __LINE__));
  6252. else
  6253. lockBlock.setown(new CLCReadLockBlock(dataRWLock, readWriteTimeout, __FILE__, __LINE__));
  6254. }
  6255. CServerRemoteTree *_tree;
  6256. Owned<CServerRemoteTree> tree;
  6257. ConnectionId connectionId = 0;
  6258. createConnection(id, mode, timeout, xpath, _tree, connectionId, true, connectCritBlock);
  6259. if (connectionId)
  6260. tree.setown(_tree);
  6261. connectCritBlock.clear();
  6262. if (connectionId)
  6263. {
  6264. CRemoteConnection *conn = new CRemoteConnection(*this, connectionId, xpath, id, mode, timeout);
  6265. assertex(conn);
  6266. CDisableFetchChangeBlock block(*conn);
  6267. CClientRemoteTree *clientTree = new CClientRemoteTree(*conn);
  6268. assertex(clientTree);
  6269. serverToClientTree(*tree, *clientTree);
  6270. conn->setRoot(clientTree);
  6271. return conn;
  6272. }
  6273. return NULL;
  6274. }
  6275. SubscriptionId CCovenSDSManager::subscribe(const char *xpath, ISDSSubscription &notify, bool sub, bool sendValue)
  6276. {
  6277. assertex(xpath);
  6278. if (sub && sendValue)
  6279. throw MakeSDSException(SDSExcpt_Unsupported, "Subscription to sub elements, with sendValue option unsupported");
  6280. StringBuffer s;
  6281. if ('/' != *xpath)
  6282. {
  6283. s.append('/').append(xpath);
  6284. xpath = s.str();
  6285. }
  6286. CSDSSubscriberProxy *subscriber = new CSDSSubscriberProxy(xpath, sub, sendValue, notify);
  6287. querySubscriptionManager(SDS_PUBLISHER)->add(subscriber, subscriber->getId());
  6288. return subscriber->getId();
  6289. }
  6290. SubscriptionId CCovenSDSManager::subscribeExact(const char *xpath, ISDSNodeSubscription &notify, bool sendValue)
  6291. {
  6292. assertex(xpath);
  6293. StringBuffer s;
  6294. if ('/' != *xpath)
  6295. {
  6296. s.append('/').append(xpath);
  6297. xpath = s.str();
  6298. }
  6299. CSDSNodeSubscriberProxy *subscriber = new CSDSNodeSubscriberProxy(xpath, sendValue, notify);
  6300. querySubscriptionManager(SDSNODE_PUBLISHER)->add(subscriber, subscriber->getId());
  6301. return subscriber->getId();
  6302. }
  6303. void CCovenSDSManager::unsubscribe(SubscriptionId id)
  6304. {
  6305. querySubscriptionManager(SDS_PUBLISHER)->remove(id);
  6306. }
  6307. void CCovenSDSManager::unsubscribeExact(SubscriptionId id)
  6308. {
  6309. querySubscriptionManager(SDSNODE_PUBLISHER)->remove(id);
  6310. }
  6311. bool CCovenSDSManager::removeNotifyHandler(const char *handlerKey)
  6312. {
  6313. return nodeNotifyHandlers.remove(handlerKey);
  6314. }
  6315. IPropertyTree *CCovenSDSManager::lockStoreRead() const
  6316. {
  6317. PROGLOG("lockStoreRead() called");
  6318. CHECKEDREADLOCKENTER(dataRWLock, readWriteTimeout);
  6319. return root;
  6320. }
  6321. void CCovenSDSManager::unlockStoreRead() const
  6322. {
  6323. PROGLOG("unlockStoreRead() called");
  6324. dataRWLock.unlockRead();
  6325. }
  6326. bool CCovenSDSManager::setSDSDebug(StringArray &params, StringBuffer &reply)
  6327. {
  6328. if (0 == params.ordinality()) return false;
  6329. else if (0 == stricmp("datalockHoldTiming", params.item(0)))
  6330. {
  6331. if (params.ordinality()<2)
  6332. {
  6333. reply.append("datalockHoldTiming currently = ").append(readWriteSlowTracing);
  6334. return false;
  6335. }
  6336. unsigned ms = atoi(params.item(1));
  6337. readWriteSlowTracing = ms;
  6338. PROGLOG("datalock, readWriteSlowTracing timing set to %d", readWriteSlowTracing);
  6339. }
  6340. else if (0 == stricmp("datalockHoldStack", params.item(0)))
  6341. {
  6342. if (params.ordinality()<2)
  6343. {
  6344. reply.append("datalockHoldStack currently set to '").append(readWriteStackTracing?"on":"off").append("'");
  6345. return false;
  6346. }
  6347. readWriteStackTracing = (0 == stricmp("on", params.item(1)));
  6348. PROGLOG("datalock, held time stacks set to '%s'", readWriteStackTracing?"on":"off");
  6349. }
  6350. else if (0 == stricmp("datalockRetryStackTiming", params.item(0)))
  6351. {
  6352. if (params.ordinality()<2)
  6353. {
  6354. reply.append("datalockRetryStackTiming currently =").append(readWriteTimeout);
  6355. return false;
  6356. }
  6357. unsigned ms = atoi(params.item(1));
  6358. readWriteTimeout = ms;
  6359. PROGLOG("datalock, readWriteTimeout timing set to %s", readWriteStackTracing?"on":"off");
  6360. }
  6361. else if (0 == stricmp("fakecritTiming", params.item(0)))
  6362. {
  6363. if (params.ordinality()<2)
  6364. {
  6365. reply.append("fakeCritTimeout currently =").append(fakeCritTimeout);
  6366. return false;
  6367. }
  6368. unsigned ms = atoi(params.item(1));
  6369. fakeCritTimeout = ms;
  6370. PROGLOG("fakecrit, fakeCritTimeout timing set to %d", fakeCritTimeout);
  6371. }
  6372. else
  6373. {
  6374. reply.append("Unknown command");
  6375. return false;
  6376. }
  6377. return true;
  6378. }
  6379. void CCovenSDSManager::saveRequest()
  6380. {
  6381. blockingSave();
  6382. }
  6383. IPropertyTree &CCovenSDSManager::queryProperties() const
  6384. {
  6385. return *properties;
  6386. }
  6387. void CCovenSDSManager::installNotifyHandler(const char *handlerKey, ISDSNotifyHandler *handler)
  6388. {
  6389. nodeNotifyHandlers.replace(* new CSDSNotifyHandlerMapping(handlerKey, *handler));
  6390. }
  6391. // ISDSConnectionManager impl.
  6392. void CCovenSDSManager::commit(CRemoteConnection &connection, bool *disconnectDeleteRoot)
  6393. {
  6394. Owned<CLCWriteLockBlock> lockBlock;
  6395. if (!RTM_MODE(connection.queryMode(), RTM_INTERNAL))
  6396. lockBlock.setown(new CLCWriteLockBlock(dataRWLock, readWriteTimeout, __FILE__, __LINE__));
  6397. CClientRemoteTree *tree = (CClientRemoteTree *) connection.queryRoot();
  6398. bool lazyFetch = connection.setLazyFetch(false);
  6399. Owned<IPropertyTree> changeTree = tree->collateData();
  6400. connection.setLazyFetch(lazyFetch);
  6401. if (NULL == disconnectDeleteRoot && !changeTree) return;
  6402. ConnectionId connectionId = connection.queryConnectionId();
  6403. CServerConnection *serverConnection = queryConnection(connectionId);
  6404. if (!serverConnection)
  6405. throw MakeSDSException(SDSExcpt_ConnectionAbsent, " [commit]");
  6406. try
  6407. {
  6408. CServerRemoteTree *serverTree = (CServerRemoteTree *)serverConnection->queryRoot();
  6409. assertex(serverTree);
  6410. MemoryBuffer newIds, inc;
  6411. if (changeTree && serverTree->processData(*serverConnection, *changeTree, newIds))
  6412. { // something commited, if RTM_Create was used need to remember this.
  6413. StringBuffer path;
  6414. serverConnection->queryPTreePath().getAbsolutePath(path);
  6415. saveDelta(path.str(), *changeTree);
  6416. bool lazyFetch = connection.setLazyFetch(false);
  6417. tree->clearCommitChanges(&newIds);
  6418. assertex(newIds.getPos() == newIds.length()); // must have read it all
  6419. connection.setLazyFetch(lazyFetch);
  6420. }
  6421. }
  6422. catch (IException *)
  6423. {
  6424. if (disconnectDeleteRoot)
  6425. {
  6426. connection.setConnected(false);
  6427. disconnect(connectionId, *disconnectDeleteRoot);
  6428. }
  6429. throw;
  6430. }
  6431. if (disconnectDeleteRoot)
  6432. {
  6433. connection.setConnected(false);
  6434. disconnect(connectionId, *disconnectDeleteRoot);
  6435. }
  6436. }
  6437. CRemoteTreeBase *CCovenSDSManager::get(CRemoteConnection &connection, __int64 serverId)
  6438. {
  6439. Owned<CLCReadLockBlock> lockBlock;
  6440. if (!RTM_MODE(connection.queryMode(), RTM_INTERNAL))
  6441. lockBlock.setown(new CLCReadLockBlock(dataRWLock, readWriteTimeout, __FILE__, __LINE__));
  6442. CDisableFetchChangeBlock block(connection);
  6443. CRemoteTreeBase *connectionRoot = (CRemoteTreeBase *) connection.queryRoot();
  6444. Owned<CServerRemoteTree> tree = getRegisteredTree(connectionRoot->queryServerId());
  6445. if (!tree)
  6446. return NULL;
  6447. CClientRemoteTree *newTree = new CClientRemoteTree(connection);
  6448. serverToClientTree(*tree, *newTree);
  6449. return newTree;
  6450. }
  6451. void CCovenSDSManager::getChildren(CRemoteTreeBase &parent, CRemoteConnection &connection, unsigned levels)
  6452. {
  6453. Owned<CLCReadLockBlock> lockBlock;
  6454. if (!RTM_MODE(connection.queryMode(), RTM_INTERNAL))
  6455. lockBlock.setown(new CLCReadLockBlock(dataRWLock, readWriteTimeout, __FILE__, __LINE__));
  6456. CDisableFetchChangeBlock block(connection);
  6457. Owned<CServerRemoteTree> serverParent = (CServerRemoteTree *)getRegisteredTree(parent.queryServerId());
  6458. if (serverParent)
  6459. _getChildren(parent, *serverParent, connection, levels);
  6460. }
  6461. void CCovenSDSManager::getChildrenFor(CRTArray &childLessList, CRemoteConnection &connection, unsigned levels)
  6462. {
  6463. Owned<CLCReadLockBlock> lockBlock;
  6464. if (!RTM_MODE(connection.queryMode(), RTM_INTERNAL))
  6465. lockBlock.setown(new CLCReadLockBlock(dataRWLock, readWriteTimeout, __FILE__, __LINE__));
  6466. CDisableFetchChangeBlock block(connection);
  6467. ForEachItemIn(f, childLessList)
  6468. {
  6469. CRemoteTreeBase &tree = childLessList.item(f);
  6470. Owned<CServerRemoteTree> serverParent = (CServerRemoteTree *)getRegisteredTree(tree.queryServerId());
  6471. if (serverParent)
  6472. _getChildren(tree, *serverParent, connection, levels);
  6473. }
  6474. }
  6475. static void addServerChildren(CClientRemoteTree &clientParent, CServerRemoteTree &serverParent, bool recurse)
  6476. {
  6477. Owned<IPropertyTreeIterator> iter = serverParent.getElements("*");
  6478. ForEach (*iter)
  6479. {
  6480. CServerRemoteTree &serverChild = (CServerRemoteTree &)iter->query();
  6481. CClientRemoteTree *clientChild = (CClientRemoteTree *)clientParent.create(NULL);
  6482. serverToClientTree(serverChild, *clientChild);
  6483. clientChild = (CClientRemoteTree *)clientParent.addPropTree(clientChild->queryName(), clientChild);
  6484. if (recurse)
  6485. addServerChildren(*clientChild, serverChild, recurse);
  6486. }
  6487. }
  6488. void CCovenSDSManager::matchServerTree(CClientRemoteTree *local, IPropertyTree &matchTree, bool allTail)
  6489. {
  6490. Owned<IPropertyTreeIterator> matchIter = matchTree.getElements("*");
  6491. if (matchIter->first())
  6492. {
  6493. if (local->hasChildren() && NULL == local->queryChildren())
  6494. {
  6495. local->createChildMap();
  6496. Owned<CServerRemoteTree> tree = getRegisteredTree(matchTree.getPropInt64("@serverId"));
  6497. addServerChildren(*local, *tree, false);
  6498. }
  6499. do
  6500. {
  6501. IPropertyTree &elem = matchIter->query();
  6502. StringBuffer path(elem.queryName());
  6503. path.append('[').append(elem.getPropInt("@pos")).append(']');
  6504. CClientRemoteTree *child = (CClientRemoteTree *)local->queryPropTree(path.str());
  6505. if (child) // if not would imply some other thread deleted I think.
  6506. matchServerTree(child, elem, allTail);
  6507. }
  6508. while (matchIter->next());
  6509. }
  6510. else
  6511. {
  6512. if (local->hasChildren() && NULL == local->queryChildren())
  6513. {
  6514. local->createChildMap();
  6515. Owned<CServerRemoteTree> tree = getRegisteredTree(matchTree.getPropInt64("@serverId"));
  6516. addServerChildren(*local, *tree, allTail);
  6517. }
  6518. }
  6519. }
  6520. void CCovenSDSManager::ensureLocal(CRemoteConnection &connection, CRemoteTreeBase &_parent, IPropertyTree *serverMatchTree, IPTIteratorCodes flags)
  6521. {
  6522. CClientRemoteTree &parent = (CClientRemoteTree &)_parent;
  6523. bool getLeaves = iptiter_remotegetbranch == (flags & iptiter_remotegetbranch);
  6524. CDisableFetchChangeBlock block(connection);
  6525. matchServerTree(&parent, *serverMatchTree, getLeaves);
  6526. }
  6527. void CCovenSDSManager::_getChildren(CRemoteTreeBase &parent, CServerRemoteTree &serverParent, CRemoteConnection &connection, unsigned levels)
  6528. {
  6529. Owned<IPropertyTreeIterator> iter = serverParent.getElements("*");
  6530. assertex(iter);
  6531. if (levels && serverParent.getPropBool("@fetchEntire"))
  6532. levels = 0;
  6533. ForEach(*iter)
  6534. {
  6535. CServerRemoteTree &child = (CServerRemoteTree &)iter->query();
  6536. CClientRemoteTree *newChild = new CClientRemoteTree(connection); // clone create
  6537. serverToClientTree(child, *newChild);
  6538. parent.addPropTree(newChild->queryName(), newChild);
  6539. if (0==levels || child.getPropBool("@fetchEntire"))
  6540. _getChildren(*newChild, child, connection, 0);
  6541. }
  6542. }
  6543. IPropertyTreeIterator *CCovenSDSManager::getElements(CRemoteConnection &connection, const char *xpath)
  6544. {
  6545. Owned<CLCReadLockBlock> lockBlock = new CLCReadLockBlock(dataRWLock, readWriteTimeout, __FILE__, __LINE__);
  6546. CDisableFetchChangeBlock block(connection);
  6547. Owned<CServerRemoteTree> serverConnRoot = (CServerRemoteTree *)getRegisteredTree(((CClientRemoteTree *)connection.queryRoot())->queryServerId());
  6548. Owned<CPTArrayIterator> elements = new CPTArrayIterator();
  6549. Owned<IPropertyTreeIterator> iter = serverConnRoot->getElements(xpath);
  6550. ForEach (*iter)
  6551. {
  6552. CServerRemoteTree &child = (CServerRemoteTree &)iter->query();
  6553. Owned<CClientRemoteTree> newChild = new CClientRemoteTree(connection);
  6554. serverToClientTree(child, *newChild);
  6555. elements->array.append(*LINK(newChild));
  6556. }
  6557. return LINK(elements);
  6558. }
  6559. void CCovenSDSManager::changeMode(CRemoteConnection &connection, unsigned mode, unsigned timeout, bool suppressReloads)
  6560. {
  6561. if (mode & RTM_CREATE_MASK)
  6562. throw MakeSDSException(SDSExcpt_BadMode, "calling changeMode");
  6563. ConnectionId connectionId = connection.queryConnectionId();
  6564. Linked<CServerConnection> serverConnection = queryConnection(connectionId);
  6565. if (!serverConnection)
  6566. throw MakeSDSException(SDSExcpt_ConnectionAbsent, " [changeMode]");
  6567. CServerRemoteTree *connectionRoot = (CServerRemoteTree *) serverConnection->queryRoot();
  6568. unsigned prevMode = connection.queryMode();
  6569. if (RTM_MODE(prevMode, RTM_LOCK_WRITE) && !RTM_MODE(mode, RTM_LOCK_WRITE))
  6570. commit(connection, NULL);
  6571. changeLockMode(*serverConnection, mode, timeout);
  6572. if (!suppressReloads)
  6573. {
  6574. if (RTM_MODE(mode, RTM_LOCK_WRITE) && !RTM_MODE(prevMode, RTM_LOCK_WRITE) && !RTM_MODE(prevMode, RTM_LOCK_READ))
  6575. connection.reload();
  6576. }
  6577. }
  6578. IPropertyTree *CCovenSDSManager::getXPaths(__int64 serverId, const char *xpath, bool getServerIds)
  6579. {
  6580. Owned<CServerRemoteTree> tree = getRegisteredTree(serverId);
  6581. if (!tree)
  6582. return NULL;
  6583. IPropertyTree *matchTree = getXPathMatchTree(*tree, xpath);
  6584. if (!matchTree)
  6585. return NULL;
  6586. if (getServerIds)
  6587. populateWithServerIds(matchTree, tree);
  6588. return matchTree;
  6589. }
  6590. IPropertyTreeIterator *CCovenSDSManager::getXPathsSortLimit(const char *baseXPath, const char *matchXPath, const char *sortBy, bool caseinsensitive, bool ascending, unsigned from, unsigned limit)
  6591. {
  6592. Owned<IPropertyTree> matchTree = getXPathsSortLimitMatchTree(baseXPath, matchXPath, sortBy, caseinsensitive, ascending, from, limit);
  6593. if (!matchTree)
  6594. return createNullPTreeIterator();
  6595. IPropertyTree *baseTree = SDSManager->queryRoot()->queryPropTree(baseXPath);
  6596. return new CXPathIterator(baseTree, matchTree, iptiter_null);
  6597. }
  6598. IPropertyTree *CCovenSDSManager::getXPathsSortLimitMatchTree(const char *baseXPath, const char *matchXPath, const char *sortby, bool caseinsensitive, bool ascending, unsigned from, unsigned limit)
  6599. {
  6600. UNIMPLEMENTED;
  6601. return NULL;
  6602. }
  6603. void CCovenSDSManager::getExternalValue(__int64 index, MemoryBuffer &mb)
  6604. {
  6605. IExternalHandler *handler = queryExternalHandler(EF_BinaryValue);
  6606. assertex(handler);
  6607. StringBuffer name(EXTERNAL_NAME_PREFIX);
  6608. name.append(index);
  6609. handler->readValue(name.str(), mb);
  6610. }
  6611. void CCovenSDSManager::getExternalValueFromServerId(__int64 serverId, MemoryBuffer &mb)
  6612. {
  6613. Owned<CServerRemoteTree> idTree = (CServerRemoteTree *) SDSManager->getRegisteredTree(serverId);
  6614. if (idTree)
  6615. {
  6616. CHECKEDCRITICALBLOCK(extCrit, fakeCritTimeout);
  6617. __int64 index = idTree->getPropInt64(EXT_ATTR);
  6618. if (index)
  6619. getExternalValue(index, mb);
  6620. else
  6621. WARNLOG("External file reference missing (node name='%s', id=%" I64F "d)", idTree->queryName(), serverId);
  6622. }
  6623. }
  6624. IPropertyTreeIterator *CCovenSDSManager::getElementsRaw(const char *xpath,INode *remotedali, unsigned timeout)
  6625. {
  6626. assertex(!remotedali); // only client side
  6627. CHECKEDDALIREADLOCKBLOCK(dataRWLock, readWriteTimeout);
  6628. return root->getElements(xpath);
  6629. }
  6630. void CCovenSDSManager::setConfigOpt(const char *opt, const char *value)
  6631. {
  6632. IPropertyTree &props = queryProperties();
  6633. if (props.hasProp(opt) && (0 == strcmp(value, props.queryProp(opt))))
  6634. return;
  6635. ensurePTree(&queryProperties(), opt);
  6636. queryProperties().setProp(opt, value);
  6637. }
  6638. unsigned CCovenSDSManager::queryCount(const char *xpath)
  6639. {
  6640. unsigned count = 0;
  6641. if (xpath && *xpath == '/')
  6642. ++xpath;
  6643. Owned<IPropertyTreeIterator> iter = root->getElements(xpath);
  6644. ForEach(*iter)
  6645. ++count;
  6646. return count;
  6647. }
  6648. void CCovenSDSManager::start()
  6649. {
  6650. server.start();
  6651. if (coalesce) coalesce->start();
  6652. }
  6653. void CCovenSDSManager::stop()
  6654. {
  6655. server.stop();
  6656. PROGLOG("waiting for coalescer to stop");
  6657. if (coalesce) coalesce->stop();
  6658. }
  6659. void CCovenSDSManager::restart(IException * e)
  6660. {
  6661. LOG(MCwarning, unknownJob, "-------: stopping SDS server");
  6662. StringBuffer msg;
  6663. msg.append("Unhandled exception, restarting: ").append(e->errorCode()).append(": ");
  6664. e->errorMessage(msg);
  6665. stop();
  6666. connections.kill();
  6667. LOG(MCwarning, unknownJob, "-------: stopped");
  6668. LOG(MCwarning, unknownJob, "-------: saving current store . . . . . .");
  6669. saveStore();
  6670. LOG(MCwarning, unknownJob, "-------: store saved.");
  6671. LOG(MCwarning, unknownJob, "-------: restarting SDS server restart");
  6672. start();
  6673. LOG(MCwarning, unknownJob, "-------: restarted");
  6674. }
  6675. CServerConnection *CCovenSDSManager::createConnectionInstance(CRemoteTreeBase *root, SessionId sessionId, unsigned mode, unsigned timeout, const char *xpath, CRemoteTreeBase *&tree, ConnectionId _connectionId, StringAttr *deltaPath, Owned<IPropertyTree> &deltaChange, Owned<CBranchChange> &branchChange, unsigned &additions)
  6676. {
  6677. IPropertyTree *parent = NULL;
  6678. ConnInfoFlags connInfoFlags = (ConnInfoFlags)0;
  6679. const char *_xpath = ('/' == *xpath)?xpath+1:xpath;
  6680. StringBuffer tXpath;
  6681. if (!RTM_MODE(mode, RTM_CREATE_UNIQUE))
  6682. {
  6683. unsigned l = strlen(_xpath);
  6684. if (l && '/' == _xpath[l-1])
  6685. {
  6686. tXpath.append(l-1, _xpath);
  6687. _xpath = tXpath.str();
  6688. }
  6689. }
  6690. bool newNode = false;
  6691. bool replaceNode = false;
  6692. IPropertyTree *created = NULL, *createdParent = NULL;
  6693. StringBuffer head;
  6694. if (mode & RTM_CREATE)
  6695. {
  6696. const char *prop = splitXPath(_xpath, head);
  6697. if (head.length())
  6698. {
  6699. try
  6700. {
  6701. parent = createPropBranch(root, head.str(), true, &created, &createdParent);
  6702. if (created)
  6703. newNode = true;
  6704. }
  6705. catch (IException *) { if (created) createdParent->removeTree(created); throw; }
  6706. catch (DALI_CATCHALL) { if (created) createdParent->removeTree(created); throw; }
  6707. }
  6708. else
  6709. parent = root->splitBranchProp(_xpath, prop);
  6710. if (parent)
  6711. {
  6712. connInfoFlags = ci_newParent;
  6713. StringBuffer uProp;
  6714. if (RTM_MODE(mode, RTM_CREATE_UNIQUE))
  6715. {
  6716. Owned<IPropertyTreeIterator> iter = parent->getElements(prop);
  6717. if (iter->first())
  6718. {
  6719. uProp.append(prop).append('-');
  6720. unsigned l = uProp.length();
  6721. unsigned n=1;
  6722. loop
  6723. {
  6724. n += getRandom() % 5; // better chance of finding a mismatch soon.
  6725. uProp.append(n);
  6726. iter.setown(parent->getElements(uProp.str()));
  6727. if (!iter->first())
  6728. break;
  6729. uProp.setLength(l);
  6730. }
  6731. prop = uProp.str();
  6732. if (head.length())
  6733. tXpath.append(head).append('/');
  6734. _xpath = tXpath.append(prop).str();
  6735. }
  6736. }
  6737. if (RTM_MODE(mode, RTM_CREATE_QUERY))
  6738. tree = (CRemoteTreeBase *) root->queryCreateBranch(parent, prop, &newNode);
  6739. else
  6740. {
  6741. IPropertyTree *newTree = ((CRemoteTreeBase *) parent)->create(NULL);
  6742. if (RTM_MODE(mode, RTM_CREATE_ADD))
  6743. tree = (CRemoteTreeBase *) parent->addPropTree(prop, newTree);
  6744. else // Default - RTM_CREATE - replace existing
  6745. {
  6746. Owned<IPropertyTreeIterator> iter = parent->getElements(prop);
  6747. if (iter->first())
  6748. replaceNode = true;
  6749. tree = (CRemoteTreeBase *) parent->setPropTree(prop, newTree);
  6750. }
  6751. newNode = true;
  6752. }
  6753. }
  6754. else
  6755. tree = NULL;
  6756. }
  6757. else
  6758. {
  6759. if (NULL == _xpath || '\0' == *_xpath)
  6760. tree = root;
  6761. else
  6762. {
  6763. StringBuffer path;
  6764. const char *prop = splitXPath(_xpath, path);
  6765. if (!prop) prop = _xpath;
  6766. // establish parent
  6767. tree = NULL;
  6768. Owned<IPropertyTreeIterator> iter = root->getElements(path.str());
  6769. ForEach (*iter)
  6770. {
  6771. IPropertyTree *_parent = &iter->query();
  6772. IPropertyTree *match = _parent->queryPropTree(prop);
  6773. if (tree)
  6774. {
  6775. if (match)
  6776. throw MakeSDSException(SDSExcpt_AmbiguousXpath, "Ambiguous: %s", _xpath);
  6777. }
  6778. else
  6779. {
  6780. parent = _parent;
  6781. tree = (CRemoteTreeBase *)match;
  6782. }
  6783. }
  6784. }
  6785. }
  6786. if (!tree)
  6787. return NULL;
  6788. assertex(tree->queryServerId());
  6789. ConnectionId connectionId = _connectionId;
  6790. if (!connectionId)
  6791. connectionId = coven.getUniqueId();
  6792. CServerConnection *connection = new CServerConnection(*this, connectionId, _xpath, sessionId, mode, timeout, parent, connInfoFlags);
  6793. Owned<LinkingCriticalBlock> b;
  6794. if (!RTM_MODE(mode, RTM_INTERNAL))
  6795. b.setown(new LinkingCriticalBlock(lockCrit, __FILE__, __LINE__));
  6796. connection->initPTreePath(*root, *tree);
  6797. if (newNode)
  6798. {
  6799. writeTransactions++;
  6800. // add tree into stack temporarily, or add manually at end.
  6801. deltaChange.setown(createPTree(RESERVED_CHANGE_NODE));
  6802. IPropertyTree *tail = deltaChange;
  6803. if (created) // some elems in "head" created
  6804. {
  6805. // iterate stack until createdParent to build up new head path.
  6806. StringBuffer _deltaPath;
  6807. unsigned s = 0;
  6808. StringBuffer headPath;
  6809. connection->queryPTreePath().getAbsolutePath(headPath);
  6810. if (headPath.length() && headPath.charAt(0) == '/')
  6811. headPath.remove(0, 1);
  6812. loop
  6813. {
  6814. _deltaPath.append('/');
  6815. IPropertyTree &tree = connection->queryPTreePath().item(s);
  6816. if (&tree == createdParent)
  6817. {
  6818. ++s;
  6819. break;
  6820. }
  6821. unsigned l = _deltaPath.length();
  6822. const char *t = queryHead(headPath.str(), _deltaPath);
  6823. assertex(l != _deltaPath.length());
  6824. headPath.clear();
  6825. if (t)
  6826. headPath.append(t);
  6827. if (++s>=connection->queryPTreePath().ordinality())
  6828. break;
  6829. }
  6830. additions = connection->queryPTreePath().ordinality()-s;
  6831. deltaPath->set(_deltaPath.str());
  6832. branchChange.setown(new CBranchChange(*(CRemoteTreeBase *)createdParent));
  6833. CBranchChange *topChange = branchChange;
  6834. // iterate remaining stack, iterate marking as 'new'
  6835. for (;s<connection->queryPTreePath().ordinality();s++)
  6836. {
  6837. IPropertyTree &tree = connection->queryPTreePath().item(s);
  6838. IPropertyTree *n = tail->addPropTree(RESERVED_CHANGE_NODE, createPTree());
  6839. n->setProp("@name", tree.queryName());
  6840. n->setPropBool("@new", true);
  6841. tail = n;
  6842. Owned<CBranchChange> childChange = new CBranchChange((CRemoteTreeBase &)tree);
  6843. childChange->noteChange(PDS_Added, PDS_Added);
  6844. topChange->addChildBranch(*LINK(childChange));
  6845. topChange = childChange;
  6846. }
  6847. }
  6848. else
  6849. {
  6850. StringBuffer s, _deltaPath;
  6851. connection->queryPTreePath().getAbsolutePath(s);
  6852. const char *t = splitXPath(s.str(), _deltaPath);
  6853. deltaPath->set(_deltaPath.str());
  6854. IPropertyTree *n = tail->addPropTree(RESERVED_CHANGE_NODE, createPTree());
  6855. n->setProp("@name", tree->queryName());
  6856. if (replaceNode)
  6857. n->setPropBool("@replace", true);
  6858. else
  6859. n->setPropBool("@new", true);
  6860. branchChange.setown(new CBranchChange(*(CRemoteTreeBase *)tree));
  6861. if (replaceNode)
  6862. branchChange->noteChange(PDS_Data, PDS_Data);
  6863. else
  6864. branchChange->noteChange(PDS_Added, PDS_Added);
  6865. additions=1;
  6866. }
  6867. }
  6868. connection->setRoot(LINK(tree));
  6869. return connection;
  6870. }
  6871. void CCovenSDSManager::clearSDSLocks()
  6872. {
  6873. CHECKEDCRITICALBLOCK(lockCrit, fakeCritTimeout);
  6874. SuperHashIteratorOf<CLock> iter(lockTable.queryBaseTable());
  6875. ICopyArrayOf<CLock> locks;
  6876. ForEach(iter)
  6877. locks.append(iter.query());
  6878. ForEachItemIn(l, locks)
  6879. locks.item(l).unlockAll();
  6880. }
  6881. void CCovenSDSManager::changeLockMode(CServerConnection &connection, unsigned newMode, unsigned timeout)
  6882. {
  6883. CServerRemoteTree *tree = (CServerRemoteTree *) connection.queryRoot();
  6884. ConnectionId connectionId = connection.queryConnectionId();
  6885. __int64 treeId = tree->queryServerId();
  6886. newMode = newMode & (RTM_LOCKBASIC_MASK|RTM_LOCK_SUB);
  6887. newMode |= connection.queryMode() & ~(RTM_LOCKBASIC_MASK|RTM_LOCK_SUB);
  6888. CUnlockCallback callback(connection.queryXPath(), connectionId, *tree);
  6889. {
  6890. CHECKEDCRITICALBLOCK(lockCrit, fakeCritTimeout);
  6891. CLock *lock = queryLock(treeId);
  6892. if (lock)
  6893. {
  6894. lock->changeMode(connectionId, connection.querySessionId(), newMode, timeout, callback);
  6895. connection.setMode(newMode);
  6896. return;
  6897. }
  6898. }
  6899. // no existing lock for connection
  6900. lock(*tree, connection.queryXPath(), connectionId, connection.querySessionId(), newMode, timeout, callback);
  6901. connection.setMode(newMode);
  6902. }
  6903. bool CCovenSDSManager::unlock(__int64 connectionId, bool close, StringBuffer &connectionInfo)
  6904. {
  6905. Owned<CServerConnection> connection = getConnection(connectionId);
  6906. if (!connection) return false;
  6907. MemoryBuffer connInfo;
  6908. connection->getInfo(connInfo);
  6909. formatConnectionInfo(connInfo, connectionInfo);
  6910. if (close)
  6911. {
  6912. PROGLOG("forcing unlock & disconnection of connection : %s", connectionInfo.str());
  6913. Owned<CLCLockBlock> lockBlock = new CLCWriteLockBlock(dataRWLock, readWriteTimeout, __FILE__, __LINE__);
  6914. SDSManager->disconnect(connectionId, false);
  6915. }
  6916. else // leave connection open, just unlock
  6917. {
  6918. PROGLOG("forcing unlock for connection : %s", connectionInfo.str());
  6919. __int64 nodeId = ((CRemoteTreeBase *)connection->queryRoot())->queryServerId();
  6920. CHECKEDCRITICALBLOCK(lockCrit, fakeCritTimeout);
  6921. CLock *lock = queryLock(nodeId);
  6922. if (lock)
  6923. lock->unlock(connectionId);
  6924. }
  6925. return true;
  6926. }
  6927. bool CCovenSDSManager::unlock(__int64 treeId, ConnectionId connectionId, bool delayDelete)
  6928. {
  6929. CHECKEDCRITICALBLOCK(lockCrit, fakeCritTimeout);
  6930. CLock *lock = queryLock(treeId);
  6931. if (lock)
  6932. return lock->unlock(connectionId, delayDelete);
  6933. return false;
  6934. }
  6935. void CCovenSDSManager::unlockAll(__int64 treeId)
  6936. {
  6937. CHECKEDCRITICALBLOCK(lockCrit, fakeCritTimeout);
  6938. CLock *lock = queryLock(treeId);
  6939. if (lock)
  6940. lock->unlockAll();
  6941. }
  6942. LockStatus CCovenSDSManager::establishLock(CLock &lock, __int64 treeId, ConnectionId connectionId, SessionId sessionId, unsigned mode, unsigned timeout, IUnlockCallback &lockCallback)
  6943. {
  6944. LockStatus res = lock.lock(mode, timeout, connectionId, sessionId, lockCallback);
  6945. if (res == LockSucceeded && server.queryStopped())
  6946. {
  6947. lock.unlock(connectionId);
  6948. throw MakeSDSException(SDSExcpt_ServerStoppedLockAborted);
  6949. }
  6950. return res;
  6951. }
  6952. void CCovenSDSManager::lock(CServerRemoteTree &tree, const char *xpath, ConnectionId connectionId, SessionId sessionId, unsigned mode, unsigned timeout, IUnlockCallback &callback)
  6953. {
  6954. if (0 == ((RTM_LOCK_READ | RTM_LOCK_WRITE) & mode)) // no point in creating lock.
  6955. return;
  6956. CLock *lock = NULL;
  6957. StringBuffer sxpath;
  6958. if ('/' != *xpath)
  6959. {
  6960. sxpath.append('/').append(xpath);
  6961. if ('/' != sxpath.charAt(sxpath.length()-1))
  6962. sxpath.append('/');
  6963. xpath = sxpath.str();
  6964. }
  6965. else if ('/' != xpath[strlen(xpath)-1])
  6966. {
  6967. sxpath.append(xpath);
  6968. sxpath.append('/');
  6969. xpath = sxpath.str();
  6970. }
  6971. __int64 treeId = tree.queryServerId();
  6972. CHECKEDCRITICALBLOCK(lockCrit, fakeCritTimeout);
  6973. lock = lockTable.find(&treeId);
  6974. if (!lock)
  6975. {
  6976. IdPath idPath;
  6977. lock = new CLock(lockTable, treeId, idPath, xpath, mode, connectionId, sessionId);
  6978. lockTable.replace(*lock);
  6979. }
  6980. else
  6981. {
  6982. Linked<CLock> tmp = lock; // keep it alive could be destroyed whilst blocked in call below.
  6983. LockStatus result = establishLock(*lock, treeId, connectionId, sessionId, mode, timeout, callback);
  6984. if (result != LockSucceeded)
  6985. {
  6986. if (!queryConnection(connectionId)) return; // connection aborted.
  6987. StringBuffer s;
  6988. switch (result)
  6989. {
  6990. case LockFailed:
  6991. throw MakeSDSException(SDSExcpt_ConnectionAbsent, "Lost connection trying to establish lock on connection to : %s", xpath);
  6992. case LockTimedOut:
  6993. throw MakeSDSException(SDSExcpt_LockTimeout, "Lock timeout trying to establish lock to %s, existing lock info: %s", xpath, lock->toString(s).str());
  6994. case LockHeld:
  6995. throw MakeSDSException(SDSExcpt_LockHeld, "Lock is held trying to establish lock to %s, existing lock info: %s", xpath, lock->toString(s).str());
  6996. }
  6997. }
  6998. }
  6999. }
  7000. void CCovenSDSManager::createConnection(SessionId sessionId, unsigned mode, unsigned timeout, const char *xpath, CServerRemoteTree *&tree, ConnectionId &connectionId, bool primary, Owned<LinkingCriticalBlock> &connectCritBlock)
  7001. {
  7002. CRemoteTreeBase *_tree;
  7003. Linked<CRemoteTreeBase> linkedTree;
  7004. Owned<CServerConnection> connection;
  7005. StringBuffer _xpath;
  7006. if (!xpath || '/'!=*xpath)
  7007. xpath = _xpath.append('/').append(xpath).str();
  7008. struct CFreeExistingLocks
  7009. {
  7010. ~CFreeExistingLocks() { clear(); }
  7011. void clear()
  7012. {
  7013. ForEachItemIn(l, existingLockTrees)
  7014. SDSManager->unlock(existingLockTrees.item(l).queryServerId(), connId);
  7015. existingLockTrees.kill();
  7016. }
  7017. void setConnectionId(ConnectionId _connId) { connId = _connId; }
  7018. void add(CServerRemoteTree &tree)
  7019. {
  7020. tree.Link();
  7021. existingLockTrees.append(tree);
  7022. }
  7023. void remove(CServerRemoteTree &tree)
  7024. {
  7025. existingLockTrees.zap(tree);
  7026. }
  7027. bool isExisting(CServerRemoteTree &tree) { return NotFound != existingLockTrees.find(tree); }
  7028. ConnectionId connId;
  7029. IArrayOf<CServerRemoteTree> existingLockTrees;
  7030. } freeExistingLocks;
  7031. StringAttr deltaPath;
  7032. Owned<IPropertyTree> deltaChange;
  7033. Owned<CBranchChange> branchChange;
  7034. unsigned additions = 0;
  7035. try
  7036. {
  7037. struct LockUnblock
  7038. {
  7039. LockUnblock(ReadWriteLock &_rWLock) : rWLock(_rWLock)
  7040. {
  7041. lockedForWrite = rWLock.queryWriteLocked();
  7042. if (lockedForWrite) rWLock.unlockWrite();
  7043. else rWLock.unlockRead();
  7044. }
  7045. ~LockUnblock() { if (lockedForWrite) rWLock.lockWrite(); else rWLock.lockRead(); }
  7046. bool lockedForWrite;
  7047. ReadWriteLock &rWLock;
  7048. };
  7049. bool locked = false;
  7050. class CConnectExistingLockCallback : implements IUnlockCallback
  7051. {
  7052. CUnlockCallback lcb;
  7053. CheckedCriticalSection *connectCrit;
  7054. public:
  7055. CConnectExistingLockCallback(const char *xpath, ConnectionId connectionId, CServerRemoteTree &tree, CheckedCriticalSection *_connectCrit) : lcb(xpath, connectionId, tree), connectCrit(_connectCrit) { }
  7056. virtual void block()
  7057. {
  7058. if (connectCrit)
  7059. CHECKEDCRITENTER(*connectCrit, fakeCritTimeout);
  7060. lcb.block();
  7061. }
  7062. virtual void unblock()
  7063. {
  7064. lcb.unblock();
  7065. if (connectCrit)
  7066. CHECKEDCRITLEAVE(*connectCrit);
  7067. }
  7068. };
  7069. if (!RTM_MODE(mode, RTM_CREATE_ADD) && !RTM_MODE(mode, RTM_CREATE_UNIQUE)) // cannot be pending on existing locked nodes in these cases
  7070. {
  7071. CTimeMon tm(timeout);
  7072. connectionId = coven.getUniqueId();
  7073. Owned<CServerConnection> tmpConn = new CServerConnection(*this, connectionId, xpath, sessionId, mode, timeout, NULL, (ConnInfoFlags)0);
  7074. { CHECKEDCRITICALBLOCK(cTableCrit, fakeCritTimeout);
  7075. connections.replace(*LINK(tmpConn));
  7076. }
  7077. if (sessionId)
  7078. {
  7079. { LockUnblock b(dataRWLock);
  7080. tmpConn->subscribe(sessionId);
  7081. }
  7082. if (!queryConnection(connectionId)) // aborted
  7083. {
  7084. connectionId = 0;
  7085. throw MakeSDSException(SDSExcpt_AbortDuringConnection, " during connect");
  7086. }
  7087. }
  7088. freeExistingLocks.setConnectionId(connectionId);
  7089. try
  7090. {
  7091. loop
  7092. {
  7093. try
  7094. {
  7095. Owned<IPropertyTreeIterator> iter = root->getElements(xpath+1);
  7096. iter->first();
  7097. while (iter->isValid())
  7098. {
  7099. CServerRemoteTree &existing = (CServerRemoteTree &) iter->query();
  7100. if (freeExistingLocks.isExisting(existing))
  7101. iter->next();
  7102. else
  7103. {
  7104. freeExistingLocks.add(existing);
  7105. {
  7106. unsigned remaining;
  7107. if (timeout)
  7108. {
  7109. if (tm.timedout(&remaining))
  7110. {
  7111. Linked<CLock> lock;
  7112. VStringBuffer timeoutMsg("Failed to establish lock to %s, timeout whilst retrying connection to orphaned connection path", xpath);
  7113. ForEachItemIn(f, freeExistingLocks.existingLockTrees)
  7114. {
  7115. CServerRemoteTree &e = freeExistingLocks.existingLockTrees.item(f);
  7116. {
  7117. CHECKEDCRITICALBLOCK(lockCrit, fakeCritTimeout);
  7118. CLock *_lock = queryLock(e.queryServerId());
  7119. if (_lock)
  7120. {
  7121. if (!lock)
  7122. timeoutMsg.append(", existing lock info: ");
  7123. timeoutMsg.newline();
  7124. lock.set(_lock);
  7125. }
  7126. }
  7127. if (lock)
  7128. lock->toString(timeoutMsg);
  7129. }
  7130. throw MakeSDSException(SDSExcpt_LockTimeout, "%s", timeoutMsg.str());
  7131. }
  7132. }
  7133. else
  7134. remaining = 0; // a timeout of 0 means fail immediately if locked
  7135. CConnectExistingLockCallback connectLockCallback(xpath, connectionId, existing, &connectCrit);
  7136. lock(existing, xpath, connectionId, sessionId, mode, remaining, connectLockCallback);
  7137. }
  7138. if (!queryConnection(connectionId)) // aborted
  7139. {
  7140. connectionId = 0;
  7141. throw MakeSDSException(SDSExcpt_AbortDuringConnection, " during connect");
  7142. }
  7143. iter.setown(root->getElements(xpath+1));
  7144. iter->first();
  7145. }
  7146. }
  7147. break;
  7148. }
  7149. catch (ISDSException *e) // don't treat waiting on a now orpahned node an error, since trying to lock to create (retry)
  7150. {
  7151. if (SDSExcpt_OrphanedNode != e->errorCode())
  7152. throw;
  7153. else
  7154. e->Release();
  7155. }
  7156. freeExistingLocks.clear();
  7157. }
  7158. }
  7159. catch (IException *)
  7160. {
  7161. CHECKEDCRITICALBLOCK(cTableCrit, fakeCritTimeout);
  7162. tmpConn->unsubscribeSession();
  7163. connections.removeExact(tmpConn);
  7164. connectionId = 0;
  7165. throw;
  7166. }
  7167. { CHECKEDCRITICALBLOCK(cTableCrit, fakeCritTimeout);
  7168. tmpConn->unsubscribeSession();
  7169. connections.removeExact(tmpConn);
  7170. }
  7171. }
  7172. try
  7173. {
  7174. if (RTM_MODE(mode, RTM_CREATE_ADD) || RTM_MODE(mode, RTM_CREATE_UNIQUE))
  7175. {
  7176. CHECKEDCRITICALBLOCK(blockedSaveCrit, fakeCritTimeout);
  7177. connection.setown(createConnectionInstance(root, sessionId, mode, timeout, xpath, _tree, connectionId, &deltaPath, deltaChange, branchChange, additions));
  7178. }
  7179. else
  7180. connection.setown(createConnectionInstance(root, sessionId, mode, timeout, xpath, _tree, connectionId, &deltaPath, deltaChange, branchChange, additions));
  7181. }
  7182. catch (IException *) // do not want to miss recording change to inc, under any circumstances.
  7183. {
  7184. if (deltaChange.get())
  7185. {
  7186. PROGLOG("Exception on RTM_CREATE caused call to saveDelta, xpath=%s", xpath);
  7187. saveDelta(deltaPath, *deltaChange);
  7188. }
  7189. throw;
  7190. }
  7191. linkedTree.set(_tree);
  7192. if (!connection)
  7193. {
  7194. connectionId = 0;
  7195. return;
  7196. }
  7197. assertex(_tree);
  7198. if (deltaChange.get())
  7199. {
  7200. CPTStack stack = connection->queryPTreePath();
  7201. if (connection->queryRoot() == SDSManager->queryRoot())
  7202. stack.pop();
  7203. stack.popn(additions);
  7204. connection->notify();
  7205. SDSManager->startNotification(*deltaChange, stack, *branchChange);
  7206. saveDelta(deltaPath, *deltaChange);
  7207. }
  7208. connectionId = connection->queryConnectionId();
  7209. if (freeExistingLocks.isExisting(*(CServerRemoteTree *)_tree))
  7210. {
  7211. locked = true;
  7212. freeExistingLocks.remove(*(CServerRemoteTree *)_tree);
  7213. connectCritBlock.clear();
  7214. }
  7215. { CHECKEDCRITICALBLOCK(cTableCrit, fakeCritTimeout);
  7216. connections.replace(*LINK(connection));
  7217. }
  7218. try
  7219. {
  7220. if (!locked)
  7221. {
  7222. CConnectExistingLockCallback connectLockCallback(xpath, connectionId, *(CServerRemoteTree *)_tree, connectCritBlock.get()?&connectCrit:NULL);
  7223. lock(*(CServerRemoteTree *)_tree, xpath, connectionId, sessionId, mode, timeout, connectLockCallback);
  7224. }
  7225. }
  7226. catch (IException *)
  7227. {
  7228. { CHECKEDCRITICALBLOCK(cTableCrit, fakeCritTimeout);
  7229. connections.removeExact(connection);
  7230. }
  7231. throw;
  7232. }
  7233. catch (DALI_CATCHALL)
  7234. {
  7235. { CHECKEDCRITICALBLOCK(cTableCrit, fakeCritTimeout);
  7236. connections.removeExact(connection);
  7237. }
  7238. throw;
  7239. }
  7240. if (sessionId)
  7241. {
  7242. // unlock global lock whilst subscription occurs, incase it calls me back inline (e.g. during an immediate abort)
  7243. {
  7244. LockUnblock b(dataRWLock);
  7245. connection->subscribe(sessionId);
  7246. }
  7247. // subscription may have already disconnected by this stage.
  7248. if (!queryConnection(connectionId))
  7249. {
  7250. connectionId = 0;
  7251. throw MakeSDSException(SDSExcpt_AbortDuringConnection, " during connect");
  7252. }
  7253. }
  7254. }
  7255. catch (IException *e)
  7256. {
  7257. if (SDSExcpt_OrphanedNode != e->errorCode())
  7258. throw;
  7259. e->Release();
  7260. connectionId = 0;
  7261. return;
  7262. }
  7263. // could have been blocked, now freed but in the meantime *this* connection has been aborted.
  7264. if (!queryConnection(connectionId))
  7265. {
  7266. unlock(_tree->queryServerId(), connectionId);
  7267. connectionId = 0;
  7268. throw MakeSDSException(SDSExcpt_AbortDuringConnection, " during connect");
  7269. }
  7270. connection->setEstablished();
  7271. tree = (CServerRemoteTree *) LINK(_tree);
  7272. connectionId = connection->queryConnectionId();
  7273. }
  7274. CServerConnection *CCovenSDSManager::queryConnection(ConnectionId id)
  7275. {
  7276. CHECKEDCRITICALBLOCK(cTableCrit, fakeCritTimeout);
  7277. return (CServerConnection *)connections.find(&id);
  7278. }
  7279. CServerConnection *CCovenSDSManager::getConnection(ConnectionId id)
  7280. {
  7281. CHECKEDCRITICALBLOCK(cTableCrit, fakeCritTimeout);
  7282. CServerConnection *conn = (CServerConnection *)connections.find(&id);
  7283. if (conn) conn->Link();
  7284. return conn;
  7285. }
  7286. void CCovenSDSManager::disconnect(ConnectionId id, bool deleteRoot, Owned<CLCLockBlock> *lockBlock)
  7287. {
  7288. Linked<CServerConnection> connection;
  7289. { CHECKEDCRITICALBLOCK(cTableCrit, fakeCritTimeout);
  7290. connection.set(queryConnection(id));
  7291. if (!connection)
  7292. return;
  7293. connections.removeExact(connection);
  7294. }
  7295. Linked<CServerRemoteTree> tree = (CServerRemoteTree *)connection->queryRootUnvalidated();
  7296. if (!tree) return;
  7297. unsigned index = (unsigned)-1;
  7298. StringBuffer path;
  7299. connection->queryPTreePath().getAbsolutePath(path);
  7300. if (connection->queryParent())
  7301. {
  7302. if (deleteRoot || RTM_MODE(connection->queryMode(), RTM_DELETE_ON_DISCONNECT))
  7303. {
  7304. CHECKEDCRITICALBLOCK(lockCrit, fakeCritTimeout);
  7305. CLock *lock = queryLock(tree->queryServerId());
  7306. if (lock)
  7307. {
  7308. deleteRoot = false;
  7309. lock->setDROLR((CServerRemoteTree *)connection->queryParent(), tree);
  7310. }
  7311. else
  7312. deleteRoot = true;
  7313. }
  7314. if ((unsigned)-1 == index)
  7315. index = connection->queryParent()->queryChildIndex(connection->queryRootUnvalidated());
  7316. }
  7317. else
  7318. deleteRoot = false;
  7319. bool orphaned = ((CServerRemoteTree*)connection->queryRootUnvalidated())->isOrphaned();
  7320. // Still want disconnection to be performed & recorded, if orphaned
  7321. if (!deleteRoot && unlock(tree->queryServerId(), id, true)) // unlock returns true if last unlock and there was a setDROLR on it
  7322. deleteRoot = true;
  7323. if (deleteRoot)
  7324. {
  7325. if (lockBlock)
  7326. {
  7327. lockBlock->clear();
  7328. lockBlock->setown(new CLCWriteLockBlock(dataRWLock, readWriteTimeout, __FILE__, __LINE__));
  7329. }
  7330. connection->queryParent()->removeTree(tree);
  7331. writeTransactions++;
  7332. if (!orphaned)
  7333. {
  7334. Owned<IPropertyTree> changeTree = createPTree(RESERVED_CHANGE_NODE);
  7335. IPropertyTree *d = changeTree->setPropTree(DELETE_TAG, createPTree());
  7336. d->setProp("@name", tree->queryName());
  7337. d->setPropInt("@pos", index+1);
  7338. Owned<CBranchChange> branchChange = new CBranchChange(*tree);
  7339. branchChange->noteChange(PDS_Deleted, PDS_Deleted);
  7340. CPTStack stack = connection->queryPTreePath();
  7341. stack.pop();
  7342. if (connection->queryRootUnvalidated() == SDSManager->queryRoot())
  7343. stack.pop();
  7344. if (!RTM_MODE(connection->queryMode(), RTM_INTERNAL))
  7345. {
  7346. connection->notify();
  7347. SDSManager->startNotification(*changeTree, stack, *branchChange);
  7348. }
  7349. StringBuffer head;
  7350. const char *tail = splitXPath(path.str(), head);
  7351. CHECKEDCRITICALBLOCK(blockedSaveCrit, fakeCritTimeout);
  7352. if (NotFound != index)
  7353. saveDelta(head.str(), *changeTree);
  7354. else
  7355. { // NB: don't believe this can happen, but last thing want to do is save duff delete delta.
  7356. WARNLOG("** CCovenSDSManager::disconnect - index position lost **");
  7357. PrintStackReport();
  7358. }
  7359. }
  7360. }
  7361. tree.clear();
  7362. connection->unsubscribeSession();
  7363. }
  7364. StringBuffer &formatUsageStats(MemoryBuffer &src, StringBuffer &out)
  7365. {
  7366. unsigned c;
  7367. src.read(c);
  7368. out.append("Connections : ").append(c).newline();
  7369. src.read(c);
  7370. out.append("Locks : ").append(c).newline();
  7371. src.read(c);
  7372. out.append("Subscribers : ").append(c).newline();
  7373. src.read(c);
  7374. out.append("Connection subscriptions : ").append(c).newline();
  7375. return out;
  7376. }
  7377. StringBuffer &formatConnectionInfo(MemoryBuffer &src, StringBuffer &out)
  7378. {
  7379. ConnectionId connectionId;
  7380. StringAttr xpath;
  7381. SessionId sessionId;
  7382. unsigned mode;
  7383. unsigned timeout;
  7384. bool established;
  7385. src.read(connectionId).read(xpath).read(sessionId).read(mode).read(timeout).read(established);
  7386. out.append("ConnectionId=").appendf("%" I64F "x", connectionId).append(", xpath=").append(xpath).append(", sessionId=").appendf("%" I64F "x", sessionId).append(", mode=").append(mode).append(", timeout=");
  7387. if (INFINITE == timeout)
  7388. out.append("INFINITE");
  7389. else
  7390. out.append(timeout);
  7391. out.append(established?"":" [BLOCKED]");
  7392. return out;
  7393. }
  7394. StringBuffer &formatSubscriberInfo(MemoryBuffer &src, StringBuffer &out)
  7395. {
  7396. SubscriptionId subscriptionId;
  7397. bool sub;
  7398. StringAttr xpath;
  7399. src.read(subscriptionId).read(sub).read(xpath);
  7400. out.append("SubscriptionId=").appendf("%" I64F "x", subscriptionId).append(", xpath=").append(xpath).append(", sub=").append(sub?"true":"false");
  7401. return out;
  7402. }
  7403. StringBuffer &formatNodeSubscriberInfo(MemoryBuffer &src, StringBuffer &out)
  7404. {
  7405. SubscriptionId subscriptionId;
  7406. StringAttr xpath;
  7407. unsigned nodeCount;
  7408. src.read(subscriptionId).read(xpath).read(nodeCount);
  7409. out.append("SubscriptionId=").appendf("%" I64F "x", subscriptionId).append(", xpath=").append(xpath).append(", nodes=").append(nodeCount);
  7410. return out;
  7411. }
  7412. StringBuffer &formatConnections(MemoryBuffer &src, StringBuffer &out)
  7413. {
  7414. unsigned count;
  7415. src.read(count);
  7416. if (count)
  7417. {
  7418. while (count--)
  7419. {
  7420. formatConnectionInfo(src, out);
  7421. if (count) out.newline();
  7422. }
  7423. }
  7424. else
  7425. out.append("No current connections");
  7426. return out;
  7427. }
  7428. StringBuffer &formatSubscribers(MemoryBuffer &src, StringBuffer &out)
  7429. {
  7430. unsigned sdsSubscribers, sdsNodeSubscribers=0;
  7431. src.read(sdsSubscribers);
  7432. unsigned s = sdsSubscribers;
  7433. while (s--)
  7434. {
  7435. formatSubscriberInfo(src, out);
  7436. if (s) out.newline();
  7437. }
  7438. if (src.remaining())
  7439. {
  7440. src.read(sdsNodeSubscribers);
  7441. s = sdsNodeSubscribers;
  7442. while (s--)
  7443. {
  7444. formatNodeSubscriberInfo(src, out);
  7445. if (s) out.newline();
  7446. }
  7447. }
  7448. out.newline().appendf("%d xpath subscribers, %d node subscribers\n", sdsSubscribers, sdsNodeSubscribers);
  7449. return out;
  7450. }
  7451. unsigned CCovenSDSManager::countConnections()
  7452. {
  7453. CHECKEDCRITICALBLOCK(cTableCrit, fakeCritTimeout);
  7454. return connections.count();
  7455. }
  7456. unsigned CCovenSDSManager::countActiveLocks()
  7457. {
  7458. unsigned activeLocks = 0;
  7459. CHECKEDCRITICALBLOCK(lockCrit, fakeCritTimeout);
  7460. SuperHashIteratorOf<CLock> iter(lockTable.queryBaseTable());
  7461. ForEach(iter) {
  7462. CLock &lock = iter.query();
  7463. if (lock.lockCount()) activeLocks++;
  7464. }
  7465. return activeLocks;
  7466. }
  7467. ILockInfoCollection *CCovenSDSManager::getLocks(const char *ipPattern, const char *xpathPattern)
  7468. {
  7469. Owned<ILockInfoCollection> lockInfoCollection = createLockInfoCollection();
  7470. bool filteredConnections = !isEmptyString(ipPattern);
  7471. bool filteredXPaths = !isEmptyString(xpathPattern);
  7472. CLockInfoArray locks;
  7473. {
  7474. CHECKEDCRITICALBLOCK(lockCrit, fakeCritTimeout);
  7475. SuperHashIteratorOf<CLock> iter(lockTable.queryBaseTable());
  7476. ForEach(iter)
  7477. {
  7478. CLock &lock = iter.query();
  7479. if (lock.lockCount())
  7480. {
  7481. if (!filteredXPaths || WildMatch(lock.queryXPath(), xpathPattern))
  7482. locks.append(* lock.getLockInfo());
  7483. }
  7484. }
  7485. }
  7486. if (filteredConnections)
  7487. {
  7488. ForEachItemIn(c, locks)
  7489. {
  7490. ILockInfo &lockInfo = locks.item(c);
  7491. lockInfo.prune(ipPattern);
  7492. }
  7493. }
  7494. ForEachItemIn(l, locks)
  7495. lockInfoCollection->add(* LINK(&locks.item(l)));
  7496. return lockInfoCollection.getClear();
  7497. }
  7498. MemoryBuffer &CCovenSDSManager::collectUsageStats(MemoryBuffer &out)
  7499. {
  7500. { CHECKEDCRITICALBLOCK(cTableCrit, fakeCritTimeout);
  7501. out.append(connections.count());
  7502. }
  7503. unsigned activeLocks = 0;
  7504. { CHECKEDCRITICALBLOCK(lockCrit, fakeCritTimeout);
  7505. SuperHashIteratorOf<CLock> iter(lockTable.queryBaseTable());
  7506. ForEach(iter)
  7507. {
  7508. CLock &lock = iter.query();
  7509. if (lock.lockCount()) activeLocks++;
  7510. }
  7511. }
  7512. out.append(activeLocks);
  7513. out.append(subscribers.count());
  7514. out.append(connectionSubscriptionManager->querySubscribers());
  7515. return out;
  7516. }
  7517. MemoryBuffer &CCovenSDSManager::collectConnections(MemoryBuffer &out)
  7518. {
  7519. CHECKEDCRITICALBLOCK(cTableCrit, fakeCritTimeout);
  7520. out.append(connections.count());
  7521. SuperHashIteratorOf<CServerConnection> iter(connections.queryBaseTable());
  7522. ForEach(iter)
  7523. iter.query().getInfo(out);
  7524. return out;
  7525. }
  7526. MemoryBuffer &CCovenSDSManager::collectSubscribers(MemoryBuffer &out)
  7527. {
  7528. CHECKEDCRITICALBLOCK(sTableCrit, fakeCritTimeout);
  7529. out.append(subscribers.count());
  7530. SuperHashIteratorOf<CSubscriberContainer> sdsIter(subscribers.queryBaseTable());
  7531. ForEach(sdsIter)
  7532. sdsIter.query().getInfo(out);
  7533. nodeSubscriptionManager->collectSubscribers(out);
  7534. return out;
  7535. }
  7536. void CCovenSDSManager::blockingSave(unsigned *writeTransactions)
  7537. {
  7538. CHECKEDDALIREADLOCKBLOCK(SDSManager->dataRWLock, readWriteTimeout); // block all write actions whilst saving
  7539. CHECKEDCRITICALBLOCK(blockedSaveCrit, fakeCritTimeout);
  7540. if (writeTransactions)
  7541. *writeTransactions = SDSManager->writeTransactions;
  7542. // JCS - could in theory, not block, but abort save.
  7543. SDSManager->saveStore();
  7544. }
  7545. bool CCovenSDSManager::updateEnvironment(IPropertyTree *newEnv, bool forceGroupUpdate, StringBuffer &response)
  7546. {
  7547. Owned<IRemoteConnection> conn = querySDS().connect("/",myProcessSession(),0, INFINITE);
  7548. if (conn)
  7549. {
  7550. Owned<IPropertyTree> root = conn->getRoot();
  7551. Owned<IPropertyTree> oldEnvironment = root->getPropTree("Environment");
  7552. if (oldEnvironment.get())
  7553. {
  7554. StringBuffer bakname;
  7555. Owned<IFileIO> io = createUniqueFile(NULL, "environment", "bak", bakname);
  7556. Owned<IFileIOStream> fstream = createBufferedIOStream(io);
  7557. toXML(oldEnvironment, *fstream); // formatted (default)
  7558. root->removeTree(oldEnvironment);
  7559. }
  7560. root->addPropTree("Environment", LINK(newEnv));
  7561. root.clear();
  7562. conn->commit();
  7563. conn->close();
  7564. StringBuffer messages;
  7565. initClusterGroups(forceGroupUpdate, messages, oldEnvironment);
  7566. response.append(messages);
  7567. PROGLOG("Environment and node groups updated");
  7568. }
  7569. return true;
  7570. }
  7571. // TODO
  7572. StringBuffer &CCovenSDSManager::getExternalReport(StringBuffer &out)
  7573. {
  7574. return out;
  7575. }
  7576. StringBuffer &CCovenSDSManager::getConnections(StringBuffer &out)
  7577. {
  7578. MemoryBuffer mb;
  7579. formatConnections(collectConnections(mb), out);
  7580. return out;
  7581. }
  7582. StringBuffer &CCovenSDSManager::getSubscribers(StringBuffer &out)
  7583. {
  7584. MemoryBuffer mb;
  7585. formatSubscribers(collectSubscribers(mb), out);
  7586. return out;
  7587. }
  7588. StringBuffer &CCovenSDSManager::getUsageStats(StringBuffer &out)
  7589. {
  7590. MemoryBuffer mb;
  7591. formatUsageStats(collectUsageStats(mb), out);
  7592. return out;
  7593. }
  7594. void CCovenSDSManager::handleNodeNotify(notifications n, CServerRemoteTree &tree)
  7595. {
  7596. const char *handlerKey = tree.queryProp(NOTIFY_ATTR);
  7597. assertex(handlerKey);
  7598. CSDSNotifyHandlerMapping *m = nodeNotifyHandlers.find(handlerKey);
  7599. if (!m)
  7600. {
  7601. LOG(MCwarning, unknownJob, "Unknown notify handler name \"%s\", handing event %s", handlerKey, notificationStr(n));
  7602. return;
  7603. }
  7604. switch (n)
  7605. {
  7606. case notify_delete:
  7607. try { m->query().removed(tree); }
  7608. catch (IException *e)
  7609. {
  7610. StringBuffer s("Exception calling ISDSNotifyHandler->removed(<tree>), for handler\"");
  7611. s.append(notificationStr(n)).append("\"");
  7612. EXCLOG(e, s.str());
  7613. e->Release();
  7614. }
  7615. break;
  7616. default:
  7617. LOG(MCerror, unknownJob, "Unknown notification type (%d)", n);
  7618. break;
  7619. }
  7620. }
  7621. void CCovenSDSManager::handleNotify(CSubscriberContainerBase *_subscriber, MemoryBuffer &notifyData)
  7622. {
  7623. Owned<CSubscriberContainerBase> subscriber = _subscriber;
  7624. class CNotifyPoolFactory : public IThreadFactory, public CInterface
  7625. {
  7626. class CNotifyHandler : implements IPooledThread, public CInterface
  7627. {
  7628. DECL_NAMEDCOUNT;
  7629. public:
  7630. IMPLEMENT_IINTERFACE;
  7631. CNotifyHandler() { INIT_NAMEDCOUNT; }
  7632. void init(void *startInfo)
  7633. {
  7634. n.setown((CSubscriberNotifier *)startInfo);
  7635. }
  7636. void main()
  7637. {
  7638. try
  7639. {
  7640. n->notify();
  7641. }
  7642. catch (IException *e)
  7643. {
  7644. EXCLOG(e, "CNotifyHandler");
  7645. e->Release();
  7646. }
  7647. n.clear();
  7648. }
  7649. bool canReuse()
  7650. {
  7651. return true;
  7652. }
  7653. bool stop()
  7654. {
  7655. return true;
  7656. }
  7657. private:
  7658. Linked<CSubscriberNotifier> n;
  7659. };
  7660. public:
  7661. IMPLEMENT_IINTERFACE;
  7662. IPooledThread *createNew()
  7663. {
  7664. return new CNotifyHandler();
  7665. }
  7666. };
  7667. if (!notifyPool)
  7668. {
  7669. CNotifyPoolFactory *factory = new CNotifyPoolFactory;
  7670. notifyPool.setown(createThreadPool("SDS Notification Pool", factory, this, SUBNTFY_POOL_SIZE));
  7671. factory->Release();
  7672. }
  7673. CHECKEDCRITICALBLOCK(nfyTableCrit, fakeCritTimeout);
  7674. SubscriptionId id = subscriber->queryId();
  7675. CSubscriberNotifier *notifier = subscriberNotificationTable.find(id);
  7676. /* Must clear 'subscriber' before leaving ntyTableCrit block, so that the notifier thread owns and destroys it
  7677. * It cannot be destroyed here, because this method may have been called inside the SDS lock during a node
  7678. * delete and do not want to call into subscriber manager as that can deadlock if processing a request
  7679. * already that requires SDS lock.
  7680. */
  7681. if (notifier)
  7682. {
  7683. subscriber.clear();
  7684. notifier->queueChange(notifyData);
  7685. }
  7686. else
  7687. {
  7688. Owned<CSubscriberNotifier> _notifier = new CSubscriberNotifier(subscriberNotificationTable, *subscriber.getClear(), notifyData);
  7689. subscriberNotificationTable.replace(*_notifier);
  7690. notifyPool->start(_notifier.getClear()); // NB: takes ownership (during init())
  7691. }
  7692. }
  7693. /////////////////
  7694. class CSubscriberNotifyScanner : public CInterface
  7695. {
  7696. DECL_NAMEDCOUNT;
  7697. CPTStack stack;
  7698. Linked<IPropertyTree> changeTree;
  7699. Linked<CBranchChange> rootChanges;
  7700. StringBuffer xpath;
  7701. CSubscriberArray subs;
  7702. public:
  7703. struct PushPop
  7704. {
  7705. PushPop(CPTStack &_stack, CRemoteTreeBase &tree) : stack(_stack) { tree.Link(); stack.append(tree); }
  7706. ~PushPop() { stack.pop(); }
  7707. CPTStack &stack;
  7708. };
  7709. CSubscriberNotifyScanner(IPropertyTree &_changeTree, CPTStack &_stack, CBranchChange &_rootChanges) : changeTree(&_changeTree), rootChanges(&_rootChanges), stack(_stack)
  7710. {
  7711. INIT_NAMEDCOUNT;
  7712. SDSManager->querySubscriberTable().getSubscribers(subs);
  7713. }
  7714. enum SubCommitType { subCommitNone, subCommitExact, subCommitBelow, subCommitAbove };
  7715. SubCommitType match(const char *head, const char *path)
  7716. {
  7717. bool wild = false;
  7718. loop
  7719. {
  7720. if (wild)
  7721. {
  7722. if (*head == *path)
  7723. {
  7724. path++;
  7725. wild = false;
  7726. }
  7727. else if ('/' == *head)
  7728. wild = false;
  7729. }
  7730. else if ('*' == *path)
  7731. {
  7732. path++;
  7733. wild = true;
  7734. }
  7735. else if (*head != *path)
  7736. return subCommitNone;
  7737. else
  7738. path++;
  7739. head++;
  7740. if ('\0' == *path)
  7741. {
  7742. if ('\0' == *head)
  7743. return subCommitExact; // absolute match
  7744. else if (!wild && '/' != *head) // e.g. change=/a/bc, subscriber=/a/b
  7745. return subCommitNone;
  7746. return subCommitBelow; // e.g. change=/a/b/c, subscriber=/a/b
  7747. }
  7748. else
  7749. {
  7750. if ('\0' == *head)
  7751. return subCommitAbove; // e.g. change=/a/b, subscriber=/a/b/c - not matched yet, but returning true keeps it from being pruned
  7752. }
  7753. }
  7754. }
  7755. void scan()
  7756. {
  7757. xpath.clear();
  7758. stack.toString(xpath);
  7759. if (stack.ordinality() && (rootChanges->tree == &(stack.tos()))) stack.pop();
  7760. CSubscriberArray pruned;
  7761. scan(*rootChanges, stack, pruned);
  7762. }
  7763. bool prune(const char *xpath, CSubscriberCopyArray &candidates, CSubscriberArray &pruned)
  7764. {
  7765. ForEachItemInRev(s, subs)
  7766. {
  7767. CSubscriberContainer &subscriber = subs.item(s);
  7768. SubCommitType subCommit;
  7769. if (subscriber.isUnsubscribed())
  7770. subCommit = subCommitNone;
  7771. else
  7772. subCommit = match(xpath, subscriber.queryXPath());
  7773. switch (subCommit)
  7774. {
  7775. case subCommitNone:
  7776. {
  7777. pruned.append(*LINK(&subscriber));
  7778. subs.remove(s);
  7779. break;
  7780. }
  7781. case subCommitExact:
  7782. {
  7783. candidates.append(subscriber);
  7784. break;
  7785. }
  7786. case subCommitBelow: // e.g. change=/a/b/c, subscriber=/a/b
  7787. {
  7788. if (!subscriber.querySub())
  7789. {
  7790. pruned.append(*LINK(&subscriber));
  7791. subs.remove(s);
  7792. }
  7793. else
  7794. candidates.append(subscriber);
  7795. break;
  7796. }
  7797. case subCommitAbove: // e.g. change=/a/b, subscriber=/a/b/c
  7798. break; // keep in subs, deeper changes may match
  7799. default:
  7800. throwUnexpected();
  7801. }
  7802. }
  7803. return (subs.ordinality() > 0);
  7804. }
  7805. // recurse down all matching subscription stubs while qualified
  7806. void scanAll(PDState state, CBranchChange &changes, CPTStack &stack, CSubscriberArray &pruned)
  7807. {
  7808. CSubscriberCopyArray candidates;
  7809. if (prune(xpath.str(), candidates, pruned))
  7810. {
  7811. MemoryBuffer notifyData;
  7812. if (candidates.ordinality())
  7813. {
  7814. ForEachItemInRev(s, candidates)
  7815. {
  7816. CSubscriberContainer &subscriber = candidates.item(s);
  7817. if (!subscriber.isUnsubscribed())
  7818. {
  7819. if (subscriber.qualify(stack, true))
  7820. {
  7821. if (0 == notifyData.length())
  7822. buildNotifyData(notifyData, state, &stack, NULL);
  7823. SDSManager->handleNotify(LINK(&subscriber), notifyData);
  7824. }
  7825. else
  7826. pruned.append(*LINK(&subscriber));
  7827. }
  7828. subs.zap(subscriber);
  7829. }
  7830. }
  7831. else
  7832. {
  7833. if (0 == changes.children.ordinality())
  7834. {
  7835. ForEachItemInRev(s, subs)
  7836. {
  7837. CSubscriberContainer &subscriber = subs.item(s);
  7838. if (!subscriber.isUnsubscribed())
  7839. {
  7840. if (subscriber.qualify(stack, true))
  7841. {
  7842. if (0 == notifyData.length())
  7843. buildNotifyData(notifyData, state, &stack, NULL);
  7844. SDSManager->handleNotify(LINK(&subscriber), notifyData);
  7845. }
  7846. else
  7847. pruned.append(*LINK(&subscriber));
  7848. }
  7849. subs.remove(s);
  7850. }
  7851. }
  7852. else
  7853. {
  7854. ForEachItemIn (c, changes.children)
  7855. {
  7856. CBranchChange &childChange = changes.children.item(c);
  7857. PushPop pp(stack, *childChange.tree);
  7858. size32_t parentLength = xpath.length();
  7859. xpath.append('/').append(childChange.tree->queryName());
  7860. CSubscriberArray _pruned;
  7861. scanAll(state, childChange, stack, _pruned);
  7862. ForEachItemIn(i, _pruned) subs.append(*LINK(&_pruned.item(i)));
  7863. if (0 == subs.ordinality())
  7864. break;
  7865. xpath.setLength(parentLength);
  7866. }
  7867. }
  7868. }
  7869. }
  7870. }
  7871. void scan(CBranchChange &changes, CPTStack &stack, CSubscriberArray &pruned)
  7872. {
  7873. CSubscriberCopyArray candidates;
  7874. if (!prune(xpath.str(), candidates, pruned))
  7875. return;
  7876. PushPop pp(stack, *changes.tree);
  7877. if (PDS_Deleted == (changes.local & PDS_Deleted))
  7878. {
  7879. scanAll(changes.local, changes, stack, pruned);
  7880. return;
  7881. }
  7882. else if (candidates.ordinality()) // xpath matched some subscribers, and/or below some, need to check for sub subscribers
  7883. {
  7884. bool ret = false;
  7885. // avoid notifying on PDS_Structure only, which signifies changes deeper down only
  7886. MemoryBuffer notifyData;
  7887. if (changes.state && changes.local && (changes.local != PDS_Structure))
  7888. {
  7889. int lastSendValue = -1;
  7890. ForEachItemInRev(s, candidates)
  7891. {
  7892. CSubscriberContainer &subscriber = candidates.item(s);
  7893. if (!subscriber.isUnsubscribed())
  7894. {
  7895. if (subscriber.qualify(stack, false))
  7896. {
  7897. if (subscriber.querySendValue())
  7898. {
  7899. if (1 != lastSendValue)
  7900. {
  7901. MemoryBuffer mb;
  7902. changes.tree->getPropBin(NULL, mb);
  7903. buildNotifyData(notifyData.clear(), changes.state, &stack, &mb);
  7904. lastSendValue = 1;
  7905. }
  7906. }
  7907. else
  7908. {
  7909. if (0 != lastSendValue)
  7910. {
  7911. buildNotifyData(notifyData.clear(), changes.state, &stack, NULL);
  7912. lastSendValue = 0;
  7913. }
  7914. }
  7915. SDSManager->handleNotify(LINK(&subscriber), notifyData);
  7916. }
  7917. else
  7918. pruned.append(*LINK(&subscriber));
  7919. }
  7920. subs.zap(subscriber);
  7921. }
  7922. }
  7923. }
  7924. ForEachItemIn(c, changes.children)
  7925. {
  7926. CBranchChange &childChanges = changes.children.item(c);
  7927. size32_t parentLength = xpath.length();
  7928. xpath.append('/').append(childChanges.tree->queryName());
  7929. CSubscriberArray pruned;
  7930. scan(childChanges, stack, pruned);
  7931. ForEachItemIn(i, pruned) subs.append(*LINK(&pruned.item(i)));
  7932. if (0 == subs.ordinality())
  7933. break;
  7934. xpath.setLength(parentLength);
  7935. }
  7936. }
  7937. };
  7938. void CCovenSDSManager::startNotification(IPropertyTree &changeTree, CPTStack &stack, CBranchChange &changes)
  7939. {
  7940. class CScanNotifyPoolFactory : public IThreadFactory, public CInterface
  7941. {
  7942. class CScanNotifyHandler : implements IPooledThread, public CInterface
  7943. {
  7944. public:
  7945. IMPLEMENT_IINTERFACE;
  7946. void init(void *startInfo)
  7947. {
  7948. n.set((CSubscriberNotifyScanner *)startInfo);
  7949. }
  7950. void main()
  7951. {
  7952. try
  7953. {
  7954. n->scan();
  7955. }
  7956. catch (IException *e)
  7957. {
  7958. EXCLOG(e, "CScanNotifyHandler");
  7959. e->Release();
  7960. }
  7961. n.clear();
  7962. }
  7963. bool canReuse()
  7964. {
  7965. return true;
  7966. }
  7967. bool stop()
  7968. {
  7969. return true;
  7970. }
  7971. private:
  7972. Linked<CSubscriberNotifyScanner> n;
  7973. };
  7974. public:
  7975. IMPLEMENT_IINTERFACE;
  7976. IPooledThread *createNew()
  7977. {
  7978. return new CScanNotifyHandler();
  7979. }
  7980. };
  7981. if (!scanNotifyPool)
  7982. {
  7983. CScanNotifyPoolFactory *factory = new CScanNotifyPoolFactory;
  7984. scanNotifyPool.setown(createThreadPool("SDS Scan-Notification Pool", factory, this, SUBSCAN_POOL_SIZE));
  7985. factory->Release();
  7986. }
  7987. Owned<CSubscriberNotifyScanner> scan = new CSubscriberNotifyScanner(changeTree, stack, changes);
  7988. scanNotifyPool->start(scan.get());
  7989. }
  7990. void CCovenSDSManager::deleteExternal(__int64 index)
  7991. {
  7992. if (server.queryStopped()) return;
  7993. CHECKEDCRITICALBLOCK(extCrit, fakeCritTimeout);
  7994. IExternalHandler *handler = queryExternalHandler(EF_BinaryValue);
  7995. assertex(handler);
  7996. StringBuffer name(EXTERNAL_NAME_PREFIX);
  7997. name.append(index);
  7998. handler->remove(name.str());
  7999. }
  8000. void CCovenSDSManager::serializeExternal(__int64 index, IPropertyTree &owner, MemoryBuffer &mb, bool withValue)
  8001. {
  8002. CHECKEDCRITICALBLOCK(extCrit, fakeCritTimeout);
  8003. IExternalHandler *extHandler = queryExternalHandler(EF_BinaryValue);
  8004. assertex(extHandler);
  8005. try
  8006. {
  8007. StringBuffer name(EXTERNAL_NAME_PREFIX);
  8008. name.append(index);
  8009. extHandler->read(name.str(), owner, mb, withValue);
  8010. }
  8011. catch (IException *)
  8012. {
  8013. extHandler->resetAsExternal(owner);
  8014. throw;
  8015. }
  8016. }
  8017. void CCovenSDSManager::writeExternal(CServerRemoteTree &tree, bool direct, __int64 existing)
  8018. {
  8019. CHECKEDCRITICALBLOCK(extCrit, fakeCritTimeout);
  8020. __int64 index;
  8021. if (existing)
  8022. index = existing;
  8023. else
  8024. index = getNextExternal();
  8025. IExternalHandler *extHandler = queryExternalHandler(EF_BinaryValue);
  8026. assertex(extHandler);
  8027. tree.removeProp(EXT_ATTR);
  8028. StringBuffer name(EXTERNAL_NAME_PREFIX);
  8029. extHandler->write(name.append(index).str(), tree);
  8030. tree.setPropInt64(EXT_ATTR, index);
  8031. extHandler->resetAsExternal(tree);
  8032. // setPropInt64(EXT_ATTR, index); // JCSMORE not necessary
  8033. }
  8034. // ISubscriptionManager
  8035. void CCovenSDSManager::add(ISubscription *sub, SubscriptionId id)
  8036. {
  8037. CHECKEDCRITICALBLOCK(sTableCrit, fakeCritTimeout);
  8038. subscribers.replace(* new CSubscriberContainer(sub, id));
  8039. }
  8040. void CCovenSDSManager::remove(SubscriptionId id)
  8041. {
  8042. CHECKEDCRITICALBLOCK(sTableCrit, fakeCritTimeout);
  8043. subscribers.remove(&id);
  8044. }
  8045. // IExceptionHandler
  8046. static bool processingUnhandled = false;
  8047. static bool handled = false;
  8048. static CheckedCriticalSection unhandledCrit;
  8049. bool CCovenSDSManager::fireException(IException *e)
  8050. {
  8051. //This code is rather dodgy (and causes more problems than it solves)
  8052. // so ignore unhandled exceptions for the moment!
  8053. LOG(MCoperatorError, unknownJob, e, "Caught unhandled exception!");
  8054. return true;
  8055. { CHECKEDCRITICALBLOCK(unhandledCrit, fakeCritTimeout);
  8056. if (processingUnhandled)
  8057. {
  8058. if (handled)
  8059. {
  8060. LOG(MCdisaster, unknownJob, e, "FATAL, too many exceptions");
  8061. return false; // did not successfully handle.
  8062. }
  8063. LOG(MCoperatorError, unknownJob, e, "Exception while restarting or shutting down");
  8064. return true;
  8065. }
  8066. handled = false;
  8067. processingUnhandled = true;
  8068. }
  8069. // Handle exception on a separate thread, to avoid complication with joining/restarting deadlocks.
  8070. class CHandleException : public Thread
  8071. {
  8072. CCovenSDSManager &manager;
  8073. Linked<IException> e;
  8074. bool restart;
  8075. public:
  8076. CHandleException(CCovenSDSManager &_manager, IException *_e, bool _restart) : Thread("sds:CHandleException"), manager(_manager), e(_e), restart(_restart)
  8077. {
  8078. start();
  8079. }
  8080. virtual int run()
  8081. {
  8082. handled=true;
  8083. try
  8084. {
  8085. if (restart)
  8086. manager.restart(e);
  8087. else
  8088. {
  8089. StringBuffer msg;
  8090. msg.append("Unhandled exception, shutting down: ").append(e->errorCode()).append(": ");
  8091. e->errorMessage(msg);
  8092. manager.stop();
  8093. manager.saveStore();
  8094. }
  8095. manager.unhandledThread.clear();
  8096. }
  8097. catch (IException *_e) { LOG(MCoperatorError, unknownJob, _e, "Exception while restarting or shutting down"); _e->Release(); }
  8098. catch (DALI_CATCHALL) { LOG(MCoperatorError, unknownJob, "Unknown exception while restarting or shutting down"); }
  8099. if (!restart)
  8100. {
  8101. e->Link();
  8102. throw e.get();
  8103. }
  8104. processingUnhandled = false;
  8105. handled = false;
  8106. return 0;
  8107. }
  8108. };
  8109. unhandledThread.setown(new CHandleException(*this, e, restartOnError));
  8110. return true;
  8111. }
  8112. void CCovenSDSManager::addNodeSubscriber(ISubscription *sub, SubscriptionId id)
  8113. {
  8114. MemoryBuffer mb;
  8115. mb.setBuffer(sub->queryData().length(), (void *)sub->queryData().get());
  8116. StringAttr xpath;
  8117. bool sendValue;
  8118. mb.read(xpath);
  8119. mb.read(sendValue);
  8120. Owned<IPropertyTreeIterator> iter = root->getElements(xpath+1);
  8121. if (!iter->first())
  8122. throw MakeSDSException(SDSExcpt_SubscriptionNoMatch, "Failed to match any nodes: %s", xpath.get());
  8123. else
  8124. {
  8125. Owned<CNodeSubscriberContainer> subscriber = new CNodeSubscriberContainer(sub, id, sendValue, xpath);
  8126. do
  8127. {
  8128. CServerRemoteTree &node = (CServerRemoteTree &)iter->query();
  8129. node.setSubscribed(true);
  8130. subscriber->add(node);
  8131. }
  8132. while (iter->next());
  8133. nodeSubscriptionManager->associateSubscriber(*subscriber);
  8134. }
  8135. }
  8136. void CCovenSDSManager::removeNodeSubscriber(SubscriptionId id)
  8137. {
  8138. nodeSubscriptionManager->removeSubscriberAssociation(id);
  8139. }
  8140. void CCovenSDSManager::notifyNodeDelete(CServerRemoteTree &node)
  8141. {
  8142. nodeSubscriptionManager->notifyDelete(&node);
  8143. }
  8144. void CCovenSDSManager::notifyNode(CServerRemoteTree &node, PDState state)
  8145. {
  8146. nodeSubscriptionManager->notify(node, state);
  8147. }
  8148. ///////////////////////
  8149. class CDaliSDSServer: implements IDaliServer, public CInterface
  8150. {
  8151. public:
  8152. IMPLEMENT_IINTERFACE;
  8153. CDaliSDSServer(IPropertyTree *_config) : config(_config)
  8154. {
  8155. manager = NULL;
  8156. cancelLoad = false;
  8157. storeLoaded = false;
  8158. }
  8159. ~CDaliSDSServer()
  8160. {
  8161. delete manager;
  8162. }
  8163. void start()
  8164. {
  8165. CriticalBlock b(crit);
  8166. ICoven &coven=queryCoven();
  8167. assertex(coven.inCoven()); // must be member of coven
  8168. if (config)
  8169. sdsConfig.setown(config->getPropTree("SDS"));
  8170. if (!sdsConfig)
  8171. sdsConfig.setown(createPTree());
  8172. manager = new CCovenSDSManager(coven, *sdsConfig, config?config->queryProp("@dataPath"):NULL);
  8173. SDSManager = manager;
  8174. addThreadExceptionHandler(manager);
  8175. try { manager->loadStore(NULL, &cancelLoad); }
  8176. catch (IException *)
  8177. {
  8178. LOG(MCdebugInfo(100), unknownJob, "Failed to load main store");
  8179. throw;
  8180. }
  8181. storeLoaded = true;
  8182. manager->start();
  8183. }
  8184. void ready()
  8185. {
  8186. #ifdef TEST_NOTIFY_HANDLER
  8187. class CTestHan : public CInterface, implements ISDSNotifyHandler
  8188. {
  8189. public:
  8190. IMPLEMENT_IINTERFACE;
  8191. virtual void removed(IPropertyTree &tree)
  8192. {
  8193. PrintLog("Hello, tree(%s) handler(%s), being deleted", tree.queryName(), queryNotifyHandlerName(&tree));
  8194. }
  8195. };
  8196. Owned<ISDSNotifyHandler> myHan = new CTestHan();
  8197. ISDSManagerServer &sdsManager = querySDSServer();
  8198. sdsManager.installNotifyHandler("testHandler", myHan);
  8199. Owned<IRemoteConnection> conn = manager->connect("/", 0, 0, 0);
  8200. IPropertyTree *root = conn->queryRoot();
  8201. IPropertyTree *tree = root->setPropTree("test", createPTree());
  8202. setNotifyHandlerName("testHandler", tree);
  8203. conn->commit();
  8204. root->removeProp("test");
  8205. conn->commit();
  8206. sdsManager.removeNotifyHandler("testHandler");
  8207. #endif
  8208. }
  8209. void suspend()
  8210. {
  8211. }
  8212. void stop()
  8213. {
  8214. cancelLoad = true; // if in progress
  8215. CriticalBlock b(crit);
  8216. if (storeLoaded)
  8217. {
  8218. manager->stop();
  8219. manager->saveStore();
  8220. }
  8221. removeThreadExceptionHandler(manager);
  8222. ::Release(manager);
  8223. manager = NULL;
  8224. }
  8225. void nodeDown(rank_t rank)
  8226. {
  8227. TBD;
  8228. }
  8229. private:
  8230. CCovenSDSManager *manager;
  8231. Linked<IPropertyTree> config;
  8232. Owned<IPropertyTree> sdsConfig;
  8233. bool cancelLoad, storeLoaded;
  8234. CriticalSection crit;
  8235. };
  8236. unsigned SDSLockTimeoutCount = 0;
  8237. unsigned querySDSLockTimeoutCount()
  8238. {
  8239. return SDSLockTimeoutCount;
  8240. }
  8241. ISDSException *MakeSDSException(int errorCode)
  8242. {
  8243. ISDSException *ret = new CSDSException(errorCode);
  8244. return ret;
  8245. }
  8246. ISDSException *MakeSDSException(int errorCode, const char *errorMsg, ...)
  8247. {
  8248. if(errorCode == SDSExcpt_LockTimeout)
  8249. SDSLockTimeoutCount++;
  8250. va_list args;
  8251. va_start(args, errorMsg);
  8252. ISDSException *ret = new CSDSException(errorCode, errorMsg, args);
  8253. va_end(args);
  8254. return ret;
  8255. }
  8256. IDaliServer *createDaliSDSServer(IPropertyTree *config)
  8257. {
  8258. return new CDaliSDSServer(config);
  8259. }
  8260. //////////////////////
  8261. bool applyXmlDeltas(IPropertyTree &root, IIOStream &stream, bool stopOnError)
  8262. {
  8263. class CDeltaProcessor : implements IPTreeNotifyEvent, public CInterface
  8264. {
  8265. unsigned level;
  8266. IPTreeMaker *maker;
  8267. IPropertyTree &store;
  8268. offset_t sectionEndOffset;
  8269. StringAttr headerPath;
  8270. bool stopOnError;
  8271. public:
  8272. IMPLEMENT_IINTERFACE;
  8273. bool hadError;
  8274. CDeltaProcessor(IPropertyTree &_store, bool _stopOnError) : store(_store), stopOnError(_stopOnError), level(0)
  8275. {
  8276. sectionEndOffset = 0;
  8277. hadError = false;
  8278. maker = createRootLessPTreeMaker();
  8279. }
  8280. ~CDeltaProcessor()
  8281. {
  8282. ::Release(maker);
  8283. }
  8284. void apply(IPropertyTree &change, IPropertyTree &currentBranch)
  8285. {
  8286. if (change.getPropBool("@localValue"))
  8287. {
  8288. bool binary = change.isBinary(NULL);
  8289. if (binary)
  8290. {
  8291. MemoryBuffer mb;
  8292. change.getPropBin(NULL, mb);
  8293. currentBranch.setPropBin(NULL, mb.length(), mb.toByteArray());
  8294. }
  8295. else
  8296. currentBranch.setProp(NULL, change.queryProp(NULL));
  8297. }
  8298. else if (change.getPropBool("@appendValue"))
  8299. {
  8300. if (change.queryProp(NULL))
  8301. {
  8302. bool binary=change.isBinary(NULL);
  8303. __int64 index = currentBranch.getPropInt64(EXT_ATTR);
  8304. MemoryBuffer mb;
  8305. if (index && QUERYINTERFACE(&currentBranch, CServerRemoteTree))
  8306. {
  8307. MemoryBuffer mbv;
  8308. SDSManager->getExternalValue(index, mbv);
  8309. CPTValue v(mbv);
  8310. v.getValue(mb, binary);
  8311. }
  8312. else
  8313. currentBranch.getPropBin(NULL, mb);
  8314. change.getPropBin(NULL, mb);
  8315. if (binary)
  8316. currentBranch.setPropBin(NULL, mb.length(), mb.toByteArray());
  8317. else
  8318. currentBranch.setProp(NULL, (const char *)mb.toByteArray());
  8319. }
  8320. }
  8321. Owned<IPropertyTreeIterator> iter = change.getElements(RENAME_TAG);
  8322. ForEach (*iter)
  8323. {
  8324. IPropertyTree &d = iter->query();
  8325. StringBuffer xpath(d.queryProp("@from"));
  8326. xpath.append('[').append(d.queryProp("@pos")).append(']');
  8327. verifyex(currentBranch.renameProp(xpath.str(), d.queryProp("@to")));
  8328. }
  8329. iter.setown(change.getElements(DELETE_TAG));
  8330. ForEach (*iter)
  8331. {
  8332. IPropertyTree &d = iter->query();
  8333. StringBuffer xpath(d.queryProp("@name"));
  8334. xpath.append('[').append(d.queryProp("@pos")).append(']');
  8335. if (!currentBranch.removeProp(xpath.str()))
  8336. LOG(MCoperatorWarning, unknownJob, "Property '%s' missing, but recorded as being present at time of delete, in section '%s'", xpath.str(), headerPath.get());
  8337. }
  8338. IPropertyTree *ac = change.queryPropTree(ATTRCHANGE_TAG);
  8339. if (ac)
  8340. {
  8341. Owned<IAttributeIterator> aIter = ac->getAttributes();
  8342. ForEach (*aIter)
  8343. currentBranch.setProp(aIter->queryName(), aIter->queryValue());
  8344. }
  8345. IPropertyTree *ad = change.queryPropTree(ATTRDELETE_TAG);
  8346. if (ad)
  8347. {
  8348. Owned<IAttributeIterator> aIter = ad->getAttributes();
  8349. ForEach (*aIter)
  8350. {
  8351. if (!currentBranch.removeProp(aIter->queryName()))
  8352. LOG(MCoperatorWarning, unknownJob, "Property '%s' missing, but recorded as being present at time of delete, in section '%s'", aIter->queryName(), headerPath.get());
  8353. }
  8354. }
  8355. processChildren(change, currentBranch);
  8356. }
  8357. void processChildren(IPropertyTree &change, IPropertyTree &currentBranch)
  8358. {
  8359. // process children
  8360. Owned<IPropertyTreeIterator> iter = change.getElements("T");
  8361. ForEach (*iter)
  8362. {
  8363. try
  8364. {
  8365. IPropertyTree &child = iter->query();
  8366. const char *name = child.queryProp("@name");
  8367. if (child.getPropBool("@new"))
  8368. {
  8369. IPropertyTree *newBranch = currentBranch.addPropTree(name, createPTree());
  8370. apply(child, *newBranch);
  8371. }
  8372. else if (child.getPropBool("@replace"))
  8373. {
  8374. IPropertyTree *newBranch = currentBranch.setPropTree(name, createPTree());
  8375. apply(child, *newBranch);
  8376. }
  8377. else
  8378. {
  8379. const char *pos = child.queryProp("@pos");
  8380. if (!pos)
  8381. throw MakeStringException(0, "Missing position attribute in child reference, section end offset=%" I64F "d", sectionEndOffset);
  8382. StringBuffer xpath(name);
  8383. xpath.append('[').append(pos).append(']');
  8384. IPropertyTree *existingBranch = currentBranch.queryPropTree(xpath.str());
  8385. if (!existingBranch)
  8386. throw MakeStringException(0, "Failed to locate delta change in %s, section end offset=%" I64F "d", xpath.str(), sectionEndOffset);
  8387. apply(child, *existingBranch);
  8388. }
  8389. }
  8390. catch (IException *e)
  8391. {
  8392. StringBuffer s("Error processing delta section: sectionEndOffset=");
  8393. LOG(MCoperatorError, unknownJob, e, s.append(sectionEndOffset).str());
  8394. if (stopOnError) throw;
  8395. hadError = true;
  8396. e->Release();
  8397. }
  8398. }
  8399. }
  8400. void process(IPropertyTree &match, offset_t endOffset)
  8401. {
  8402. sectionEndOffset = endOffset;
  8403. const char *xpath = match.queryProp("@path");
  8404. if (xpath && '/' == *xpath)
  8405. xpath++;
  8406. IPropertyTree *root = store.queryPropTree(xpath);
  8407. if (!root)
  8408. throw MakeStringException(0, "Failed to locate header xpath = %s", xpath);
  8409. IPropertyTree *start = match.queryPropTree("Delta/T");
  8410. if (!start)
  8411. throw MakeStringException(0, "Badly constructed delta format (missing Delta/T) in header path=%s, section end offset=%" I64F "d", xpath, endOffset);
  8412. headerPath.set(xpath);
  8413. apply(*start, *root);
  8414. }
  8415. // IPTreeNotifyEvent
  8416. virtual void beginNode(const char *tag, offset_t startOffset) { maker->beginNode(tag, startOffset); }
  8417. virtual void newAttribute(const char *name, const char *value) { maker->newAttribute(name, value); }
  8418. virtual void beginNodeContent(const char *tag) { level++; }
  8419. virtual void endNode(const char *tag, unsigned length, const void *value, bool binary, offset_t endOffset)
  8420. {
  8421. try
  8422. {
  8423. --level;
  8424. IPropertyTree *match = NULL;
  8425. if (0 == level)
  8426. {
  8427. if (0 == strcmp("Header", tag))
  8428. match = maker->queryCurrentNode();
  8429. }
  8430. maker->endNode(tag, length, value, binary, endOffset);
  8431. if (match)
  8432. {
  8433. process(*match, endOffset);
  8434. verifyex(maker->queryRoot()->removeTree(match)); // no longer needed.
  8435. }
  8436. }
  8437. catch (IException *e)
  8438. {
  8439. StringBuffer s("Error processing delta section: sectionEndOffset=");
  8440. LOG(MCoperatorError, unknownJob, e, s.append(endOffset).str());
  8441. if (stopOnError) throw;
  8442. hadError = true;
  8443. e->Release();
  8444. }
  8445. }
  8446. } deltaProcessor(root, stopOnError);
  8447. Owned<IPullPTreeReader> xmlReader = createPullXMLStreamReader(stream, deltaProcessor, (PTreeReaderOptions)((unsigned)ptr_ignoreWhiteSpace+(unsigned)ptr_noRoot), false);
  8448. try
  8449. {
  8450. xmlReader->load();
  8451. }
  8452. catch (IException *e)
  8453. {
  8454. if (stopOnError)
  8455. throw;
  8456. LOG(MCoperatorError, unknownJob, e, "XML parse error on delta load - load truncated");
  8457. e->Release();
  8458. }
  8459. return !deltaProcessor.hadError;
  8460. }
  8461. void LogRemoteConn(IRemoteConnection *conn)
  8462. {
  8463. CConnectionBase *conbase = QUERYINTERFACE(conn,CConnectionBase);
  8464. if (!conbase)
  8465. {
  8466. PROGLOG("Could not get base for %x",(unsigned)(memsize_t)conn);
  8467. return;
  8468. }
  8469. IPropertyTree *root = conn->queryRoot();
  8470. CRemoteTreeBase *remotetree = root?QUERYINTERFACE(root,CRemoteTreeBase):NULL;
  8471. unsigned rcount = remotetree?remotetree->getLinkCount()-1:((unsigned)-1);
  8472. PROGLOG("CONN(%x,%" I64F "x,%" I64F "x) path = '%s' mode = %x, link %d,%d",
  8473. (unsigned)(memsize_t)conn,
  8474. (__int64)conbase->querySessionId(),
  8475. (__int64)conbase->queryConnectionId(),
  8476. conbase->queryXPath(),
  8477. conbase->queryMode(),
  8478. conbase->getLinkCount()-1,
  8479. rcount);
  8480. }
  8481. #ifdef _POOLED_SERVER_REMOTE_TREE
  8482. MODULE_INIT(INIT_PRIORITY_STANDARD)
  8483. {
  8484. CServerRemoteTree_Allocator = new CFixedSizeAllocator(sizeof(CServerRemoteTree));
  8485. return true;
  8486. }
  8487. MODULE_EXIT()
  8488. {
  8489. delete CServerRemoteTree_Allocator;
  8490. }
  8491. #endif