jfile.cpp 178 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645
  1. /*##############################################################################
  2. HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. ############################################################################## */
  13. #include "platform.h"
  14. #ifdef _WIN32
  15. #include <errno.h>
  16. //#include <winsock.h> // for TransmitFile
  17. #endif
  18. #include <sys/types.h>
  19. #include <sys/stat.h>
  20. #include <algorithm>
  21. #if defined (__linux__) || defined (__APPLE__)
  22. #include <time.h>
  23. #include <dirent.h>
  24. #include <utime.h>
  25. #include <sys/syscall.h>
  26. #include <sys/mman.h>
  27. #endif
  28. #if defined (__linux__)
  29. #include <sys/vfs.h>
  30. #include <sys/sendfile.h>
  31. #endif
  32. #include "time.h"
  33. #include "jerror.hpp"
  34. #include "jlib.hpp"
  35. #include "jio.hpp"
  36. #include "jmisc.hpp"
  37. #include "jsort.hpp"
  38. #include "jmutex.hpp"
  39. #include "jfile.hpp"
  40. #include "jfile.ipp"
  41. #include <limits.h>
  42. #include "jexcept.hpp"
  43. #include "jsocket.hpp" // for IpAddress
  44. #include "jregexp.hpp"
  45. #include "portlist.h"
  46. #include "build-config.h"
  47. #include "jprop.hpp"
  48. // #define REMOTE_DISCONNECT_ON_DESTRUCTOR // enable to disconnect on IFile destructor
  49. // this should not be enabled in WindowRemoteDirectory used
  50. #ifdef _DEBUG
  51. #define ASSERTEX(e) assertex(e);
  52. #else
  53. #define ASSERTEX(e)
  54. #endif
  55. #ifdef __64BIT__
  56. #define DEFAULT_STREAM_BUFFER_SIZE 0x100000
  57. #else
  58. // Restrict buffer sizes on 32-bit systems
  59. #define DEFAULT_STREAM_BUFFER_SIZE 0x10000
  60. #endif
  61. #ifdef _WIN32
  62. #define NULLFILE INVALID_HANDLE_VALUE
  63. #else
  64. #define NULLFILE -1
  65. #endif
  66. // #define CFILEIOTRACE 1
  67. static IFile *createIFileByHook(const RemoteFilename & filename);
  68. static IFile *createContainedIFileByHook(const char *filename);
  69. static inline bool isPCFlushAllowed();
  70. static char ShareChar='$';
  71. bool isShareChar(char c)
  72. {
  73. return (c==ShareChar)||(c=='$');
  74. }
  75. char getShareChar()
  76. {
  77. return ShareChar;
  78. }
  79. void setShareChar(char c)
  80. {
  81. ShareChar = c;
  82. }
  83. StringBuffer &setPathDrive(StringBuffer &filename,unsigned drvnum)
  84. {
  85. return swapPathDrive(filename,(unsigned)-1,drvnum);
  86. }
  87. unsigned getPathDrive(const char *s)
  88. {
  89. char c = *s;
  90. if (isPathSepChar(c)&&(s[1]==c)) {
  91. s = strchr(s+2,c);
  92. if (!s)
  93. return 0;
  94. }
  95. if (isPathSepChar(c))
  96. s++;
  97. if (*s&&((s[1]==':')||(isShareChar(s[1]))))
  98. return *s-'c';
  99. return 0;
  100. }
  101. StringBuffer &swapPathDrive(StringBuffer &filename,unsigned fromdrvnum,unsigned todrvnum,const char *frommask,const char *tomask)
  102. {
  103. const char *s = filename.str();
  104. char c = *s;
  105. if (isPathSepChar(c)&&(s[1]==c)) {
  106. s = strchr(s+2,c);
  107. if (!s)
  108. return filename;
  109. }
  110. if (isPathSepChar(c))
  111. s++;
  112. if (*s&&((fromdrvnum==(unsigned)-1)||(*s==(fromdrvnum+'c')))&&((s[1]==':')||(isShareChar(s[1]))))
  113. filename.setCharAt((size32_t)(s-filename.str()),todrvnum+'c');
  114. else if (frommask&&*frommask) { // OSS
  115. StringBuffer tmp;
  116. if (replaceConfigurationDirectoryEntry(filename.str(),frommask,tomask, tmp))
  117. tmp.swapWith(filename);
  118. }
  119. return filename;
  120. }
  121. const char *pathTail(const char *path)
  122. {
  123. if (!path)
  124. return NULL;
  125. const char *tail=path;
  126. const char *s = path;
  127. while (*s)
  128. if (isPathSepChar(*(s++)))
  129. tail = s;
  130. return tail;
  131. }
  132. const char * pathExtension(const char * path)
  133. {
  134. const char * tail = pathTail(path);
  135. if (tail)
  136. return strrchr(tail, '.');
  137. return NULL;
  138. }
  139. bool checkFileExists(const char * filename)
  140. {
  141. #ifdef _WIN32
  142. for (unsigned i=0;i<10;i++) {
  143. DWORD ret = (DWORD)GetFileAttributes(filename);
  144. if (ret!=(DWORD)-1)
  145. return true;
  146. DWORD err = GetLastError();
  147. if (err!=ERROR_IO_PENDING)
  148. break;
  149. Sleep(100*i);
  150. }
  151. return false;
  152. #else
  153. struct stat info;
  154. return (stat(filename, &info) == 0);
  155. #endif
  156. }
  157. bool checkDirExists(const char * filename)
  158. {
  159. #ifdef _WIN32
  160. DWORD attr = GetFileAttributes(filename);
  161. return (attr != (DWORD)-1)&&(attr & FILE_ATTRIBUTE_DIRECTORY);
  162. #else
  163. struct stat info;
  164. if (stat(filename, &info) != 0)
  165. return false;
  166. return S_ISDIR(info.st_mode);
  167. #endif
  168. }
  169. static void set_inherit(HANDLE handle, bool inherit)
  170. {
  171. #ifndef _WIN32
  172. long flag = fcntl(handle, F_GETFD);
  173. if(inherit)
  174. flag &= ~FD_CLOEXEC;
  175. else
  176. flag |= FD_CLOEXEC;
  177. fcntl(handle, F_SETFD, flag);
  178. #endif
  179. }
  180. static StringBuffer &getLocalOrRemoteName(StringBuffer &name,const RemoteFilename & filename)
  181. {
  182. if (filename.isLocal()&&!filename.isNull()) {
  183. filename.getLocalPath(name);
  184. #ifdef _WIN32
  185. // kludge to allow local linux paths on windows
  186. const char *n=name.str();
  187. if (n[0]=='/') {
  188. StringBuffer tmp;
  189. if (isShareChar(n[2])) {
  190. tmp.append(n[1]).append(':');
  191. n+=3;
  192. }
  193. while (*n) {
  194. if (*n=='/')
  195. tmp.append('\\');
  196. else
  197. tmp.append(*n);
  198. n++;
  199. }
  200. name.clear().append(tmp);
  201. }
  202. #endif
  203. }
  204. else
  205. filename.getRemotePath(name);
  206. return name;
  207. }
  208. CFile::CFile(const char * _filename)
  209. {
  210. filename.set(_filename);
  211. flags = ((unsigned)IFSHread)|((S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH)<<16);
  212. }
  213. void CFile::setCreateFlags(unsigned cflags)
  214. {
  215. flags |= (cflags<<16);
  216. }
  217. void CFile::setShareMode(IFSHmode shmode)
  218. {
  219. flags &= ~(IFSHfull|IFSHread);
  220. flags |= (unsigned)(shmode&(IFSHfull|IFSHread));
  221. }
  222. bool CFile::exists()
  223. {
  224. if (stdIoHandle(filename)>=0)
  225. return true;
  226. return checkFileExists(filename);
  227. }
  228. #ifdef _WIN32
  229. bool WindowsCreateDirectory(const char * path)
  230. {
  231. unsigned retry = 0;
  232. loop {
  233. if (CreateDirectory(path, NULL))
  234. return true;
  235. DWORD err = GetLastError();
  236. if ((err==ERROR_FILE_NOT_FOUND) || (err==ERROR_PATH_NOT_FOUND) || (err==ERROR_FILE_EXISTS) || (err==ERROR_CANNOT_MAKE))
  237. break;
  238. else if (err==ERROR_ALREADY_EXISTS) {
  239. DWORD attr = GetFileAttributes(path);
  240. if ((attr != -1)&&( attr & FILE_ATTRIBUTE_DIRECTORY))
  241. return true;
  242. return false;
  243. }
  244. if ((retry++==10)|| // some or all of the following can occur when the domain controller gets busy
  245. // retrying improves chance of success
  246. ((err!=ERROR_NETNAME_DELETED)&&(err!=ERROR_DEV_NOT_EXIST)&&(err!=ERROR_GEN_FAILURE)&&(err!=ERROR_NETWORK_BUSY)&&(err!=ERROR_BAD_NET_NAME)))
  247. throw makeOsExceptionV(err,"WindowsCreateDirectory %s", path);
  248. //PROGLOG("Retrying(%d) WindowsCreateDirectory %s, err=%d",retry,filename,err);
  249. Sleep(retry*100);
  250. }
  251. return false;
  252. }
  253. #else
  254. bool LinuxCreateDirectory(const char * path)
  255. {
  256. if (!path)
  257. return false;
  258. if (CreateDirectory(path, NULL))
  259. return true;
  260. else
  261. {
  262. if (EEXIST == errno)
  263. {
  264. struct stat info;
  265. if (stat(path, &info) != 0)
  266. return false;
  267. return S_ISDIR(info.st_mode);
  268. }
  269. }
  270. return false;
  271. }
  272. #endif
  273. bool localCreateDirectory(const char *name)
  274. {
  275. if (!name)
  276. return false;
  277. size32_t l = (size32_t)strlen(name);
  278. if (l==0)
  279. return true;
  280. if (isPathSepChar(name[0])&&((l==1)||(isPathSepChar(name[1])&&!containsPathSepChar(name+2))))
  281. return true;
  282. #ifdef _WIN32
  283. if (name[1]==':') {
  284. if ((l==2)||((l==3)&&isPathSepChar(name[2])))
  285. return true;
  286. }
  287. #endif
  288. if (checkDirExists(name))
  289. return true;
  290. #ifdef _WIN32
  291. if (WindowsCreateDirectory(name))
  292. #else
  293. if (LinuxCreateDirectory(name))
  294. #endif
  295. return true;
  296. if (isPathSepChar(name[l-1])) l--;
  297. while (l&&!isPathSepChar(name[l-1]))
  298. l--;
  299. if (l<=1)
  300. return true;
  301. StringAttr parent(name,l-1);
  302. if (!localCreateDirectory(parent.get()))
  303. return false;
  304. #ifdef _WIN32
  305. return (WindowsCreateDirectory(name));
  306. #else
  307. return (LinuxCreateDirectory(name));
  308. #endif
  309. }
  310. bool CFile::createDirectory()
  311. {
  312. return localCreateDirectory(filename);
  313. }
  314. #ifdef _WIN32
  315. union TimeIntegerUnion
  316. {
  317. FILETIME ft;
  318. ULARGE_INTEGER l;
  319. };
  320. void FILETIMEtoIDateTime(CDateTime * target, const FILETIME & ft)
  321. {
  322. if (target)
  323. {
  324. TimeIntegerUnion u;
  325. memcpy(&u, &ft, sizeof(ft));
  326. unsigned __int64 hundredNanoseconds = (u.l.QuadPart % 10000000);
  327. u.l.QuadPart -= hundredNanoseconds;
  328. SYSTEMTIME systime;
  329. FileTimeToSystemTime(&u.ft, &systime);
  330. target->set(systime.wYear, systime.wMonth, systime.wDay, systime.wHour, systime.wMinute, systime.wSecond, (unsigned)(hundredNanoseconds*100));
  331. }
  332. }
  333. FILETIME * IDateTimetoFILETIME(FILETIME & ft, const CDateTime * dt)
  334. {
  335. if (!dt)
  336. return NULL;
  337. SYSTEMTIME systime;
  338. TimeIntegerUnion u;
  339. unsigned wYear, wMonth, wDay, wHour, wMinute, wSecond, nanoSeconds;
  340. dt->getDate(wYear, wMonth, wDay);
  341. dt->getTime(wHour, wMinute, wSecond, nanoSeconds);
  342. systime.wYear = wYear;
  343. systime.wMonth = wMonth;
  344. systime.wDay = wDay;
  345. systime.wHour = wHour;
  346. systime.wMinute = wMinute;
  347. systime.wSecond = wSecond;
  348. systime.wMilliseconds = 0;
  349. SystemTimeToFileTime(&systime, &u.ft);
  350. //Adjust the fractions of a second ourselves because the function above is only accurate to milli seconds.
  351. u.l.QuadPart += (nanoSeconds / 100);
  352. memcpy(&ft, &u, sizeof(ft));
  353. return &ft;
  354. }
  355. #endif
  356. bool CFile::getTime(CDateTime * createTime, CDateTime * modifiedTime, CDateTime * accessedTime)
  357. {
  358. #ifdef _WIN32
  359. //MORE could use GetFileAttributesEx() if we were allowed...
  360. FILETIME timeCreated, timeModified, timeAccessed;
  361. HANDLE handle = CreateFile(filename, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, 0, NULL);
  362. if (handle==(HANDLE)-1)
  363. return false;
  364. GetFileTime(handle, &timeCreated, &timeAccessed, &timeModified);
  365. CloseHandle(handle);
  366. FILETIMEtoIDateTime(createTime, timeCreated);
  367. FILETIMEtoIDateTime(modifiedTime, timeModified);
  368. FILETIMEtoIDateTime(accessedTime, timeAccessed);
  369. #else
  370. struct stat info;
  371. if (stat(filename, &info) != 0)
  372. return false;
  373. timetToIDateTime(accessedTime, info.st_atime);
  374. timetToIDateTime(createTime, info.st_ctime);
  375. timetToIDateTime(modifiedTime, info.st_mtime);
  376. #endif
  377. return true;
  378. }
  379. bool CFile::setTime(const CDateTime * createTime, const CDateTime * modifiedTime, const CDateTime * accessedTime)
  380. {
  381. #ifdef _WIN32
  382. FILETIME timeCreated, timeModified, timeAccessed;
  383. FILETIME *pTimeCreated, *pTimeModified, *pTimeAccessed;
  384. pTimeCreated = IDateTimetoFILETIME(timeCreated, createTime);
  385. pTimeModified = IDateTimetoFILETIME(timeModified, modifiedTime);
  386. pTimeAccessed = IDateTimetoFILETIME(timeAccessed, accessedTime);
  387. HANDLE handle = CreateFile(filename, GENERIC_WRITE, FILE_SHARE_READ, NULL, OPEN_EXISTING, 0, NULL);
  388. if (!handle)
  389. return false;
  390. SetFileTime(handle, pTimeCreated, pTimeAccessed, pTimeModified);
  391. CloseHandle(handle);
  392. return true;
  393. #else
  394. struct utimbuf am;
  395. if (!accessedTime||!modifiedTime) {
  396. struct stat info;
  397. if (stat(filename, &info) != 0)
  398. return false;
  399. am.actime = info.st_atime;
  400. am.modtime = info.st_mtime;
  401. }
  402. if (accessedTime)
  403. am.actime = timetFromIDateTime (accessedTime);
  404. if (modifiedTime)
  405. am.modtime = timetFromIDateTime (modifiedTime);
  406. return (utime(filename, &am)==0);
  407. #endif
  408. }
  409. fileBool CFile::isDirectory()
  410. {
  411. #ifdef _WIN32
  412. DWORD attr = GetFileAttributes(filename);
  413. if (attr == -1)
  414. return notFound;
  415. return ( attr & FILE_ATTRIBUTE_DIRECTORY) ? foundYes : foundNo;
  416. #else
  417. struct stat info;
  418. if (stat(filename, &info) != 0)
  419. return notFound;
  420. return S_ISDIR(info.st_mode) ? foundYes : foundNo;
  421. #endif
  422. }
  423. fileBool CFile::isFile()
  424. {
  425. if (stdIoHandle(filename)>=0)
  426. return foundYes;
  427. #ifdef _WIN32
  428. DWORD attr = GetFileAttributes(filename);
  429. if (attr == -1)
  430. return notFound;
  431. return ( attr & FILE_ATTRIBUTE_DIRECTORY) ? foundNo : foundYes;
  432. #else
  433. struct stat info;
  434. if (stat(filename, &info) != 0)
  435. return notFound;
  436. return S_ISREG(info.st_mode) ? foundYes : foundNo;
  437. #endif
  438. }
  439. fileBool CFile::isReadOnly()
  440. {
  441. #ifdef _WIN32
  442. DWORD attr = GetFileAttributes(filename);
  443. if (attr == -1)
  444. return notFound;
  445. return ( attr & FILE_ATTRIBUTE_READONLY) ? foundYes : foundNo;
  446. #else
  447. struct stat info;
  448. if (stat(filename, &info) != 0)
  449. return notFound;
  450. //MORE: I think this is correct, but someone with better unix knowledge should check!
  451. return (info.st_mode & (S_IWUSR|S_IWGRP|S_IWOTH)) ? foundNo : foundYes;
  452. #endif
  453. }
  454. #ifndef _WIN32
  455. static bool setShareLock(int fd,IFSHmode share)
  456. {
  457. struct flock fl;
  458. do {
  459. memset(&fl,0,sizeof(fl));
  460. if (share==IFSHnone)
  461. fl.l_type = F_WRLCK;
  462. else if (share==IFSHread)
  463. fl.l_type = F_RDLCK;
  464. else
  465. fl.l_type = F_UNLCK;
  466. fl.l_whence = SEEK_SET;
  467. fl.l_pid = getpid();
  468. if (fcntl(fd, F_SETLK, &fl) != -1)
  469. return true;
  470. } while (errno==EINTR);
  471. if ((errno!=EAGAIN)&&(errno!=EACCES))
  472. ERRLOG("setShareLock failure %d, mode %d", errno,(int)share);
  473. return (share==IFSHfull); // always return true for full
  474. }
  475. #endif
  476. HANDLE CFile::openHandle(IFOmode mode, IFSHmode sharemode, bool async, int stdh)
  477. {
  478. HANDLE handle = NULLFILE;
  479. #ifdef _WIN32
  480. if (stdh>=0) {
  481. DWORD mode;
  482. switch (stdh) {
  483. case 0: mode = STD_INPUT_HANDLE; break;
  484. case 1: mode = STD_OUTPUT_HANDLE; break;
  485. case 2: mode = STD_ERROR_HANDLE; break;
  486. default:
  487. return handle;
  488. }
  489. DuplicateHandle(GetCurrentProcess(), GetStdHandle(mode), GetCurrentProcess(), &handle , 0, FALSE, DUPLICATE_SAME_ACCESS);
  490. return handle;
  491. }
  492. DWORD share = 0;
  493. switch (sharemode) {
  494. case (IFSHfull|IFSHread):
  495. case IFSHfull:
  496. share |= FILE_SHARE_WRITE;
  497. // fall through
  498. case IFSHread:
  499. share |= FILE_SHARE_READ;
  500. }
  501. DWORD fflags = async?FILE_FLAG_OVERLAPPED:0;
  502. if (async&&(mode==IFOread))
  503. fflags |= FILE_FLAG_SEQUENTIAL_SCAN;
  504. switch (mode) {
  505. case IFOcreate:
  506. handle = CreateFile(filename, GENERIC_WRITE, share, NULL, CREATE_ALWAYS, fflags, NULL);
  507. break;
  508. case IFOread:
  509. handle = CreateFile(filename, GENERIC_READ, share, NULL, OPEN_EXISTING, fflags, NULL);
  510. break;
  511. case IFOwrite:
  512. handle = CreateFile(filename, GENERIC_WRITE, share, NULL, OPEN_ALWAYS, fflags, NULL);
  513. break;
  514. case IFOcreaterw:
  515. handle = CreateFile(filename, GENERIC_WRITE|GENERIC_READ, share, NULL, CREATE_ALWAYS, fflags, NULL);
  516. break;
  517. case IFOreadwrite:
  518. handle = CreateFile(filename, GENERIC_WRITE|GENERIC_READ, share, NULL, OPEN_ALWAYS, fflags, NULL);
  519. break;
  520. }
  521. if (handle == NULLFILE)
  522. {
  523. DWORD err = GetLastError();
  524. if ((IFOread!=mode) || ((err!=ERROR_FILE_NOT_FOUND) && (err!=ERROR_PATH_NOT_FOUND)))
  525. throw makeOsExceptionV(err,"CFile::open %s (%x, %x)", filename.get(), mode, share);
  526. return NULLFILE;
  527. }
  528. #else
  529. if (stdh>=0)
  530. return (HANDLE)dup(stdh);
  531. unsigned fileflags = (flags>>16) & (S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IWGRP|S_IXGRP|S_IROTH|S_IWOTH|S_IXOTH);
  532. unsigned openflags;
  533. switch (mode) {
  534. case IFOcreate:
  535. openflags = O_WRONLY | O_CREAT | O_TRUNC;
  536. break;
  537. case IFOread:
  538. openflags = O_RDONLY;
  539. break;
  540. case IFOwrite:
  541. openflags = O_WRONLY | O_CREAT;
  542. break;
  543. case IFOcreaterw:
  544. openflags = O_RDWR | O_CREAT | O_TRUNC;
  545. break;
  546. case IFOreadwrite:
  547. openflags = O_RDWR | O_CREAT;
  548. break;
  549. default:
  550. return NULLFILE;
  551. }
  552. handle = _lopen(filename.get(), openflags, fileflags);
  553. if (handle == -1)
  554. {
  555. if ((IFOread!=mode) || (errno != ENOENT))
  556. throw makeErrnoExceptionV(errno, "CFile::open %s", filename.get());
  557. return NULLFILE;
  558. }
  559. // check not a directory (compatible with windows)
  560. struct stat info;
  561. if (fstat(handle, &info) == -1) {
  562. int err = errno;
  563. close(handle);
  564. handle = NULLFILE;
  565. throw makeErrnoExceptionV(err, "CFile::open fstat %s", filename.get());
  566. }
  567. if (S_ISDIR(info.st_mode)) {
  568. close(handle);
  569. handle = NULLFILE;
  570. throw makeErrnoExceptionV(EISDIR, "CFile::open %s", filename.get());
  571. }
  572. #ifdef CFILEIOTRACE
  573. DBGLOG("CFile::openHandle(%s,%d) returns %d", filename.get(), mode, handle);
  574. #endif
  575. #endif
  576. return handle;
  577. }
  578. IFileIO * CFile::open(IFOmode mode,IFEflags extraFlags)
  579. {
  580. // we may want mode dependant defaults later
  581. return openShared(mode,(IFSHmode)(flags&(IFSHfull|IFSHread)),extraFlags);
  582. }
  583. IFileAsyncIO * CFile::openAsync(IFOmode mode)
  584. {
  585. HANDLE handle = openHandle(mode,IFSHread,true); // I don't think we want shared write to an async file
  586. #ifndef _WIN32
  587. set_inherit(handle, false);
  588. #endif
  589. return new CFileAsyncIO(handle,IFSHread);
  590. }
  591. const char * CFile::queryFilename()
  592. {
  593. return filename;
  594. }
  595. bool CFile::remove()
  596. {
  597. #ifdef _WIN32
  598. unsigned retry = 0;
  599. loop {
  600. if (isDirectory()==foundYes) {
  601. if (RemoveDirectory(filename) != 0)
  602. return true;
  603. }
  604. else {
  605. if (DeleteFile(filename) != 0)
  606. return true;
  607. }
  608. DWORD err = GetLastError();
  609. if ( (err==ERROR_FILE_NOT_FOUND) || (err==ERROR_PATH_NOT_FOUND) )
  610. break;
  611. if ((retry++==10)|| // some or all of the following can occur when the domain controller gets busy
  612. // retrying improves chance of success
  613. ((err!=ERROR_NETNAME_DELETED)&&(err!=ERROR_DEV_NOT_EXIST)&&(err!=ERROR_GEN_FAILURE)&&(err!=ERROR_NETWORK_BUSY)&&(err!=ERROR_BAD_NET_NAME)))
  614. throw makeOsExceptionV(err, "CFile::remove %s", filename.get());
  615. //PROGLOG("Retrying(%d) DeleteFile %s, err=%d",retry,filename,err);
  616. Sleep(retry*100);
  617. }
  618. return false;
  619. #else
  620. if (isDirectory()==foundYes) {
  621. if (rmdir(filename) == 0)
  622. return true;
  623. }
  624. else {
  625. if (unlink(filename) == 0)
  626. return true;
  627. }
  628. if (ENOENT!=errno)
  629. throw makeErrnoExceptionV("CFile::remove %s", filename.get());
  630. return false;
  631. #endif
  632. }
  633. void CFile::rename(const char *newname)
  634. {
  635. // now hopefully newname is just file tail
  636. // however we do allow full paths and extract tail, warning if the directory appears to mismatch
  637. StringBuffer path;
  638. splitDirTail(filename,path);
  639. StringBuffer newdir;
  640. const char *tail = splitDirTail(newname,newdir);
  641. if (path.length()&&newdir.length()) {
  642. if (strcmp(newdir.str(),path.str())!=0)
  643. WARNLOG("CFile::rename '%s' to '%s' : directory mismatch",filename.get(),newname);
  644. }
  645. const char *dst = path.append(tail);
  646. if (isPathSepChar(dst[0])&&(dst[1]==dst[0])) { // hmm is share - convert to local path
  647. RemoteFilename rfn;
  648. rfn.setRemotePath(dst);
  649. if (rfn.isLocal())
  650. dst = rfn.getLocalPath(path.clear()).str();
  651. }
  652. if (-1 == ::rename(filename, dst))
  653. throw makeErrnoExceptionV("CFile::rename(%s, %s)", filename.get(),dst);
  654. filename.set(path);
  655. }
  656. void CFile::move(const char *newname)
  657. {
  658. if (!newname||!*newname)
  659. return;
  660. StringBuffer path;
  661. if (isPathSepChar(newname[0])&&(newname[1]==newname[0])) { // hmm is share - convert to local path
  662. RemoteFilename rfn;
  663. rfn.setRemotePath(newname);
  664. if (rfn.isLocal())
  665. newname = rfn.getLocalPath(path.clear()).str();
  666. }
  667. #ifdef _WIN32
  668. unsigned retry = 0;
  669. loop {
  670. if (MoveFileEx(filename.get(),newname,0))
  671. return;
  672. DWORD err = GetLastError();
  673. if ((retry++==10)|| // some or all of the following can occur when the domain controller gets busy
  674. // retrying improves chance of success
  675. ((err!=ERROR_NETNAME_DELETED)&&(err!=ERROR_DEV_NOT_EXIST)&&(err!=ERROR_GEN_FAILURE)&&(err!=ERROR_NETWORK_BUSY)&&(err!=ERROR_BAD_NET_NAME)))
  676. throw makeOsExceptionV(err, "CFile::move(%s, %s)", filename.get(), newname);
  677. Sleep(retry*100);
  678. }
  679. #else
  680. int ret=::rename(filename.get(),newname);
  681. if (ret==-1)
  682. throw makeErrnoExceptionV("CFile::move(%s, %s)", filename.get(), newname);
  683. #endif
  684. filename.set(newname);
  685. }
  686. #ifdef _WIN32
  687. DWORD CALLBACK fastCopyProgressRoutine(
  688. LARGE_INTEGER TotalFileSize,
  689. LARGE_INTEGER TotalBytesTransferred,
  690. LARGE_INTEGER StreamSize,
  691. LARGE_INTEGER StreamBytesTransferred,
  692. DWORD dwStreamNumber,
  693. DWORD dwCallbackReason,
  694. HANDLE hSourceFile,
  695. HANDLE hDestinationFile,
  696. LPVOID lpData
  697. )
  698. {
  699. if (TotalBytesTransferred.QuadPart<=TotalFileSize.QuadPart)
  700. {
  701. CFPmode status = ((ICopyFileProgress *)lpData)->onProgress(TotalBytesTransferred.QuadPart,TotalFileSize.QuadPart);
  702. switch (status)
  703. {
  704. case CFPcontinue:
  705. return PROGRESS_CONTINUE;
  706. case CFPstop:
  707. return PROGRESS_STOP;
  708. default:
  709. return PROGRESS_CANCEL;
  710. }
  711. }
  712. return PROGRESS_CONTINUE;
  713. }
  714. #endif
  715. bool CFile::fastCopyFile(CFile &target, size32_t buffersize, ICopyFileProgress *progress)
  716. {
  717. // only currently supported for windows
  718. #ifdef _WIN32
  719. unsigned retry = 0;
  720. loop {
  721. BOOL cancel=FALSE;
  722. if (CopyFileEx(queryFilename(),target.queryFilename(),progress?fastCopyProgressRoutine:NULL,progress?progress:NULL,&cancel,0))
  723. break;
  724. DWORD err = GetLastError();
  725. if ( (err==ERROR_FILE_NOT_FOUND) || (err==ERROR_PATH_NOT_FOUND) )
  726. return false;
  727. if ((retry++==10)|| // some or all of the following can occur when the domain controller gets busy
  728. // retrying improves chance of success
  729. ((err!=ERROR_NETNAME_DELETED)&&(err!=ERROR_DEV_NOT_EXIST)&&(err!=ERROR_GEN_FAILURE)&&(err!=ERROR_NETWORK_BUSY)&&(err!=ERROR_BAD_NET_NAME)))
  730. return false;
  731. Sleep(retry*100);
  732. }
  733. return true;
  734. #else
  735. return false;
  736. #endif
  737. }
  738. void CFile::copySection(const RemoteFilename &dest, offset_t toOfs, offset_t fromOfs, offset_t size, ICopyFileProgress *progress, CFflags copyFlags)
  739. {
  740. // check to see if src and target are remote
  741. Owned<IFile> target = createIFile(dest);
  742. const size32_t buffersize = DEFAULT_COPY_BLKSIZE;
  743. IFOmode omode = IFOwrite;
  744. if (toOfs==(offset_t)-1) {
  745. if (fromOfs==0) {
  746. copyFile(target,this,buffersize,progress,copyFlags);
  747. return;
  748. }
  749. omode = IFOcreate;
  750. toOfs = 0;
  751. }
  752. IFEflags tgtFlags = IFEnone;
  753. if (copyFlags & CFflush_write)
  754. tgtFlags = IFEnocache;
  755. OwnedIFileIO targetIO = target->open(IFOwrite, tgtFlags);
  756. if (!targetIO)
  757. throw MakeStringException(-1, "copyFile: target path '%s' could not be created", target->queryFilename());
  758. MemoryAttr mb;
  759. void * buffer = mb.allocate(buffersize);
  760. IFEflags srcFlags = IFEnone;
  761. if (copyFlags & CFflush_read)
  762. srcFlags = IFEnocache;
  763. OwnedIFileIO sourceIO = open(IFOread, srcFlags);
  764. if (!sourceIO)
  765. throw MakeStringException(-1, "copySection: source '%s' not found", queryFilename());
  766. offset_t offset = 0;
  767. offset_t total;
  768. try
  769. {
  770. total = sourceIO->size();
  771. if (total<fromOfs)
  772. total = 0;
  773. else
  774. total -= fromOfs;
  775. if (total>size)
  776. total = size;
  777. while (offset<total)
  778. {
  779. size32_t got = sourceIO->read(fromOfs+offset, (buffersize>total-offset)?((size32_t)(total-offset)):buffersize, buffer);
  780. if (got == 0)
  781. break;
  782. targetIO->write(offset+toOfs, got, buffer);
  783. offset += got;
  784. if (progress && progress->onProgress(offset, total) != CFPcontinue)
  785. break;
  786. }
  787. }
  788. catch (IException *e)
  789. {
  790. StringBuffer s;
  791. s.append("copyFile target=").append(target->queryFilename()).append(" source=").append(queryFilename()).append("; read/write failure").append(": ");
  792. e->errorMessage(s);
  793. IException *e2 = makeOsExceptionV(e->errorCode(), "%s", s.str());
  794. e->Release();
  795. throw e2;
  796. }
  797. }
  798. void CFile::copyTo(IFile *dest, size32_t buffersize, ICopyFileProgress *progress,bool usetmp,CFflags copyFlags)
  799. {
  800. doCopyFile(dest,this,buffersize,progress,NULL,usetmp,copyFlags);
  801. }
  802. void CFile::setReadOnly(bool ro)
  803. {
  804. #ifdef _WIN32
  805. if (!SetFileAttributes(filename, ro ? FILE_ATTRIBUTE_READONLY : FILE_ATTRIBUTE_NORMAL))
  806. {
  807. DWORD err = GetLastError();
  808. if ( (err!=ERROR_FILE_NOT_FOUND) && (err!=ERROR_PATH_NOT_FOUND) )
  809. throw makeOsExceptionV(err, "CFile::setReadOnly %s", filename.get());
  810. }
  811. #else
  812. struct stat info;
  813. if (stat(filename, &info) != 0)
  814. throw makeErrnoExceptionV("CFile::setReadOnly() %s", filename.get());
  815. // not sure correct but consistant with isReadOnly
  816. if (ro)
  817. info.st_mode &= ~(S_IWUSR|S_IWGRP|S_IWOTH);
  818. else
  819. info.st_mode |= (S_IWUSR|S_IWGRP|S_IWOTH);
  820. chmod(filename, info.st_mode);
  821. #endif
  822. }
  823. offset_t CFile::size()
  824. {
  825. if (stdIoHandle(filename)>=0)
  826. return 0; // dummy value
  827. #ifdef _WIN32
  828. WIN32_FILE_ATTRIBUTE_DATA info;
  829. if (GetFileAttributesEx(filename, GetFileExInfoStandard, &info) != 0) {
  830. LARGE_INTEGER x;
  831. x.LowPart = info.nFileSizeLow;
  832. x.HighPart = info.nFileSizeHigh;
  833. return (offset_t)x.QuadPart;
  834. }
  835. #else
  836. struct stat info;
  837. if (stat(filename, &info) == 0)
  838. return info.st_size;
  839. #endif
  840. #if 0
  841. // we could try opening but I don't think needed
  842. Owned<IFileIO>io = openShared(IFOread,IFSHfull);
  843. if (io)
  844. return io->size();
  845. #endif
  846. return (offset_t)-1;
  847. }
  848. bool CFile::setCompression(bool set)
  849. {
  850. #ifdef _WIN32
  851. DWORD attr=::GetFileAttributes(filename.get());
  852. if(attr==-1)
  853. throw makeOsExceptionV(::GetLastError(), "CFile::setCompression %s", filename.get());
  854. if (((attr & FILE_ATTRIBUTE_COMPRESSED) != 0) == set)
  855. return true;
  856. HANDLE handle=::CreateFile(filename.get(),GENERIC_READ|GENERIC_WRITE,0,NULL,OPEN_EXISTING,FILE_FLAG_BACKUP_SEMANTICS,NULL);
  857. if(handle==INVALID_HANDLE_VALUE)
  858. throw makeOsExceptionV(::GetLastError(), "CFile::setCompression %s", filename.get());
  859. USHORT compression=set ? COMPRESSION_FORMAT_DEFAULT : COMPRESSION_FORMAT_NONE;
  860. DWORD bytes;
  861. if(::DeviceIoControl(handle, FSCTL_SET_COMPRESSION, &compression, sizeof(compression), NULL, 0, &bytes, NULL))
  862. {
  863. ::CloseHandle(handle);
  864. return true;
  865. }
  866. DWORD err=::GetLastError();
  867. ::CloseHandle(handle);
  868. throw makeOsExceptionV(err, "CFile::setCompression %s", filename.get());
  869. #else
  870. return false;
  871. #endif
  872. }
  873. offset_t CFile::compressedSize()
  874. {
  875. #ifdef _WIN32
  876. DWORD hi, lo=::GetCompressedFileSize(filename.get(),&hi), err;
  877. if(lo==INVALID_FILE_SIZE && (err=::GetLastError())!=NO_ERROR)
  878. {
  879. if ( (err!=ERROR_FILE_NOT_FOUND) && (err!=ERROR_PATH_NOT_FOUND) )
  880. throw makeOsExceptionV(err,"CFile::compressedSize %s", filename.get());
  881. return -1;
  882. }
  883. return makeint64(hi,lo);
  884. #else
  885. return size();
  886. #endif
  887. }
  888. class CDiscretionaryFileLock: public CInterface, implements IDiscretionaryLock
  889. {
  890. bool locked;
  891. bool excllock;
  892. Linked<IFile> file;
  893. Linked<IFileIO> fileio;
  894. HANDLE handle;
  895. public:
  896. CDiscretionaryFileLock(IFile *_file)
  897. : file(_file)
  898. {
  899. excllock = false;
  900. locked = false;
  901. #ifdef _WIN32
  902. handle=::CreateFile(file->queryFilename(),GENERIC_READ,FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE,NULL,OPEN_EXISTING,FILE_FLAG_BACKUP_SEMANTICS,NULL);
  903. if(handle==INVALID_HANDLE_VALUE)
  904. {
  905. handle = NULLFILE;
  906. throw makeOsExceptionV(GetLastError(), "CDiscretionaryFileLock::openhandle %s", file->queryFilename());
  907. }
  908. #else
  909. handle = _lopen(file->queryFilename(), O_RDONLY, 0);
  910. if (handle == -1)
  911. {
  912. handle = NULLFILE;
  913. throw makeErrnoExceptionV(errno, "CDiscretionaryFileLock::openhandle %s", file->queryFilename());
  914. }
  915. #endif
  916. }
  917. CDiscretionaryFileLock(IFileIO *_fileio)
  918. : fileio(_fileio)
  919. {
  920. excllock = false;
  921. locked = false;
  922. handle = NULLFILE;
  923. CFileIO *cfileio = QUERYINTERFACE(_fileio,CFileIO);
  924. if (cfileio)
  925. handle = cfileio->queryHandle();
  926. if (handle==NULLFILE)
  927. throw makeStringException(-1, "CDiscretionaryFileLock - invalid parameter");
  928. }
  929. ~CDiscretionaryFileLock()
  930. {
  931. if (locked)
  932. unlock();
  933. if ((handle!=NULLFILE)&&!fileio.get())
  934. #ifdef _WIN32
  935. CloseHandle(handle);
  936. #else
  937. _lclose(handle);
  938. #endif
  939. }
  940. IMPLEMENT_IINTERFACE;
  941. virtual bool isLocked() { return (handle!=NULLFILE); }
  942. virtual bool isExclusiveLocked() { return excllock; }
  943. bool lock(bool exclusive=true, unsigned timeout=INFINITE)
  944. {
  945. if (locked) {
  946. if (exclusive) {
  947. if (excllock)
  948. return true;
  949. unlock(); // have to unlock - unfortunate as means no window-less change mode
  950. }
  951. else {
  952. if (!excllock)
  953. return true;
  954. unlock(); // have to unlock - unfortunate as means no window-less change mode
  955. }
  956. }
  957. unsigned interval = 1;
  958. unsigned start = msTick();
  959. loop {
  960. #ifdef _WIN32
  961. OVERLAPPED overlapped;
  962. memset(&overlapped,0,sizeof(overlapped));
  963. if (LockFileEx(handle,
  964. (exclusive?LOCKFILE_EXCLUSIVE_LOCK:0)|((timeout==INFINITE)?0:LOCKFILE_FAIL_IMMEDIATELY),
  965. 0,1,0,&overlapped))
  966. break;
  967. DWORD err = ::GetLastError();
  968. if (err!=ERROR_LOCK_VIOLATION)
  969. throw makeOsException(err, "CDiscretionaryFileLock::lock");
  970. #else
  971. if (setShareLock(handle,exclusive?IFSHnone:IFSHread))
  972. break;
  973. #endif
  974. unsigned now = msTick();
  975. if (now-start>=timeout)
  976. return false;
  977. if (interval>timeout-now+start)
  978. interval = timeout-now+start;
  979. Sleep(interval);
  980. interval *= 2;
  981. if (interval>1000)
  982. interval = 1000;
  983. }
  984. excllock = exclusive;
  985. locked = true;
  986. return true;
  987. }
  988. void unlock()
  989. {
  990. if (handle!=NULLFILE)
  991. {
  992. #ifdef _WIN32
  993. OVERLAPPED overlapped;
  994. memset(&overlapped,0,sizeof(overlapped));
  995. if (!UnlockFileEx(handle,0,1,0,&overlapped))
  996. throw makeOsException(::GetLastError(), "CDiscretionaryFileLock::unlockhandle");
  997. #else
  998. setShareLock(handle,IFSHfull);
  999. #endif
  1000. }
  1001. }
  1002. };
  1003. IDiscretionaryLock *createDiscretionaryLock(IFile *file)
  1004. {
  1005. return new CDiscretionaryFileLock(file);
  1006. }
  1007. IDiscretionaryLock *createDiscretionaryLock(IFileIO *fileio)
  1008. {
  1009. return new CDiscretionaryFileLock(fileio);
  1010. }
  1011. unsigned CFile::getCRC()
  1012. {
  1013. if (stdIoHandle(filename)>=0)
  1014. return 0; // dummy value
  1015. unsigned crc=~0;
  1016. MemoryAttr ma;
  1017. void *buf = ma.allocate(0x10000);
  1018. Owned<IFileIO> fileio = open(IFOread);
  1019. if (fileio) {
  1020. offset_t pos=0;
  1021. loop {
  1022. size32_t rd = fileio->read(pos,0x10000,buf);
  1023. if (!rd)
  1024. break;
  1025. crc=crc32((const char *)buf,rd,crc);
  1026. pos += rd;
  1027. }
  1028. }
  1029. return ~crc;
  1030. }
  1031. //---------------------------------------------------------------------------
  1032. static Linked<IPasswordProvider> passwordProvider;
  1033. MODULE_INIT(INIT_PRIORITY_JFILE)
  1034. {
  1035. return true;
  1036. }
  1037. MODULE_EXIT()
  1038. {
  1039. passwordProvider.clear();
  1040. }
  1041. static bool parseShare(const char *filename,IpAddress &machine,StringBuffer &share)
  1042. { // windows share parsing
  1043. if (!filename||!isPathSepChar(filename[0])||(filename[0]!=filename[1]))
  1044. return false;
  1045. const char *start = filename+2;
  1046. const char * end = start;
  1047. while (*end && !isPathSepChar(*end))
  1048. end++;
  1049. if (!*end)
  1050. return false;
  1051. StringBuffer ipText(end-start, start);
  1052. machine.ipset(ipText.str());
  1053. end++;
  1054. while (*end && !isPathSepChar(*end))
  1055. end++;
  1056. start = filename;
  1057. while (start!=end) {
  1058. if (*start=='/')
  1059. share.append('\\');
  1060. else
  1061. share.append(*start);
  1062. start++;
  1063. }
  1064. return true;
  1065. }
  1066. static CriticalSection connectcrit;
  1067. static bool connectToExternalDrive(const char * const filename)
  1068. {
  1069. CriticalBlock block(connectcrit);
  1070. if (!passwordProvider)
  1071. return false;
  1072. #ifdef _WIN32
  1073. StringBuffer share, username, password;
  1074. IpAddress ip;
  1075. if (!parseShare(filename,ip,share))
  1076. return false;
  1077. if (!passwordProvider->getPassword(ip, username, password))
  1078. return false;
  1079. // first see if connected
  1080. char buf[255];
  1081. buf[0] = 0;
  1082. DWORD len = sizeof(buf);
  1083. DWORD err = WNetGetUser(share.str(),buf,&len);
  1084. if ((err==0)&&(stricmp(username.str(),buf)==0)) {
  1085. // check can access share as well
  1086. for (unsigned i=0;i<10;i++) {
  1087. DWORD ret = (DWORD)GetFileAttributes(share.str());
  1088. DWORD err = (ret==(DWORD)-1)?GetLastError():0;
  1089. if (err!=ERROR_IO_INCOMPLETE)
  1090. break;
  1091. Sleep(100*i);
  1092. }
  1093. if (err==0)
  1094. return true;
  1095. }
  1096. for (unsigned retry=0;retry<5;retry++) {
  1097. NETRESOURCE res;
  1098. memset(&res, 0, sizeof(res));
  1099. res.dwType = RESOURCETYPE_DISK;
  1100. res.lpRemoteName = (char *)share.str();
  1101. err = WNetAddConnection2(&res, password.str(), username.str(), 0);
  1102. if (err) {
  1103. if (err==ERROR_SESSION_CREDENTIAL_CONFLICT) {
  1104. WNetCancelConnection2(res.lpRemoteName, 0, false);
  1105. err = WNetAddConnection2(&res, password.str(), username.str(), 0);
  1106. }
  1107. if (err)
  1108. ERRLOG("WNetAddConnection2(%d): connecting to %s, User: %s",err,res.lpRemoteName,username.str());
  1109. }
  1110. if (err==0)
  1111. return true;
  1112. Sleep(retry*100);
  1113. }
  1114. #endif
  1115. return false;
  1116. }
  1117. static void disconnectFromExternalDrive(const char * const filename)
  1118. {
  1119. #ifdef _WIN32
  1120. CriticalBlock block(connectcrit);
  1121. StringBuffer share;
  1122. IpAddress ip;
  1123. if (!parseShare(filename,ip,share))
  1124. return;
  1125. if (share.length()&&isShareChar(share.charAt(share.length())))
  1126. WNetCancelConnection2((char *)share.str(), 0, 0);
  1127. #else
  1128. //TODO: fill in for linux
  1129. #endif
  1130. }
  1131. #ifdef _WIN32
  1132. class CWindowsRemoteFile : public CInterface, implements IFile
  1133. {
  1134. IFile *ifile;
  1135. StringAttr filename;
  1136. bool connected;
  1137. inline bool connect()
  1138. {
  1139. if (!connected)
  1140. connected = connectToExternalDrive(filename);
  1141. return connected;
  1142. }
  1143. public:
  1144. CWindowsRemoteFile(const char * _filename)
  1145. : filename(_filename)
  1146. {
  1147. connected = false;
  1148. ifile = new CFile(_filename);
  1149. }
  1150. ~CWindowsRemoteFile()
  1151. {
  1152. ifile->Release();
  1153. #ifdef REMOTE_DISCONNECT_ON_DESTRUCTOR
  1154. if (connected)
  1155. disconnectFromExternalDrive(filename);
  1156. #endif
  1157. }
  1158. IMPLEMENT_IINTERFACE
  1159. virtual bool exists()
  1160. {
  1161. connect();
  1162. return ifile->exists();
  1163. }
  1164. virtual bool getTime(CDateTime * createTime, CDateTime * modifiedTime, CDateTime * accessedTime)
  1165. {
  1166. connect();
  1167. return ifile->getTime(createTime, modifiedTime, accessedTime);
  1168. }
  1169. virtual bool setTime(const CDateTime * createTime, const CDateTime * modifiedTime, const CDateTime * accessedTime)
  1170. {
  1171. connect();
  1172. if (ifile->setTime(createTime, modifiedTime, accessedTime))
  1173. return true;
  1174. if (connected||!connect())
  1175. return false;
  1176. return ifile->setTime(createTime, modifiedTime, accessedTime);
  1177. }
  1178. virtual fileBool isDirectory()
  1179. {
  1180. connect();
  1181. fileBool ok = ifile->isDirectory();
  1182. if (ok == notFound && !connected && connect())
  1183. ok = ifile->isDirectory();
  1184. return ok;
  1185. }
  1186. virtual fileBool isFile()
  1187. {
  1188. connect();
  1189. fileBool ok = ifile->isFile();
  1190. if (ok == notFound && !connected && connect())
  1191. ok = ifile->isFile();
  1192. return ok;
  1193. }
  1194. virtual fileBool isReadOnly()
  1195. {
  1196. connect();
  1197. fileBool ok = ifile->isReadOnly();
  1198. if (ok == notFound && !connected && connect())
  1199. ok = ifile->isFile();
  1200. return ok;
  1201. }
  1202. virtual IFileIO * open(IFOmode mode,IFEflags extraFlags=IFEnone)
  1203. {
  1204. connect();
  1205. return ifile->open(mode,extraFlags);
  1206. }
  1207. virtual IFileIO * openShared(IFOmode mode,IFSHmode shared,IFEflags extraFlags=IFEnone)
  1208. {
  1209. connect();
  1210. return ifile->openShared(mode,shared,extraFlags);
  1211. }
  1212. virtual IFileAsyncIO * openAsync(IFOmode mode)
  1213. {
  1214. connect();
  1215. return ifile->openAsync(mode);
  1216. }
  1217. virtual const char * queryFilename()
  1218. {
  1219. return ifile->queryFilename();
  1220. }
  1221. virtual bool remove()
  1222. {
  1223. connect();
  1224. unsigned attempt=0; \
  1225. return ifile->remove();
  1226. }
  1227. virtual void rename(const char *newname)
  1228. {
  1229. connect();
  1230. StringBuffer path;
  1231. splitDirTail(filename,path);
  1232. StringBuffer newdir;
  1233. const char *tail = splitDirTail(newname,newdir);
  1234. if ((newname[0]=='\\')&&(newname[1]=='\\')) { // rename to remote
  1235. if (path.length()&&newdir.length()) {
  1236. if (strcmp(newdir.str(),path.str())!=0)
  1237. WARNLOG("CWindowsRemoteFile '%s' to '%s' : directory mismatch",filename.get(),newname);
  1238. }
  1239. newname = tail; // just rename using tail
  1240. }
  1241. ifile->rename(newname);
  1242. path.append(tail);
  1243. filename.set(path);
  1244. }
  1245. virtual void move(const char *newName)
  1246. {
  1247. connect();
  1248. ifile->move(newName);
  1249. filename.set(ifile->queryFilename());
  1250. }
  1251. virtual void setReadOnly(bool ro)
  1252. {
  1253. connect();
  1254. ifile->setReadOnly(ro);
  1255. }
  1256. virtual offset_t size()
  1257. {
  1258. connect();
  1259. offset_t ret=ifile->size();
  1260. if ((ret==(offset_t)-1)&&!connected&&connect())
  1261. ret=ifile->size();
  1262. return ret;
  1263. }
  1264. virtual bool setCompression(bool set)
  1265. {
  1266. connect();
  1267. return ifile->setCompression(set);
  1268. }
  1269. virtual offset_t compressedSize()
  1270. {
  1271. connect();
  1272. return ifile->compressedSize();
  1273. }
  1274. bool fastCopyFile(CWindowsRemoteFile &target, size32_t buffersize, ICopyFileProgress *progress)
  1275. {
  1276. #ifdef _WIN32
  1277. CFile *src = QUERYINTERFACE(ifile,CFile);
  1278. if (!src)
  1279. return false;
  1280. CFile *dst = QUERYINTERFACE(target.ifile,CFile);
  1281. if (!dst)
  1282. return false;
  1283. target.isFile(); // encourage connect on target
  1284. connect();
  1285. return src->fastCopyFile(*dst, buffersize, progress);
  1286. #else
  1287. return false;
  1288. #endif
  1289. }
  1290. bool fastCopyFile(CFile &target, size32_t buffersize, ICopyFileProgress *progress)
  1291. {
  1292. #ifdef _WIN32
  1293. CFile *src = QUERYINTERFACE(ifile,CFile);
  1294. if (!src)
  1295. return false;
  1296. connect();
  1297. return src->fastCopyFile(target, buffersize, progress);
  1298. #else
  1299. return false;
  1300. #endif
  1301. }
  1302. bool fastCopyFileRev(CFile &src, size32_t buffersize, ICopyFileProgress *progress)
  1303. {
  1304. #ifdef _WIN32
  1305. CFile *dst = QUERYINTERFACE(ifile,CFile);
  1306. if (!dst)
  1307. return false;
  1308. connect();
  1309. return src.fastCopyFile(*dst, buffersize, progress);
  1310. #else
  1311. return false;
  1312. #endif
  1313. }
  1314. void copyTo(IFile *dest, size32_t buffersize, ICopyFileProgress *progress, bool usetmp,CFflags copyFlags)
  1315. {
  1316. doCopyFile(dest,this,buffersize,progress,NULL,usetmp,copyFlags);
  1317. }
  1318. bool createDirectory()
  1319. {
  1320. connect();
  1321. return localCreateDirectory(filename);
  1322. }
  1323. IDirectoryIterator *directoryFiles(const char *mask,bool sub,bool includedirs)
  1324. {
  1325. connect();
  1326. return ifile->directoryFiles(mask,sub,includedirs);
  1327. }
  1328. IDirectoryDifferenceIterator *monitorDirectory(
  1329. IDirectoryIterator *prev=NULL, // in (NULL means use current as baseline)
  1330. const char *mask=NULL,
  1331. bool sub=false,
  1332. bool includedirs=false,
  1333. unsigned checkinterval=60*1000,
  1334. unsigned timeout=(unsigned)-1,
  1335. Semaphore *abortsem=NULL) // returns NULL if timed out or abortsem signalled
  1336. {
  1337. connect();
  1338. return ifile->monitorDirectory(prev,mask,sub,includedirs,checkinterval,timeout,abortsem);
  1339. }
  1340. bool getInfo(bool &isdir,offset_t &size,CDateTime &modtime)
  1341. {
  1342. connect();
  1343. return ifile->getInfo(isdir,size,modtime);
  1344. }
  1345. unsigned getCRC()
  1346. {
  1347. connect();
  1348. return ifile->getCRC();
  1349. }
  1350. void setCreateFlags(unsigned cflags)
  1351. {
  1352. ifile->setCreateFlags(cflags);
  1353. }
  1354. void setShareMode(IFSHmode shmode)
  1355. {
  1356. ifile->setShareMode(shmode);
  1357. }
  1358. void copySection(const RemoteFilename &dest, offset_t toOfs, offset_t fromOfs, offset_t size, ICopyFileProgress *progress, CFflags copyFlags)
  1359. {
  1360. connect();
  1361. ifile->copySection(dest,toOfs,fromOfs,size,progress,copyFlags);
  1362. }
  1363. IMemoryMappedFile *openMemoryMapped(offset_t ofs, memsize_t len, bool write)
  1364. {
  1365. throw MakeStringException(-1,"Remote file cannot be memory mapped");
  1366. return NULL;
  1367. }
  1368. void treeCopyTo(IFile *dest,IpSubNet &subnet,IpAddress &resfrom, bool usetmp, CFflags copyFlags)
  1369. {
  1370. // no special action for windows
  1371. GetHostIp(resfrom);
  1372. copyTo(dest,DEFAULT_COPY_BLKSIZE,NULL,usetmp,copyFlags);
  1373. }
  1374. };
  1375. #endif
  1376. IFileIO *_createIFileIO(const void *buffer, unsigned sz, bool readOnly)
  1377. {
  1378. class CMemoryBufferIO : public CInterface, implements IFileIO
  1379. {
  1380. MemoryBuffer mb;
  1381. void *buffer;
  1382. size32_t sz;
  1383. bool readOnly;
  1384. public:
  1385. IMPLEMENT_IINTERFACE;
  1386. CMemoryBufferIO(const void *_buffer, unsigned _sz, bool _readOnly) : readOnly(_readOnly)
  1387. {
  1388. // JCSMORE - should probably have copy as option
  1389. mb.append(_sz, _buffer);
  1390. buffer = (void *)mb.toByteArray();
  1391. sz = mb.length();
  1392. }
  1393. virtual size32_t read(offset_t pos, size32_t len, void * data)
  1394. {
  1395. if (pos>sz)
  1396. throw MakeStringException(-1, "CMemoryBufferIO: read beyond end of buffer pos=%" I64F "d, len=%d, buffer length=%d", pos, len, mb.length());
  1397. if (pos+len > sz)
  1398. len = (size32_t)(sz-pos);
  1399. memcpy(data, (byte *)buffer+pos, len);
  1400. return len;
  1401. }
  1402. virtual offset_t size() { return sz; }
  1403. virtual size32_t write(offset_t pos, size32_t len, const void * data)
  1404. {
  1405. assertex(!readOnly);
  1406. if (pos+len>sz)
  1407. throw MakeStringException(-1, "CMemoryBufferIO: UNIMPLEMENTED, writing beyond buffer, pos=%" I64F "d, len=%d, buffer length=%d", pos, len, mb.length());
  1408. memcpy((byte *)buffer+pos, data, len);
  1409. return len;
  1410. }
  1411. virtual void flush() {}
  1412. virtual void close() {}
  1413. virtual void setSize(offset_t size)
  1414. {
  1415. if (size > mb.length())
  1416. throw MakeStringException(-1, "CMemoryBufferIO: UNIMPLEMENTED, setting size %" I64F "d beyond end of buffer, buffer length=%d", size, mb.length());
  1417. mb.setLength((size32_t)size);
  1418. }
  1419. offset_t appendFile(IFile *file,offset_t pos,offset_t len)
  1420. {
  1421. if (!file)
  1422. return 0;
  1423. const size32_t buffsize = 0x10000;
  1424. void * buffer = mb.reserve(buffsize);
  1425. Owned<IFileIO> fileio = file->open(IFOread);
  1426. offset_t ret=0;
  1427. while (len) {
  1428. size32_t toread = (len>=buffsize)?buffsize:(size32_t)len;
  1429. size32_t read = fileio->read(pos,toread,buffer);
  1430. if (read<buffsize)
  1431. mb.setLength(mb.length()+read-buffsize);
  1432. if (read==0)
  1433. break;
  1434. pos += read;
  1435. len -= read;
  1436. ret += read;
  1437. }
  1438. return ret;
  1439. }
  1440. };
  1441. return new CMemoryBufferIO(buffer, sz, readOnly);
  1442. }
  1443. IFileIO * createIFileI(unsigned len, const void * buffer)
  1444. {
  1445. return _createIFileIO((void *)buffer, len, true);
  1446. }
  1447. IFileIO * createIFileIO(unsigned len, void * buffer)
  1448. {
  1449. return _createIFileIO(buffer, len, false);
  1450. }
  1451. IFileIO * createIFileIO(StringBuffer & buffer)
  1452. {
  1453. return _createIFileIO((void *)buffer.str(), buffer.length(), true);
  1454. }
  1455. IFileIO * createIFileIO(MemoryBuffer & buffer)
  1456. {
  1457. return _createIFileIO(buffer.toByteArray(), buffer.length(), false);
  1458. }
  1459. //---------------------------------------------------------------------------
  1460. class jlib_decl CSequentialFileIO : public CFileIO
  1461. {
  1462. offset_t pos;
  1463. void checkPos(const char *fn,offset_t _pos)
  1464. {
  1465. if (_pos!=pos)
  1466. throw MakeStringException(-1, "CSequentialFileIO %s out of sequence (%" I64F "d,%" I64F "d)",fn,pos,_pos);
  1467. }
  1468. public:
  1469. CSequentialFileIO(HANDLE h,IFOmode _openmode,IFSHmode _sharemode,IFEflags _extraFlags)
  1470. : CFileIO(h,_openmode,_sharemode,_extraFlags)
  1471. {
  1472. pos = 0;
  1473. }
  1474. ~CSequentialFileIO()
  1475. {
  1476. }
  1477. IMPLEMENT_IINTERFACE
  1478. size32_t read(offset_t _pos, size32_t len, void * data)
  1479. {
  1480. checkPos("read",_pos);
  1481. #ifdef _WIN32
  1482. // Can't use checked_read because don't have the c fileno for it
  1483. DWORD numRead;
  1484. if (ReadFile(file,data,len,&numRead,NULL) == 0) {
  1485. DWORD err = GetLastError();
  1486. if (err==ERROR_BROKEN_PIPE) // windows returns this at end of pipe
  1487. return 0;
  1488. throw makeOsException(GetLastError(),"CSequentialFileIO::read");
  1489. }
  1490. size32_t ret = (size32_t)numRead;
  1491. #else
  1492. size32_t ret = checked_read(file, data, len);
  1493. #endif
  1494. pos += ret;
  1495. return ret;
  1496. }
  1497. virtual size32_t write(offset_t _pos, size32_t len, const void * data)
  1498. {
  1499. checkPos("write",_pos);
  1500. size32_t ret;
  1501. #ifdef _WIN32
  1502. DWORD numWritten;
  1503. if (!WriteFile(file, data, len, &numWritten, NULL))
  1504. throw makeOsException(GetLastError(), "CSequentialFileIO::write");
  1505. if (numWritten != len)
  1506. throw makeOsException(DISK_FULL_EXCEPTION_CODE, "CSequentialFileIO::write");
  1507. ret = (size32_t) numWritten;
  1508. #else
  1509. ret = ::write(file,data,len);
  1510. if (ret==(size32_t)-1)
  1511. {
  1512. PrintStackReport();
  1513. ERRLOG("errno(%d): %" I64F "d %u",errno,pos,len);
  1514. throw makeErrnoException(errno, "CFileIO::write");
  1515. }
  1516. if (ret<len)
  1517. throw makeOsException(DISK_FULL_EXCEPTION_CODE, "CSequentialFileIO::write");
  1518. #endif
  1519. pos += ret;
  1520. return ret;
  1521. }
  1522. };
  1523. IFileIO * CFile::openShared(IFOmode mode,IFSHmode share,IFEflags extraFlags)
  1524. {
  1525. int stdh = stdIoHandle(filename);
  1526. HANDLE handle = openHandle(mode,share,false, stdh);
  1527. if (handle==NULLFILE)
  1528. return NULL;
  1529. #ifndef _WIN32
  1530. set_inherit(handle, false);
  1531. #endif
  1532. if (stdh>=0)
  1533. return new CSequentialFileIO(handle,mode,share,extraFlags);
  1534. return new CFileIO(handle,mode,share,extraFlags);
  1535. }
  1536. //---------------------------------------------------------------------------
  1537. extern jlib_decl IFileIO *createIFileIO(HANDLE handle,IFOmode openmode,IFEflags extraFlags)
  1538. {
  1539. return new CFileIO(handle,openmode,IFSHfull,extraFlags);
  1540. }
  1541. offset_t CFileIO::appendFile(IFile *file,offset_t pos,offset_t len)
  1542. {
  1543. if (!file)
  1544. return 0;
  1545. CriticalBlock procedure(cs);
  1546. MemoryAttr mb;
  1547. const size32_t buffsize = 0x10000;
  1548. void * buffer = mb.allocate(buffsize);
  1549. Owned<IFileIO> fileio = file->open(IFOread);
  1550. offset_t ret=0;
  1551. offset_t outp = size();
  1552. while (len) {
  1553. size32_t toread = (len>=buffsize) ? buffsize : (size32_t)len;
  1554. size32_t read = fileio->read(pos,toread,buffer);
  1555. if (read==0)
  1556. break;
  1557. size32_t wr = write(outp,read,buffer);
  1558. pos += read;
  1559. outp += wr;
  1560. len -= read;
  1561. ret += wr;
  1562. if (wr!=read)
  1563. break;
  1564. }
  1565. return ret;
  1566. }
  1567. #ifdef _WIN32
  1568. //-- Windows implementation -------------------------------------------------
  1569. CFileIO::CFileIO(HANDLE handle, IFOmode _openmode, IFSHmode _sharemode, IFEflags _extraFlags)
  1570. {
  1571. assertex(handle != NULLFILE);
  1572. throwOnError = false;
  1573. file = handle;
  1574. sharemode = _sharemode;
  1575. openmode = _openmode;
  1576. extraFlags = _extraFlags; // page cache flush option silently ignored on Windows for now
  1577. if (extraFlags & IFEnocache)
  1578. if (!isPCFlushAllowed())
  1579. extraFlags = static_cast<IFEflags>(extraFlags & ~IFEnocache);
  1580. atomic_set(&bytesRead, 0);
  1581. atomic_set(&bytesWritten, 0);
  1582. }
  1583. CFileIO::~CFileIO()
  1584. {
  1585. try
  1586. {
  1587. //note this will not call the virtual close() if anyone ever derived from this class.
  1588. //the clean fix is to move this code to beforeDispose()
  1589. close();
  1590. }
  1591. catch (IException * e)
  1592. {
  1593. EXCLOG(e, "CFileIO::~CFileIO");
  1594. e->Release();
  1595. }
  1596. }
  1597. void CFileIO::close()
  1598. {
  1599. if (file != NULLFILE)
  1600. {
  1601. if (!CloseHandle(file))
  1602. throw makeOsException(GetLastError(),"CFileIO::close");
  1603. }
  1604. file = NULLFILE;
  1605. }
  1606. void CFileIO::flush()
  1607. {
  1608. if (!FlushFileBuffers(file))
  1609. throw makeOsException(GetLastError(),"CFileIO::flush");
  1610. }
  1611. offset_t CFileIO::size()
  1612. {
  1613. LARGE_INTEGER pos;
  1614. pos.LowPart = GetFileSize(file, (unsigned long *)&pos.HighPart);
  1615. if (pos.LowPart==-1) {
  1616. DWORD err = GetLastError();
  1617. if (err!=0)
  1618. throw makeOsException(err,"CFileIO::size");
  1619. }
  1620. return pos.QuadPart;
  1621. }
  1622. size32_t CFileIO::read(offset_t pos, size32_t len, void * data)
  1623. {
  1624. CriticalBlock procedure(cs);
  1625. DWORD numRead;
  1626. setPos(pos);
  1627. if (ReadFile(file,data,len,&numRead,NULL) == 0)
  1628. throw makeOsException(GetLastError(),"CFileIO::read");
  1629. return (size32_t)numRead;
  1630. }
  1631. void CFileIO::setPos(offset_t newPos)
  1632. {
  1633. LARGE_INTEGER tempPos;
  1634. tempPos.QuadPart = newPos;
  1635. tempPos.LowPart = SetFilePointer(file, tempPos.LowPart, &tempPos.HighPart, FILE_BEGIN);
  1636. }
  1637. size32_t CFileIO::write(offset_t pos, size32_t len, const void * data)
  1638. {
  1639. CriticalBlock procedure(cs);
  1640. DWORD numWritten;
  1641. setPos(pos);
  1642. if (!WriteFile(file,data,len,&numWritten,NULL))
  1643. throw makeOsException(GetLastError(),"CFileIO::write");
  1644. if (numWritten != len)
  1645. throw makeOsException(DISK_FULL_EXCEPTION_CODE,"CFileIO::write");
  1646. return (size32_t)numWritten;
  1647. }
  1648. void CFileIO::setSize(offset_t pos)
  1649. {
  1650. CriticalBlock procedure(cs);
  1651. setPos(pos);
  1652. if (!SetEndOfFile(file))
  1653. throw makeOsException(GetLastError(), "CFileIO::setSize");
  1654. }
  1655. #else
  1656. //-- Unix implementation ----------------------------------------------------
  1657. // More errorno checking TBD
  1658. CFileIO::CFileIO(HANDLE handle, IFOmode _openmode, IFSHmode _sharemode, IFEflags _extraFlags)
  1659. {
  1660. assertex(handle != NULLFILE);
  1661. throwOnError = false;
  1662. file = handle;
  1663. sharemode = _sharemode;
  1664. openmode = _openmode;
  1665. extraFlags = _extraFlags;
  1666. if (extraFlags & IFEnocache)
  1667. if (!isPCFlushAllowed())
  1668. extraFlags = static_cast<IFEflags>(extraFlags & ~IFEnocache);
  1669. atomic_set(&bytesRead, 0);
  1670. atomic_set(&bytesWritten, 0);
  1671. #ifdef CFILEIOTRACE
  1672. DBGLOG("CFileIO::CfileIO(%d,%d,%d,%d)", handle, _openmode, _sharemode, _extraFlags);
  1673. #endif
  1674. }
  1675. CFileIO::~CFileIO()
  1676. {
  1677. try
  1678. {
  1679. close();
  1680. }
  1681. catch (IException * e)
  1682. {
  1683. EXCLOG(e, "CFileIO::~CFileIO");
  1684. e->Release();
  1685. }
  1686. }
  1687. void CFileIO::close()
  1688. {
  1689. if (file != NULLFILE)
  1690. {
  1691. #ifdef CFILEIOTRACE
  1692. DBGLOG("CFileIO::close(%d), extraFlags = %d", file, extraFlags);
  1693. #endif
  1694. if (extraFlags & IFEnocache)
  1695. {
  1696. if (openmode != IFOread)
  1697. {
  1698. #ifdef F_FULLFSYNC
  1699. fcntl(file, F_FULLFSYNC);
  1700. #else
  1701. fdatasync(file);
  1702. #endif
  1703. }
  1704. #ifdef POSIX_FADV_DONTNEED
  1705. posix_fadvise(file, 0, 0, POSIX_FADV_DONTNEED);
  1706. #endif
  1707. }
  1708. if (::close(file) < 0)
  1709. throw makeErrnoException(errno, "CFileIO::close");
  1710. file=NULLFILE;
  1711. }
  1712. }
  1713. void CFileIO::flush()
  1714. {
  1715. CriticalBlock procedure(cs);
  1716. #ifdef F_FULLFSYNC
  1717. if (fcntl(file, F_FULLFSYNC) != 0)
  1718. #else
  1719. if (fdatasync(file) != 0)
  1720. #endif
  1721. throw makeOsException(DISK_FULL_EXCEPTION_CODE, "CFileIO::flush");
  1722. #ifdef POSIX_FADV_DONTNEED
  1723. if (extraFlags & IFEnocache)
  1724. posix_fadvise(file, 0, 0, POSIX_FADV_DONTNEED);
  1725. #endif
  1726. }
  1727. offset_t CFileIO::size()
  1728. {
  1729. CriticalBlock procedure(cs);
  1730. offset_t savedPos = lseek(file,0,SEEK_CUR);
  1731. offset_t length = lseek(file,0,SEEK_END);
  1732. setPos(savedPos);
  1733. return length;
  1734. }
  1735. size32_t CFileIO::read(offset_t pos, size32_t len, void * data)
  1736. {
  1737. if (0==len) return 0;
  1738. size32_t ret = checked_pread(file, data, len, pos);
  1739. if ( (extraFlags & IFEnocache) && (ret > 0) )
  1740. {
  1741. if (atomic_add_and_read(&bytesRead, ret) >= PGCFLUSH_BLKSIZE)
  1742. {
  1743. atomic_set(&bytesRead, 0);
  1744. #ifdef POSIX_FADV_DONTNEED
  1745. posix_fadvise(file, 0, 0, POSIX_FADV_DONTNEED);
  1746. #endif
  1747. }
  1748. }
  1749. return ret;
  1750. }
  1751. void CFileIO::setPos(offset_t newPos)
  1752. {
  1753. if (file != NULLFILE)
  1754. _llseek(file,newPos,SEEK_SET);
  1755. }
  1756. static void sync_file_region(int fd, offset_t offset, offset_t nbytes)
  1757. {
  1758. #if defined(__linux__) && defined(__NR_sync_file_range)
  1759. (void)syscall(__NR_sync_file_range, fd, offset,
  1760. nbytes, SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WRITE);
  1761. #elif defined(F_FULLFSYNC)
  1762. fcntl(fd, F_FULLFSYNC);
  1763. #else
  1764. fdatasync(fd);
  1765. #endif
  1766. }
  1767. size32_t CFileIO::write(offset_t pos, size32_t len, const void * data)
  1768. {
  1769. size32_t ret = pwrite(file,data,len,pos);
  1770. if (ret==(size32_t)-1)
  1771. throw makeErrnoException(errno, "CFileIO::write");
  1772. if (ret<len)
  1773. throw makeOsException(DISK_FULL_EXCEPTION_CODE, "CFileIO::write");
  1774. if ( (extraFlags & IFEnocache) && (ret > 0) )
  1775. {
  1776. if (atomic_add_and_read(&bytesWritten, ret) >= PGCFLUSH_BLKSIZE)
  1777. {
  1778. atomic_set(&bytesWritten, 0);
  1779. // [possibly] non-blocking request to write-out dirty pages
  1780. sync_file_region(file, 0, 0);
  1781. #ifdef POSIX_FADV_DONTNEED
  1782. // flush written-out pages
  1783. posix_fadvise(file, 0, 0, POSIX_FADV_DONTNEED);
  1784. #endif
  1785. }
  1786. }
  1787. return ret;
  1788. }
  1789. void CFileIO::setSize(offset_t pos)
  1790. {
  1791. if (0 != ftruncate(file, pos))
  1792. throw makeErrnoException(errno, "CFileIO::setSize");
  1793. }
  1794. #endif
  1795. //---------------------------------------------------------------------------
  1796. CFileRangeIO::CFileRangeIO(IFileIO * _io, offset_t _headerSize, offset_t _maxLength)
  1797. {
  1798. io.set(_io);
  1799. headerSize = _headerSize;
  1800. maxLength = _maxLength;
  1801. }
  1802. size32_t CFileRangeIO::read(offset_t pos, size32_t len, void * data)
  1803. {
  1804. if (pos + len > maxLength)
  1805. {
  1806. if (pos > maxLength)
  1807. pos = maxLength;
  1808. len = (size32_t)(maxLength - pos);
  1809. }
  1810. return io->read(pos+headerSize, len, data);
  1811. }
  1812. offset_t CFileRangeIO::size()
  1813. {
  1814. return maxLength;
  1815. }
  1816. size32_t CFileRangeIO::write(offset_t pos, size32_t len, const void * data)
  1817. {
  1818. if (pos + len > maxLength)
  1819. {
  1820. if (pos > maxLength)
  1821. pos = maxLength;
  1822. len = (size32_t)(maxLength - pos);
  1823. }
  1824. return io->write(pos+headerSize, len, data);
  1825. }
  1826. //--------------------------------------------------------------------------
  1827. CFileAsyncIO::~CFileAsyncIO()
  1828. {
  1829. try
  1830. {
  1831. close();
  1832. }
  1833. catch (IException * e)
  1834. {
  1835. EXCLOG(e, "CFileAsyncIO::~CFileAsyncIO");
  1836. e->Release();
  1837. }
  1838. }
  1839. void CFileAsyncIO::flush()
  1840. {
  1841. // wait for all outstanding results
  1842. CriticalBlock block(cs);
  1843. ForEachItemInRev(i,results) {
  1844. size32_t dummy;
  1845. results.item(i).getResult(dummy,true);
  1846. }
  1847. }
  1848. offset_t CFileAsyncIO::appendFile(IFile *file,offset_t pos,offset_t len)
  1849. {
  1850. // will implemented if needed
  1851. UNIMPLEMENTED;
  1852. }
  1853. #ifdef _WIN32
  1854. //-- Windows implementation -------------------------------------------------
  1855. class CFileAsyncResult: public CInterface, implements IFileAsyncResult
  1856. {
  1857. protected: friend class CFileAsyncIO;
  1858. OVERLAPPED overlapped;
  1859. DWORD value;
  1860. DWORD wrsize;
  1861. CFileAsyncIO *parent;
  1862. public:
  1863. IMPLEMENT_IINTERFACE;
  1864. CFileAsyncResult(offset_t offset,size32_t _wrsize)
  1865. {
  1866. parent = NULL;
  1867. memset(&overlapped,0,sizeof(overlapped));
  1868. overlapped.hEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
  1869. overlapped.Offset = (DWORD)offset;
  1870. overlapped.OffsetHigh = (DWORD)(offset>>32);
  1871. value = (size32_t)-1;
  1872. wrsize = _wrsize;
  1873. }
  1874. ~CFileAsyncResult()
  1875. {
  1876. size32_t dummy;
  1877. getResult(dummy,true);
  1878. CloseHandle(overlapped.hEvent);
  1879. }
  1880. bool getResult(size32_t &ret,bool wait)
  1881. {
  1882. if (value==(size32_t)-1) {
  1883. if (parent) {
  1884. if (GetOverlappedResult(parent->file,&overlapped,&value,wait)==0) {
  1885. int err = GetLastError();
  1886. if (err==ERROR_IO_INCOMPLETE)
  1887. return false;
  1888. if (err!=ERROR_HANDLE_EOF) {
  1889. CriticalBlock block(parent->cs);
  1890. parent->results.zap(*this,true); // don't delete as array does not own
  1891. parent = NULL;
  1892. throw makeOsException(err,"CFileAsyncResult::getResult");
  1893. }
  1894. value = 0;
  1895. }
  1896. CriticalBlock block(parent->cs);
  1897. parent->results.zap(*this,true); // don't delete as array does not own
  1898. parent = NULL;
  1899. }
  1900. else
  1901. return false;
  1902. }
  1903. ret = value;
  1904. if (value<wrsize)
  1905. throw makeOsException(DISK_FULL_EXCEPTION_CODE,"CFileAsyncResult::getResult");
  1906. return true;
  1907. }
  1908. };
  1909. CFileAsyncIO::CFileAsyncIO(HANDLE handle, IFSHmode _sharemode)
  1910. {
  1911. assertex(handle != NULLFILE);
  1912. throwOnError = false;
  1913. file = handle;
  1914. sharemode = _sharemode;
  1915. }
  1916. void CFileAsyncIO::close()
  1917. {
  1918. flush();
  1919. // wait for all outstanding results
  1920. if (file != NULLFILE)
  1921. {
  1922. if (!CloseHandle(file))
  1923. throw makeOsException(GetLastError(),"CFileAsyncIO::close");
  1924. }
  1925. file = NULLFILE;
  1926. }
  1927. offset_t CFileAsyncIO::size()
  1928. {
  1929. LARGE_INTEGER pos;
  1930. pos.LowPart = GetFileSize(file, (unsigned long *)&pos.HighPart);
  1931. if (pos.LowPart==-1) {
  1932. DWORD err = GetLastError();
  1933. if (err!=0)
  1934. throw makeOsException(GetLastError(),"CFileAsyncIO::size");
  1935. }
  1936. return pos.QuadPart;
  1937. }
  1938. size32_t CFileAsyncIO::read(offset_t pos, size32_t len, void * data)
  1939. {
  1940. Owned<IFileAsyncResult> res = readAsync(pos,len,data);
  1941. size32_t ret;
  1942. res->getResult(ret,true);
  1943. return ret;
  1944. }
  1945. size32_t CFileAsyncIO::write(offset_t pos, size32_t len, const void * data)
  1946. {
  1947. Owned<IFileAsyncResult> res = writeAsync(pos,len,data);
  1948. size32_t ret;
  1949. res->getResult(ret,true);
  1950. return ret;
  1951. }
  1952. void CFileAsyncIO::setSize(offset_t size)
  1953. {
  1954. LARGE_INTEGER tempPos;
  1955. tempPos.QuadPart = size;
  1956. tempPos.LowPart = SetFilePointer(file, tempPos.LowPart, &tempPos.HighPart, FILE_BEGIN);
  1957. if (!SetEndOfFile(file))
  1958. throw makeOsException(GetLastError(), "CFileIO::setSize");
  1959. }
  1960. IFileAsyncResult *CFileAsyncIO::readAsync(offset_t pos, size32_t len, void * data)
  1961. {
  1962. CFileAsyncResult *res = new CFileAsyncResult(pos,0);
  1963. DWORD val;
  1964. if (ReadFile(file,data,len,&val,&res->overlapped) == 0) {
  1965. int err = GetLastError();
  1966. if (err == ERROR_HANDLE_EOF) { // bit weird
  1967. res->value = 0; // won't need to wait
  1968. }
  1969. else if (err == ERROR_IO_PENDING) {
  1970. CriticalBlock block(cs);
  1971. res->parent = this;
  1972. results.append(*res);
  1973. }
  1974. else
  1975. throw makeOsException(GetLastError(),"CFileIO::readAsync");
  1976. }
  1977. else {
  1978. res->value = val; // won't need to wait
  1979. }
  1980. return res;
  1981. }
  1982. IFileAsyncResult *CFileAsyncIO::writeAsync(offset_t pos, size32_t len, const void * data)
  1983. {
  1984. CFileAsyncResult *res = new CFileAsyncResult(pos,len);
  1985. DWORD val;
  1986. if (WriteFile(file,data,len,&val,&res->overlapped) == 0) {
  1987. int err = GetLastError();
  1988. if (err != ERROR_IO_PENDING)
  1989. throw makeOsException(GetLastError(),"CFileIO::writeAsync");
  1990. CriticalBlock block(cs);
  1991. res->parent = this;
  1992. results.append(*res);
  1993. }
  1994. else {
  1995. res->value = val; // wont need to wait
  1996. }
  1997. return res;
  1998. }
  1999. #else
  2000. //-- Unix implementation ----------------------------------------------------
  2001. class CFileAsyncResult: public CInterface, implements IFileAsyncResult
  2002. {
  2003. protected:
  2004. friend class CFileAsyncIO;
  2005. DWORD value;
  2006. DWORD wrsize;
  2007. aiocb cb;
  2008. public:
  2009. IMPLEMENT_IINTERFACE;
  2010. CFileAsyncResult(offset_t offset,size32_t _wrsize)
  2011. {
  2012. value = (size32_t)-1;
  2013. wrsize = _wrsize;
  2014. }
  2015. ~CFileAsyncResult()
  2016. {
  2017. size32_t dummy;
  2018. getResult(dummy,true);
  2019. }
  2020. bool getResult(size32_t &ret,bool wait)
  2021. {
  2022. if (value==(size32_t)-1) {
  2023. loop {
  2024. int aio_errno = aio_error(&cb);
  2025. if (aio_errno==ECANCELED)
  2026. return false;
  2027. if (aio_errno != EINPROGRESS)
  2028. {
  2029. if (aio_errno)
  2030. throw makeErrnoException(aio_errno, "CFileAsyncResult::getResult");
  2031. value = aio_return(&cb);
  2032. if (value<wrsize)
  2033. throw makeOsException(DISK_FULL_EXCEPTION_CODE, "CFileAsyncResult::getResult");
  2034. break;
  2035. }
  2036. if (!wait)
  2037. return false;
  2038. loop {
  2039. struct timespec timeout;
  2040. timeout.tv_sec = 60*60*24; // a long time
  2041. timeout.tv_nsec = 0;
  2042. aiocb * cb_list[1];
  2043. cb_list[0] = &cb;
  2044. int rc = aio_suspend(cb_list, 1, &timeout);
  2045. if ((rc == 0)||(errno != EAGAIN))
  2046. break;
  2047. if (rc==ECANCELED)
  2048. return false;
  2049. }
  2050. }
  2051. }
  2052. ret = value;
  2053. return true;
  2054. }
  2055. };
  2056. CFileAsyncIO::CFileAsyncIO(HANDLE handle, IFSHmode _sharemode)
  2057. {
  2058. assertex(handle != NULLFILE);
  2059. throwOnError = false;
  2060. file = handle;
  2061. sharemode = _sharemode;
  2062. }
  2063. void CFileAsyncIO::close()
  2064. {
  2065. if (file != NULLFILE)
  2066. {
  2067. aio_cancel(file,NULL);
  2068. if (_lclose(file) < 0)
  2069. throw makeErrnoException(errno, "CFileAsyncIO::close");
  2070. }
  2071. file=NULLFILE;
  2072. }
  2073. offset_t CFileAsyncIO::size()
  2074. {
  2075. CriticalBlock procedure(cs);
  2076. offset_t savedPos = _llseek(file,0,SEEK_CUR);
  2077. offset_t length = _llseek(file,0,SEEK_END);
  2078. _llseek(file, savedPos, SEEK_SET);
  2079. return length;
  2080. }
  2081. size32_t CFileAsyncIO::read(offset_t pos, size32_t len, void * data)
  2082. {
  2083. CriticalBlock procedure(cs);
  2084. _llseek(file,pos,SEEK_SET);
  2085. size32_t ret = _lread(file,data,len);
  2086. if (ret==(size32_t)-1)
  2087. throw makeErrnoException(errno, "CFileAsyncIO::read");
  2088. return ret;
  2089. }
  2090. size32_t CFileAsyncIO::write(offset_t pos, size32_t len, const void * data)
  2091. {
  2092. CriticalBlock procedure(cs);
  2093. _llseek(file,pos,SEEK_SET);
  2094. size32_t ret = _lwrite(file,data,len);
  2095. if (ret==(size32_t)-1)
  2096. throw makeErrnoException(errno, "CFileAsyncIO::write");
  2097. return ret;
  2098. }
  2099. void CFileAsyncIO::setSize(offset_t pos)
  2100. {
  2101. CriticalBlock procedure(cs);
  2102. if ((file != NULLFILE)&&(0 != ftruncate(file, pos)))
  2103. throw makeErrnoException(errno, "CFileIO::setSize");
  2104. }
  2105. IFileAsyncResult *CFileAsyncIO::readAsync(offset_t pos, size32_t len, void * data)
  2106. {
  2107. CFileAsyncResult *res = new CFileAsyncResult(pos,0);
  2108. bzero( &(res->cb), sizeof (struct aiocb));
  2109. res->cb.aio_fildes = file;
  2110. res->cb.aio_offset = pos;
  2111. res->cb.aio_buf = data;
  2112. res->cb.aio_nbytes = len;
  2113. res->cb.aio_sigevent.sigev_notify = SIGEV_NONE;
  2114. int retval = aio_read(&(res->cb));
  2115. if (retval==-1)
  2116. throw makeErrnoException(errno, "CFileAsyncIO::readAsync");
  2117. return res;
  2118. }
  2119. IFileAsyncResult *CFileAsyncIO::writeAsync(offset_t pos, size32_t len, const void * data)
  2120. {
  2121. CFileAsyncResult *res = new CFileAsyncResult(pos,len);
  2122. bzero( &(res->cb), sizeof (struct aiocb));
  2123. res->cb.aio_fildes = file;
  2124. res->cb.aio_offset = pos;
  2125. res->cb.aio_buf = (void*)data;
  2126. res->cb.aio_nbytes = len;
  2127. res->cb.aio_sigevent.sigev_signo = SIGUSR1;
  2128. res->cb.aio_sigevent.sigev_notify = SIGEV_NONE;
  2129. res->cb.aio_sigevent.sigev_value.sival_ptr = (void*)res;
  2130. int retval = aio_write(&(res->cb));
  2131. if (retval==-1)
  2132. throw makeErrnoException(errno, "CFileAsyncIO::writeAsync");
  2133. return res;
  2134. }
  2135. #endif
  2136. //---------------------------------------------------------------------------
  2137. CFileIOStream::CFileIOStream(IFileIO * _io)
  2138. {
  2139. io.set(_io);
  2140. curOffset = 0;
  2141. }
  2142. void CFileIOStream::flush()
  2143. {
  2144. }
  2145. size32_t CFileIOStream::read(size32_t len, void * data)
  2146. {
  2147. size32_t numRead = io->read(curOffset, len, data);
  2148. curOffset += numRead;
  2149. return numRead;
  2150. }
  2151. void CFileIOStream::seek(offset_t pos, IFSmode origin)
  2152. {
  2153. switch (origin)
  2154. {
  2155. case IFScurrent:
  2156. curOffset += pos;
  2157. break;
  2158. case IFSend:
  2159. curOffset = io->size() + pos;
  2160. break;
  2161. case IFSbegin:
  2162. curOffset = pos;
  2163. break;
  2164. }
  2165. }
  2166. offset_t CFileIOStream::size()
  2167. {
  2168. return io->size();
  2169. }
  2170. offset_t CFileIOStream::tell()
  2171. {
  2172. return curOffset;
  2173. }
  2174. size32_t CFileIOStream::write(size32_t len, const void * data)
  2175. {
  2176. size32_t numWritten = io->write(curOffset, len, data);
  2177. curOffset += numWritten;
  2178. return numWritten;
  2179. }
  2180. //---------------------------------------------------------------------------
  2181. class CBufferedFileIOStreamBase : public CBufferedIOStreamBase, implements IFileIOStream
  2182. {
  2183. protected:
  2184. virtual offset_t directSize() = 0;
  2185. offset_t curOffset;
  2186. public:
  2187. IMPLEMENT_IINTERFACE;
  2188. CBufferedFileIOStreamBase(unsigned bufSize) : CBufferedIOStreamBase(bufSize), curOffset(0) { }
  2189. virtual void flush() { doflush(); }
  2190. void seek(offset_t pos, IFSmode origin)
  2191. {
  2192. offset_t newOffset = 0;
  2193. switch (origin)
  2194. {
  2195. case IFScurrent:
  2196. newOffset = tell() + pos;
  2197. break;
  2198. case IFSend:
  2199. newOffset = size() + pos;
  2200. break;
  2201. case IFSbegin:
  2202. newOffset = pos;
  2203. break;
  2204. default:
  2205. throwUnexpected();
  2206. }
  2207. if (reading)
  2208. {
  2209. // slightly weird but curoffset is end of buffer when reading
  2210. if ((newOffset >= curOffset-numInBuffer) && (newOffset <= curOffset))
  2211. {
  2212. curBufferOffset = (size32_t)(newOffset - (curOffset-numInBuffer));
  2213. return;
  2214. }
  2215. }
  2216. else
  2217. {
  2218. if ((newOffset >= curOffset) && (newOffset <= curOffset + numInBuffer))
  2219. {
  2220. curBufferOffset = (size32_t)(newOffset - curOffset);
  2221. return;
  2222. }
  2223. flush();
  2224. }
  2225. curOffset = newOffset;
  2226. numInBuffer = 0;
  2227. curBufferOffset = 0;
  2228. }
  2229. offset_t size()
  2230. {
  2231. offset_t curSize = directSize();
  2232. if (!reading)
  2233. curSize = std::max(curSize, curOffset + numInBuffer);
  2234. return curSize;
  2235. }
  2236. offset_t tell()
  2237. {
  2238. if (reading)
  2239. return curOffset - numInBuffer + curBufferOffset;
  2240. return curOffset + curBufferOffset;
  2241. }
  2242. size32_t read(size32_t len, void * data)
  2243. {
  2244. return CBufferedIOStreamBase::doread(len, data);
  2245. }
  2246. size32_t write(size32_t len, const void * data)
  2247. {
  2248. if (reading)
  2249. curOffset -= bytesRemaining();
  2250. return CBufferedIOStreamBase::dowrite(len, data);
  2251. }
  2252. };
  2253. class CBufferedFileIOStream : public CBufferedFileIOStreamBase
  2254. {
  2255. public:
  2256. CBufferedFileIOStream(IFileIO * _io, unsigned _bufferSize) : CBufferedFileIOStreamBase(_bufferSize), io(_io)
  2257. {
  2258. buffer = new byte[_bufferSize];
  2259. }
  2260. ~CBufferedFileIOStream()
  2261. {
  2262. flush();
  2263. delete [] buffer;
  2264. }
  2265. protected:
  2266. virtual void doflush()
  2267. {
  2268. if (!reading && numInBuffer)
  2269. {
  2270. try {
  2271. io->write(curOffset, numInBuffer, buffer);
  2272. }
  2273. catch (IException *) {
  2274. // if we get exception, clear buffer so doen't reoccur on destructor as well
  2275. numInBuffer = 0;
  2276. curBufferOffset = 0;
  2277. throw;
  2278. }
  2279. curOffset += curBufferOffset;
  2280. numInBuffer = 0;
  2281. curBufferOffset = 0;
  2282. }
  2283. }
  2284. virtual bool fillBuffer()
  2285. {
  2286. reading = true;
  2287. numInBuffer = io->read(curOffset, bufferSize, buffer);
  2288. curOffset += numInBuffer;
  2289. curBufferOffset = 0;
  2290. return numInBuffer!=0;
  2291. }
  2292. virtual size32_t directRead(size32_t len, void * data)
  2293. {
  2294. size32_t sz = io->read(curOffset,len,data);
  2295. curOffset += sz;
  2296. return sz;
  2297. }
  2298. virtual size32_t directWrite(size32_t len, const void * data)
  2299. {
  2300. size32_t sz = io->write(curOffset,len,data);
  2301. curOffset += sz;
  2302. return sz;
  2303. }
  2304. virtual offset_t directSize() { return io->size(); }
  2305. protected:
  2306. IFileIOAttr io;
  2307. };
  2308. //---------------------------------------------------------------------------
  2309. class CBufferedAsyncIOStream: public CBufferedFileIOStreamBase
  2310. {
  2311. Linked<IFileAsyncIO> io;
  2312. byte * blk1;
  2313. byte * blk2;
  2314. IFileAsyncResult *readasyncres;
  2315. IFileAsyncResult *writeasyncres;
  2316. bool readeof;
  2317. public:
  2318. IMPLEMENT_IINTERFACE
  2319. CBufferedAsyncIOStream(IFileAsyncIO * _io, size32_t _bufferSize)
  2320. : CBufferedFileIOStreamBase(_bufferSize/2), io(_io)
  2321. {
  2322. blk1 = new byte[bufferSize];
  2323. blk2 = new byte[bufferSize];
  2324. buffer = blk1;
  2325. readasyncres = NULL;
  2326. writeasyncres = NULL;
  2327. readeof = false;
  2328. minDirectSize = (size32_t)-1; // async always writes using buffer
  2329. }
  2330. ~CBufferedAsyncIOStream()
  2331. {
  2332. flush();
  2333. waitAsyncWrite();
  2334. waitAsyncRead();
  2335. delete [] blk1;
  2336. delete [] blk2;
  2337. }
  2338. void waitAsyncWrite()
  2339. {
  2340. if (writeasyncres) {
  2341. size32_t res;
  2342. writeasyncres->getResult(res,true);
  2343. writeasyncres->Release();
  2344. writeasyncres = NULL;
  2345. }
  2346. }
  2347. size32_t waitAsyncRead()
  2348. {
  2349. size32_t res = 0;
  2350. if (readasyncres) {
  2351. readasyncres->getResult(res,true);
  2352. readasyncres->Release();
  2353. readasyncres = NULL;
  2354. }
  2355. return res;
  2356. }
  2357. void primeAsyncRead(offset_t pos,size32_t size, void *dst)
  2358. {
  2359. assertex(!readasyncres);
  2360. readasyncres = io->readAsync(pos, size, dst);
  2361. }
  2362. void primeAsyncWrite(offset_t pos,size32_t size, const void *src)
  2363. {
  2364. assertex(!writeasyncres);
  2365. writeasyncres = io->writeAsync(pos, size, src);
  2366. }
  2367. // CBufferedFileIOStream overloads
  2368. virtual bool fillBuffer()
  2369. {
  2370. if (!reading) {
  2371. waitAsyncWrite();
  2372. reading = true;
  2373. }
  2374. if (readeof) {
  2375. numInBuffer = 0;
  2376. curBufferOffset = 0;
  2377. return false;
  2378. }
  2379. buffer=(buffer==blk1)?blk2:blk1;
  2380. if (readasyncres==NULL) // first time
  2381. primeAsyncRead(curOffset, bufferSize, buffer);
  2382. numInBuffer = waitAsyncRead();
  2383. curOffset += numInBuffer;
  2384. curBufferOffset = 0;
  2385. if (numInBuffer)
  2386. primeAsyncRead(curOffset, bufferSize, (buffer==blk1)?blk2:blk1);
  2387. else
  2388. readeof = true;
  2389. return !readeof;
  2390. }
  2391. virtual void doflush()
  2392. {
  2393. if (!reading && numInBuffer)
  2394. {
  2395. waitAsyncWrite();
  2396. primeAsyncWrite(curOffset, numInBuffer, buffer);
  2397. buffer=(buffer==blk1)?blk2:blk1;
  2398. curOffset += curBufferOffset;
  2399. numInBuffer = 0;
  2400. curBufferOffset = 0;
  2401. }
  2402. }
  2403. virtual size32_t directRead(size32_t len, void * data) { assertex(false); return 0; } // shouldn't get called
  2404. virtual size32_t directWrite(size32_t len, const void * data) { assertex(false); return 0; } // shouldn't get called
  2405. virtual offset_t directSize() { waitAsyncWrite(); return io->size(); }
  2406. };
  2407. //-- Helper routines --------------------------------------------------------
  2408. enum GblFlushEnum { FLUSH_INIT, FLUSH_DISALLOWED, FLUSH_ALLOWED };
  2409. static GblFlushEnum gbl_flush_allowed = FLUSH_INIT;
  2410. static CriticalSection flushsect;
  2411. static inline bool isPCFlushAllowed()
  2412. {
  2413. CriticalBlock block(flushsect);
  2414. if (gbl_flush_allowed == FLUSH_INIT)
  2415. {
  2416. Owned<IProperties> conf = createProperties(CONFIG_DIR PATHSEPSTR "environment.conf", true);
  2417. if (conf->getPropBool("allow_pgcache_flush", true))
  2418. gbl_flush_allowed = FLUSH_ALLOWED;
  2419. else
  2420. gbl_flush_allowed = FLUSH_DISALLOWED;
  2421. }
  2422. if (gbl_flush_allowed == FLUSH_ALLOWED)
  2423. return true;
  2424. return false;
  2425. }
  2426. static inline size32_t doread(IFileIOStream * stream,void *dst, size32_t size)
  2427. {
  2428. size32_t toread=size;
  2429. while (toread)
  2430. {
  2431. int read = stream->read(toread, dst);
  2432. if (!read)
  2433. return size-toread;
  2434. toread -= read;
  2435. dst = (char *) dst + read;
  2436. }
  2437. return size;
  2438. }
  2439. CIOStreamReadWriteSeq::CIOStreamReadWriteSeq(IFileIOStream * _stream, offset_t _offset, size32_t _size)
  2440. {
  2441. stream.set(_stream);
  2442. // stream->setThrowOnError(true);
  2443. size = _size;
  2444. offset = _offset; // assumption that stream at correct location already
  2445. }
  2446. void CIOStreamReadWriteSeq::put(const void *src)
  2447. {
  2448. stream->write(size, src);
  2449. }
  2450. void CIOStreamReadWriteSeq::putn(const void *src, unsigned n)
  2451. {
  2452. stream->write(size*n, src);
  2453. }
  2454. void CIOStreamReadWriteSeq::flush()
  2455. {
  2456. stream->flush();
  2457. }
  2458. offset_t CIOStreamReadWriteSeq::getPosition()
  2459. {
  2460. return stream->tell();
  2461. }
  2462. bool CIOStreamReadWriteSeq::get(void *dst)
  2463. {
  2464. return doread(stream,dst,size)==size;
  2465. }
  2466. unsigned CIOStreamReadWriteSeq::getn(void *dst, unsigned n)
  2467. {
  2468. return doread(stream,dst,size*n)/size;
  2469. }
  2470. void CIOStreamReadWriteSeq::reset()
  2471. {
  2472. stream->seek(offset, IFSbegin);
  2473. }
  2474. //-- Helper routines --------------------------------------------------------
  2475. size32_t read(IFileIO * in, offset_t pos, size32_t len, MemoryBuffer & buffer)
  2476. {
  2477. const size32_t checkLengthLimit = 0x1000;
  2478. if (len >= checkLengthLimit)
  2479. {
  2480. //Don't allocate a stupid amount of memory....
  2481. offset_t fileLength = in->size();
  2482. if (pos > fileLength)
  2483. pos = fileLength;
  2484. if ((len == (size32_t)-1) || (pos + len > fileLength))
  2485. len = (size32_t)(fileLength - pos);
  2486. }
  2487. void * data = buffer.reserve(len);
  2488. size32_t lenRead = in->read(pos, len, data);
  2489. if (lenRead != len)
  2490. buffer.rewrite(buffer.length() - (len - lenRead));
  2491. return lenRead;
  2492. }
  2493. void copyFile(const char *target, const char *source, size32_t buffersize, ICopyFileProgress *progress, CFflags copyFlags)
  2494. {
  2495. OwnedIFile src = createIFile(source);
  2496. if (!src)
  2497. throw MakeStringException(-1, "copyFile: source '%s' not found", source);
  2498. OwnedIFile tgt = createIFile(target);
  2499. if (!tgt)
  2500. throw MakeStringException(-1, "copyFile: target path '%s' could not be created", target);
  2501. copyFile(tgt, src, buffersize, progress, copyFlags);
  2502. }
  2503. void copyFile(IFile * target, IFile * source, size32_t buffersize, ICopyFileProgress *progress, CFflags copyFlags)
  2504. {
  2505. source->copyTo(target, buffersize, progress, false, copyFlags);
  2506. }
  2507. void doCopyFile(IFile * target, IFile * source, size32_t buffersize, ICopyFileProgress *progress, ICopyFileIntercept *copyintercept, bool usetmp, CFflags copyFlags)
  2508. {
  2509. if (!buffersize)
  2510. buffersize = DEFAULT_COPY_BLKSIZE;
  2511. #ifdef _WIN32
  2512. if (!usetmp) {
  2513. CFile *src = QUERYINTERFACE(source,CFile);
  2514. CFile *dst = QUERYINTERFACE(target,CFile);
  2515. if (src) {
  2516. if (dst) {
  2517. if (src->fastCopyFile(*dst, buffersize, progress))
  2518. return;
  2519. }
  2520. CWindowsRemoteFile *dst2 = QUERYINTERFACE(target,CWindowsRemoteFile);
  2521. if (dst2) {
  2522. if (dst2->fastCopyFileRev(*src, buffersize, progress))
  2523. return;
  2524. }
  2525. }
  2526. CWindowsRemoteFile *src2 = QUERYINTERFACE(source,CWindowsRemoteFile);
  2527. if (src2) {
  2528. if (dst) {
  2529. if (src2->fastCopyFile(*dst,buffersize, progress))
  2530. return;
  2531. }
  2532. CWindowsRemoteFile *dst2 = QUERYINTERFACE(target,CWindowsRemoteFile);
  2533. if (dst2) {
  2534. if (src2->fastCopyFile(*dst2, buffersize, progress))
  2535. return;
  2536. }
  2537. }
  2538. }
  2539. #endif
  2540. IFEflags srcFlags = IFEnone;
  2541. if (copyFlags & CFflush_read)
  2542. srcFlags = IFEnocache;
  2543. OwnedIFileIO sourceIO = source->open(IFOread, srcFlags);
  2544. if (!sourceIO)
  2545. throw MakeStringException(-1, "copyFile: source '%s' not found", source->queryFilename());
  2546. #ifdef __linux__
  2547. // this is not really needed in windows - if it is we will have to
  2548. // test the file extenstion - .exe, .bat
  2549. struct stat info;
  2550. if (stat(source->queryFilename(), &info) == 0) // cannot fail - exception would have been thrown above
  2551. target->setCreateFlags(info.st_mode&(S_IRUSR|S_IRGRP|S_IROTH|S_IWUSR|S_IWGRP|S_IWOTH|S_IXUSR|S_IXGRP|S_IXOTH));
  2552. #endif
  2553. Owned<IFileIO> targetIO;
  2554. Owned<IFile> tmpfile;
  2555. IFile *dest;
  2556. if (usetmp) {
  2557. StringBuffer tmpname;
  2558. makeTempCopyName(tmpname,target->queryFilename());
  2559. tmpfile.setown(createIFile(tmpname.str()));
  2560. dest = tmpfile;
  2561. }
  2562. else
  2563. dest = target;
  2564. IFEflags tgtFlags = IFEnone;
  2565. if (copyFlags & CFflush_write)
  2566. tgtFlags = IFEnocache;
  2567. targetIO.setown(dest->open(IFOcreate, tgtFlags));
  2568. if (!targetIO)
  2569. throw MakeStringException(-1, "copyFile: target path '%s' could not be created", dest->queryFilename());
  2570. MemoryAttr mb;
  2571. void * buffer = copyintercept?NULL:mb.allocate(buffersize);
  2572. offset_t offset = 0;
  2573. offset_t total = 0;
  2574. Owned<IException> exc;
  2575. try
  2576. {
  2577. if (progress)
  2578. total = sourceIO->size(); // only needed for progress
  2579. loop
  2580. {
  2581. size32_t got;
  2582. if (copyintercept) {
  2583. got = (size32_t)copyintercept->copy(sourceIO,targetIO,offset,buffersize);
  2584. if (got == 0)
  2585. break;
  2586. }
  2587. else {
  2588. got = sourceIO->read(offset, buffersize, buffer);
  2589. if (got == 0)
  2590. break;
  2591. targetIO->write(offset, got, buffer);
  2592. }
  2593. offset += got;
  2594. if (progress && progress->onProgress(offset, total) != CFPcontinue)
  2595. break;
  2596. }
  2597. targetIO.clear();
  2598. if (usetmp) {
  2599. StringAttr tail(pathTail(target->queryFilename()));
  2600. target->remove();
  2601. dest->rename(tail);
  2602. }
  2603. }
  2604. catch (IException *e)
  2605. {
  2606. // try to delete partial copy
  2607. StringBuffer s;
  2608. s.append("copyFile target=").append(dest->queryFilename()).append(" source=").append(source->queryFilename()).appendf("; read/write failure (%d): ",e->errorCode());
  2609. exc.setown(MakeStringException(e->errorCode(), "%s", s.str()));
  2610. e->Release();
  2611. EXCLOG(exc, "doCopyFile");
  2612. }
  2613. if (exc.get()) {
  2614. try {
  2615. sourceIO.clear();
  2616. }
  2617. catch (IException *e) {
  2618. EXCLOG(e, "doCopyFile closing source");
  2619. e->Release();
  2620. }
  2621. try {
  2622. targetIO.clear();
  2623. }
  2624. catch (IException *e) {
  2625. EXCLOG(e, "doCopyFile closing dest");
  2626. e->Release();
  2627. }
  2628. try {
  2629. dest->remove();
  2630. }
  2631. catch (IException *e) {
  2632. StringBuffer s;
  2633. EXCLOG(e, s.clear().append("Removing partial copy file: ").append(dest->queryFilename()).str());
  2634. e->Release();
  2635. }
  2636. throw exc.getClear();
  2637. }
  2638. CDateTime createTime, modifiedTime;
  2639. if (source->getTime(&createTime, &modifiedTime, NULL))
  2640. target->setTime(&createTime, &modifiedTime, NULL);
  2641. }
  2642. void makeTempCopyName(StringBuffer &tmpname,const char *destname)
  2643. {
  2644. // simple for the moment (maybe used uid later)
  2645. tmpname.append(destname).append("__");
  2646. #ifdef _WIN32
  2647. genUUID(tmpname,true);
  2648. #else
  2649. genUUID(tmpname);
  2650. #endif
  2651. tmpname.append(".tmp");
  2652. }
  2653. //---------------------------------------------------------------------------
  2654. #ifndef _WIN32
  2655. /// This code is dangerous - pass it arrays smaller than _MAX_xxx and it will write off the end even if you KNOW your strings would fit
  2656. // Should NOT use strncpy here
  2657. void _splitpath(const char *path, char *drive, char *dir, char *fname, char *ext)
  2658. {
  2659. strncpy(dir, path, (_MAX_DIR-1));
  2660. char* last = strrchr(dir, '.');
  2661. if (last != NULL)
  2662. {
  2663. if (strrchr(last, PATHSEPCHAR)==NULL)
  2664. {
  2665. strncpy(ext, last, (_MAX_EXT-1));
  2666. *last = '\0';
  2667. }
  2668. else
  2669. {
  2670. *ext = '\0';
  2671. }
  2672. }
  2673. else
  2674. {
  2675. *ext = '\0';
  2676. }
  2677. last = strrchr(dir, PATHSEPCHAR);
  2678. if (last != NULL)
  2679. {
  2680. strncpy(fname, ++last, (_MAX_FNAME-1));
  2681. *last = '\0';
  2682. }
  2683. else
  2684. {
  2685. strncpy(fname, dir, (_MAX_FNAME-1));
  2686. *dir = '\0';
  2687. }
  2688. *drive = '\0';
  2689. }
  2690. #endif
  2691. void splitFilename(const char * filename, StringBuffer * drive, StringBuffer * path, StringBuffer * tail, StringBuffer * ext, bool longExt)
  2692. {
  2693. char tdrive[_MAX_DRIVE];
  2694. char tdir[_MAX_DIR];
  2695. char ttail[_MAX_FNAME];
  2696. char text[_MAX_EXT];
  2697. ::_splitpath(filename, tdrive, tdir, ttail, text);
  2698. char *longExtStart = longExt ? strchr(ttail, '.') : NULL;
  2699. if (drive)
  2700. drive->append(tdrive);
  2701. if (path)
  2702. path->append(tdir);
  2703. if (tail)
  2704. tail->append(longExtStart ? longExtStart-ttail : strlen(ttail), ttail);
  2705. if (ext)
  2706. {
  2707. if (longExtStart)
  2708. ext->append(longExtStart);
  2709. ext->append(text);
  2710. }
  2711. }
  2712. StringBuffer &createUNCFilename(const char * filename, StringBuffer &UNC, bool useHostNames)
  2713. {
  2714. char buf[255];
  2715. #ifdef _WIN32
  2716. char *dummy;
  2717. GetFullPathName(filename, sizeof(buf), buf, &dummy);
  2718. if (buf[1]==':')
  2719. {
  2720. // MORE - assumes it's a local drive not a mapped one
  2721. UNC.append("\\\\");
  2722. if (useHostNames)
  2723. UNC.append(GetCachedHostName());
  2724. else
  2725. queryHostIP().getIpText(UNC);
  2726. UNC.append("\\").append((char)tolower(buf[0])).append(getShareChar()).append(buf+2);
  2727. }
  2728. else
  2729. {
  2730. assertex(buf[0]=='\\' && buf[1]=='\\');
  2731. UNC.append(buf);
  2732. }
  2733. return UNC;
  2734. #else
  2735. if (filename[0]=='/' && filename[1]=='/')
  2736. UNC.append(filename);
  2737. else
  2738. {
  2739. UNC.append("//");
  2740. if (useHostNames)
  2741. UNC.append(GetCachedHostName());
  2742. else
  2743. queryHostIP().getIpText(UNC);
  2744. if (*filename != '/')
  2745. {
  2746. if (!GetCurrentDirectory(sizeof(buf), buf)) {
  2747. ERRLOG("createUNCFilename: Current directory path too big, bailing out");
  2748. throwUnexpected();
  2749. }
  2750. UNC.append(buf).append("/");
  2751. }
  2752. UNC.append(filename);
  2753. }
  2754. return UNC;
  2755. #endif
  2756. }
  2757. bool splitUNCFilename(const char * filename, StringBuffer * machine, StringBuffer * path, StringBuffer * tail, StringBuffer * ext)
  2758. {
  2759. if (!filename || !isPathSepChar(filename[0]) || !isPathSepChar(filename[1]))
  2760. return false;
  2761. const char * cur = filename+2;
  2762. while (*cur && !isPathSepChar(*cur))
  2763. cur++;
  2764. if (!*cur)
  2765. return false;
  2766. const char * startPath = cur;
  2767. const char * lastExt = NULL;
  2768. const char * startTail = NULL;
  2769. char next;
  2770. while ((next = *cur) != 0)
  2771. {
  2772. if (isPathSepChar(next))
  2773. {
  2774. lastExt = NULL;
  2775. startTail = cur+1;
  2776. }
  2777. else if (next == '.')
  2778. lastExt = cur;
  2779. cur++;
  2780. }
  2781. assertex(startTail);
  2782. if (machine)
  2783. machine->append(startPath-filename, filename);
  2784. if (path)
  2785. path->append(startTail - startPath, startPath);
  2786. if (lastExt)
  2787. {
  2788. if (tail)
  2789. tail->append(lastExt - startTail, startTail);
  2790. if (ext)
  2791. ext->append(lastExt);
  2792. }
  2793. else
  2794. {
  2795. if (tail)
  2796. tail->append(startTail);
  2797. }
  2798. return true;
  2799. }
  2800. /**
  2801. * Ensure the filename has desired extension.
  2802. * If it has no extension, add the desired extension, return true.
  2803. * If it has an extension different from the desiredExtension, return false.
  2804. * Otherwise, return true.
  2805. */
  2806. bool ensureFileExtension(StringBuffer& filename, const char* desiredExtension)
  2807. {
  2808. char drive[_MAX_DRIVE];
  2809. char dir[_MAX_DIR];
  2810. char fname[_MAX_FNAME];
  2811. char ext[_MAX_EXT];
  2812. _splitpath(filename.str(), drive, dir, fname, ext);
  2813. if (ext[0]==0)
  2814. {
  2815. filename.append(desiredExtension);
  2816. return true;
  2817. }
  2818. if (strcmp(ext,desiredExtension)!=0)
  2819. return false;
  2820. return true;
  2821. }
  2822. /* Get full file name. If noExtension is true, the extension (if any) will be trimmed */
  2823. StringBuffer& getFullFileName(StringBuffer& filename, bool noExtension)
  2824. {
  2825. char drive[_MAX_DRIVE];
  2826. char dir[_MAX_DIR];
  2827. char fname[_MAX_FNAME];
  2828. char ext[_MAX_EXT];
  2829. _splitpath(filename.str(), drive, dir, fname, ext);
  2830. filename.clear();
  2831. filename.append(drive).append(dir).append(fname);
  2832. if (!noExtension)
  2833. filename.append(ext);
  2834. return filename;
  2835. }
  2836. /* Get the file name only. If noExtension is true, the extension (if any) will be trimmed */
  2837. StringBuffer& getFileNameOnly(StringBuffer& filename, bool noExtension)
  2838. {
  2839. char drive[_MAX_DRIVE];
  2840. char dir[_MAX_DIR];
  2841. char fname[_MAX_FNAME];
  2842. char ext[_MAX_EXT];
  2843. _splitpath(filename.str(), drive, dir, fname, ext);
  2844. filename.clear();
  2845. filename.append(fname);
  2846. if (!noExtension)
  2847. filename.append(ext);
  2848. return filename;
  2849. }
  2850. //---------------------------------------------------------------------------
  2851. class CNullDirectoryIterator : public CInterface, implements IDirectoryIterator
  2852. {
  2853. public:
  2854. IMPLEMENT_IINTERFACE;
  2855. bool first()
  2856. {
  2857. return false;
  2858. }
  2859. bool next()
  2860. {
  2861. return false;
  2862. }
  2863. StringBuffer &getName(StringBuffer &buf)
  2864. {
  2865. return buf;
  2866. }
  2867. __int64 getFileSize()
  2868. {
  2869. return -1;
  2870. }
  2871. bool getModifiedTime(CDateTime &ret)
  2872. {
  2873. return false;
  2874. }
  2875. virtual bool isValid() { return false; }
  2876. virtual IFile & query() { return *(IFile *)NULL; }
  2877. virtual bool isDir() { return false; }
  2878. };
  2879. extern jlib_decl IDirectoryIterator *createNullDirectoryIterator()
  2880. {
  2881. return new CNullDirectoryIterator;
  2882. }
  2883. class CDirectoryIterator : public CInterface, implements IDirectoryIterator
  2884. {
  2885. public:
  2886. CDirectoryIterator(const char * _path, const char * _mask, bool _sub, bool _includedir)
  2887. {
  2888. StringBuffer tmp;
  2889. if (!_path || !*_path)
  2890. _path = "." PATHSEPSTR;
  2891. else if (_path[strlen(_path)-1] != PATHSEPCHAR)
  2892. _path = tmp.append(_path).append(PATHSEPCHAR);
  2893. path.set(_path);
  2894. mask.set(_mask);
  2895. sub = _sub;
  2896. includedir = _includedir;
  2897. subidx = 0;
  2898. curisdir = false;
  2899. }
  2900. IMPLEMENT_IINTERFACE
  2901. virtual bool first()=0;
  2902. virtual bool next()=0;
  2903. virtual bool isValid() { return cur != NULL; }
  2904. virtual IFile & query() { return *cur; }
  2905. virtual StringBuffer &getName(StringBuffer &buf)=0;
  2906. virtual bool isDir() { return curisdir; }
  2907. protected:
  2908. Owned<IFile> cur;
  2909. bool curisdir;
  2910. StringAttr path;
  2911. StringAttr mask;
  2912. bool includedir;
  2913. bool sub; // TBD
  2914. StringAttrArray subpaths;
  2915. unsigned subidx; // -1 to index subpaths
  2916. };
  2917. #ifdef _WIN32
  2918. class CWindowsDirectoryIterator : public CDirectoryIterator
  2919. {
  2920. WIN32_FIND_DATA info;
  2921. HANDLE handle;
  2922. bool setCurrent()
  2923. {
  2924. if (strcmp(info.cFileName, ".") == 0 || strcmp(info.cFileName, "..") == 0)
  2925. return false;
  2926. bool match = (!mask.length() || WildMatch(info.cFileName, mask, true));
  2927. curisdir = (info.dwFileAttributes&FILE_ATTRIBUTE_DIRECTORY)!=0;
  2928. if (!match&&!curisdir)
  2929. return false;
  2930. StringBuffer f(path);
  2931. if (subidx)
  2932. f.append(subpaths.item(subidx-1).text).append('\\');
  2933. f.append(info.cFileName);
  2934. if (curisdir) {
  2935. if (sub) {
  2936. const char *s = f.str()+path.length();
  2937. unsigned i;
  2938. for (i=0;i<subpaths.ordinality();i++)
  2939. if (stricmp(subpaths.item(i).text,s)==0)
  2940. break;
  2941. if (i==subpaths.ordinality())
  2942. subpaths.append(*new StringAttrItem(s));
  2943. }
  2944. if (!includedir)
  2945. return false;
  2946. }
  2947. if (!match)
  2948. return false;
  2949. cur.setown(createIFile(f.str()));
  2950. return true;
  2951. }
  2952. bool open()
  2953. {
  2954. close();
  2955. while (subidx<=subpaths.ordinality()) {
  2956. StringBuffer location(path);
  2957. if (subidx)
  2958. location.append(subpaths.item(subidx-1).text).append('\\');
  2959. location.append("*");
  2960. handle = FindFirstFile(location.str(), &info);
  2961. if (handle != INVALID_HANDLE_VALUE)
  2962. return true;
  2963. subidx++;
  2964. }
  2965. return false;
  2966. }
  2967. void close()
  2968. {
  2969. cur.clear();
  2970. if (handle != INVALID_HANDLE_VALUE) {
  2971. FindClose(handle);
  2972. handle = INVALID_HANDLE_VALUE;
  2973. }
  2974. };
  2975. public:
  2976. CWindowsDirectoryIterator(const char * _path, const char * _mask, bool _sub, bool _includedir)
  2977. : CDirectoryIterator(_path,_mask,_sub,_includedir)
  2978. {
  2979. handle = INVALID_HANDLE_VALUE;
  2980. }
  2981. ~CWindowsDirectoryIterator()
  2982. {
  2983. close();
  2984. }
  2985. bool first()
  2986. {
  2987. subpaths.kill();
  2988. subidx = 0;
  2989. if (!open())
  2990. return false;
  2991. if (setCurrent())
  2992. return true;
  2993. return next();
  2994. }
  2995. bool next()
  2996. {
  2997. loop {
  2998. loop {
  2999. if (!FindNextFile(handle, &info))
  3000. break;
  3001. if (setCurrent())
  3002. return true;
  3003. }
  3004. subidx++;
  3005. if (!open())
  3006. break;
  3007. if (setCurrent())
  3008. return true;
  3009. }
  3010. return false;
  3011. }
  3012. StringBuffer &getName(StringBuffer &buf)
  3013. {
  3014. if (subidx)
  3015. buf.append(subpaths.item(subidx-1).text).append('\\');
  3016. return buf.append(info.cFileName);
  3017. }
  3018. __int64 getFileSize()
  3019. {
  3020. if (curisdir)
  3021. return -1;
  3022. LARGE_INTEGER x;
  3023. x.LowPart = info.nFileSizeLow;
  3024. x.HighPart = info.nFileSizeHigh;
  3025. return x.QuadPart;
  3026. }
  3027. bool getModifiedTime(CDateTime &ret)
  3028. {
  3029. FILETIMEtoIDateTime(&ret, info.ftLastWriteTime);
  3030. return true;
  3031. }
  3032. };
  3033. IDirectoryIterator * createDirectoryIterator(const char * path, const char * mask)
  3034. {
  3035. if (mask&&!*mask) // only NULL is wild
  3036. return new CNullDirectoryIterator;
  3037. if (!path || !*path) // cur directory so no point in checking for remote etc.
  3038. return new CWindowsDirectoryIterator(path, mask,false,true);
  3039. OwnedIFile iFile = createIFile(path);
  3040. if (!iFile||(iFile->isDirectory()!=foundYes))
  3041. return new CNullDirectoryIterator;
  3042. return iFile->directoryFiles(mask, false, true);
  3043. }
  3044. IDirectoryIterator *CFile::directoryFiles(const char *mask,bool sub,bool includedirs)
  3045. {
  3046. if ((mask&&!*mask)|| // only NULL is wild
  3047. (isDirectory()!=foundYes))
  3048. return new CNullDirectoryIterator;
  3049. return new CWindowsDirectoryIterator(filename, mask,sub,includedirs);
  3050. }
  3051. bool CFile::getInfo(bool &isdir,offset_t &size,CDateTime &modtime)
  3052. {
  3053. WIN32_FILE_ATTRIBUTE_DATA info;
  3054. if (GetFileAttributesEx(filename, GetFileExInfoStandard, &info) != 0) {
  3055. LARGE_INTEGER x;
  3056. x.LowPart = info.nFileSizeLow;
  3057. x.HighPart = info.nFileSizeHigh;
  3058. size = (offset_t)x.QuadPart;
  3059. isdir = (info.dwFileAttributes != (DWORD)-1)&&(info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY);
  3060. FILETIMEtoIDateTime(&modtime, info.ftLastWriteTime);
  3061. return true;
  3062. }
  3063. size = 0;
  3064. modtime.clear();
  3065. isdir = false;
  3066. return false;
  3067. }
  3068. #else
  3069. class CLinuxDirectoryIterator : public CDirectoryIterator
  3070. {
  3071. StringAttr tail;
  3072. DIR * handle;
  3073. struct stat st;
  3074. bool gotst;
  3075. bool loadst()
  3076. {
  3077. if (!gotst&&cur)
  3078. gotst = (stat(cur->queryFilename(), &st) == 0);
  3079. return gotst;
  3080. }
  3081. public:
  3082. CLinuxDirectoryIterator(const char * _path, const char * _mask, bool _sub,bool _includedir)
  3083. : CDirectoryIterator(_path,_mask,_sub,_includedir)
  3084. {
  3085. handle = NULL;
  3086. gotst = false;
  3087. }
  3088. ~CLinuxDirectoryIterator()
  3089. {
  3090. close();
  3091. }
  3092. bool open()
  3093. {
  3094. close();
  3095. while (subidx<=subpaths.ordinality()) {
  3096. StringBuffer location(path);
  3097. if (subidx)
  3098. location.append(subpaths.item(subidx-1).text);
  3099. // not sure if should remove trailing '/'
  3100. handle = ::opendir(location.str());
  3101. // better error handling here?
  3102. if (handle)
  3103. return true;
  3104. subidx++;
  3105. }
  3106. return false;
  3107. }
  3108. void close()
  3109. {
  3110. cur.clear();
  3111. if (handle) {
  3112. closedir(handle);
  3113. handle = NULL;
  3114. }
  3115. }
  3116. bool first()
  3117. {
  3118. subpaths.kill();
  3119. subidx = 0;
  3120. if (open())
  3121. return next();
  3122. return false;
  3123. }
  3124. bool next()
  3125. {
  3126. loop {
  3127. struct dirent dirEntry;
  3128. struct dirent *entry;
  3129. loop {
  3130. gotst = false;
  3131. readdir_r(handle, &dirEntry, &entry);
  3132. // need better checking here?
  3133. if (!entry)
  3134. break;
  3135. if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
  3136. continue;
  3137. bool match = (!mask.length() || WildMatch(entry->d_name, mask, false));
  3138. curisdir = (entry->d_type==DT_DIR);
  3139. bool islnk = (entry->d_type==DT_LNK);
  3140. bool isunknown = (entry->d_type==DT_UNKNOWN); // to work around xfs bug
  3141. if (match||curisdir||islnk||isunknown) {
  3142. StringBuffer f(path);
  3143. if (subidx)
  3144. f.append(subpaths.item(subidx-1).text).append('/');
  3145. f.append(entry->d_name);
  3146. if (islnk||isunknown) {
  3147. struct stat info;
  3148. if (stat(f.str(), &info) == 0) // will follow link
  3149. curisdir = S_ISDIR(info.st_mode);
  3150. else
  3151. curisdir = false;
  3152. }
  3153. if (curisdir) {
  3154. if (sub) {
  3155. const char *s = f.str()+path.length();
  3156. unsigned i;
  3157. for (i=0;i<subpaths.ordinality();i++)
  3158. if (strcmp(subpaths.item(i).text,s)==0)
  3159. break;
  3160. if (i==subpaths.ordinality())
  3161. subpaths.append(*new StringAttrItem(s));
  3162. }
  3163. if (!includedir)
  3164. match = false;
  3165. }
  3166. if (match) {
  3167. tail.set(entry->d_name);
  3168. cur.setown(createIFile(f.str()));
  3169. return true;
  3170. }
  3171. }
  3172. }
  3173. subidx++;
  3174. if (!open())
  3175. break;
  3176. }
  3177. return false;
  3178. }
  3179. StringBuffer &getName(StringBuffer &buf)
  3180. {
  3181. if (subidx)
  3182. buf.append(subpaths.item(subidx-1).text).append('/');
  3183. return buf.append(tail);
  3184. }
  3185. __int64 getFileSize()
  3186. {
  3187. if (curisdir)
  3188. return -1;
  3189. if (!loadst())
  3190. return -1;
  3191. return st.st_size ;
  3192. }
  3193. bool getModifiedTime(CDateTime &ret)
  3194. {
  3195. if (!loadst())
  3196. return false;
  3197. timetToIDateTime(&ret,st.st_mtime);
  3198. return true;
  3199. }
  3200. };
  3201. IDirectoryIterator * createDirectoryIterator(const char * path, const char * mask)
  3202. {
  3203. if (mask&&!*mask) // only NULL is wild
  3204. return new CNullDirectoryIterator;
  3205. if (!path || !*path) // no point in checking for remote etc.
  3206. return new CLinuxDirectoryIterator(path, mask,false,true);
  3207. OwnedIFile iFile = createIFile(path);
  3208. if (!iFile||(iFile->isDirectory()!=foundYes))
  3209. return new CNullDirectoryIterator;
  3210. return iFile->directoryFiles(mask, false, true);
  3211. }
  3212. IDirectoryIterator *CFile::directoryFiles(const char *mask,bool sub,bool includedirs)
  3213. {
  3214. if ((mask&&!*mask)|| // only NULL is wild
  3215. (isDirectory()!=foundYes))
  3216. return new CNullDirectoryIterator;
  3217. return new CLinuxDirectoryIterator(filename, mask,sub,includedirs);
  3218. }
  3219. bool CFile::getInfo(bool &isdir,offset_t &size,CDateTime &modtime)
  3220. {
  3221. struct stat info;
  3222. if (stat(filename, &info) == 0) {
  3223. size = (offset_t)info.st_size;
  3224. isdir = S_ISDIR(info.st_mode);
  3225. timetToIDateTime(&modtime, info.st_mtime);
  3226. return true;
  3227. }
  3228. size = 0;
  3229. modtime.clear();
  3230. isdir = false;
  3231. return false;
  3232. }
  3233. #endif
  3234. class CDirEntry: extends CInterface
  3235. {
  3236. public:
  3237. StringAttr name;
  3238. Owned<IFile> file;
  3239. __int64 size;
  3240. CDateTime modified;
  3241. byte flags; // IDDI*
  3242. bool isdir;
  3243. CDirEntry(IDirectoryIterator *iter)
  3244. {
  3245. StringBuffer n;
  3246. name.set(iter->getName(n).str());
  3247. size = iter->getFileSize();
  3248. iter->getModifiedTime(modified);
  3249. file.set(&iter->query());
  3250. flags = 0;
  3251. isdir = iter->isDir();
  3252. }
  3253. bool match(byte f)
  3254. {
  3255. return (flags==0)||((f&flags)!=0);
  3256. }
  3257. int compare(const CDirEntry *e)
  3258. {
  3259. return strcmp(name.get(),e->name.get());
  3260. }
  3261. int compareProp(const CDirEntry *e)
  3262. {
  3263. int c = compare(e);
  3264. if (c)
  3265. return c;
  3266. if (isdir!=e->isdir)
  3267. return isdir?-1:1;
  3268. if (size!=e->size)
  3269. return (size<e->size)?-1:1;
  3270. return modified.compare(e->modified,false);
  3271. }
  3272. };
  3273. class CDirectoryDifferenceIterator : public CIArrayOf<CDirEntry>, extends CInterface, implements IDirectoryDifferenceIterator
  3274. {
  3275. static int compare(CInterface * const *_a, CInterface * const *_b)
  3276. {
  3277. CDirEntry *a = *(CDirEntry **)_a;
  3278. CDirEntry *b = *(CDirEntry **)_b;
  3279. return a->compare(b);
  3280. }
  3281. unsigned idx;
  3282. byte mask;
  3283. public:
  3284. IMPLEMENT_IINTERFACE;
  3285. CDirectoryDifferenceIterator(IDirectoryIterator *iter, CDirectoryDifferenceIterator *cmp)
  3286. {
  3287. mask = IDDIstandard;
  3288. idx = 0;
  3289. ForEach(*iter)
  3290. append(*new CDirEntry(iter));
  3291. CIArrayOf<CDirEntry>::sort(compare);
  3292. if (cmp) { // assumes cmp sorted
  3293. unsigned i = 0;
  3294. unsigned ni = ordinality();
  3295. unsigned j = 0;
  3296. unsigned nj = cmp->ordinality();
  3297. loop {
  3298. CDirEntry *a = NULL;
  3299. CDirEntry *b = NULL;
  3300. if (i>=ni) {
  3301. if (j>=nj)
  3302. break;
  3303. b = &cmp->item(j++);
  3304. }
  3305. else if (j>=nj) {
  3306. a = &item(i++);
  3307. }
  3308. else {
  3309. a = &item(i);
  3310. b = &cmp->item(j);
  3311. int c = a->compare(b);
  3312. if (c==0) {
  3313. if (a->compareProp(b)==0) {
  3314. a->flags = IDDIunchanged;
  3315. a = NULL;
  3316. b = NULL;
  3317. }
  3318. i++;
  3319. j++;
  3320. }
  3321. else if (c<0) {
  3322. b = NULL;
  3323. i++;
  3324. }
  3325. else {
  3326. a = NULL;
  3327. j++;
  3328. }
  3329. }
  3330. if (a) {
  3331. if (b)
  3332. a->flags|=IDDImodified;
  3333. else
  3334. a->flags|=IDDIadded;
  3335. }
  3336. else if (b) {
  3337. b->Link();
  3338. b->flags = IDDIdeleted;
  3339. add(*b,i);
  3340. i++;
  3341. ni++;
  3342. }
  3343. }
  3344. }
  3345. }
  3346. virtual bool first()
  3347. {
  3348. idx = 0;
  3349. while (idx<ordinality()) {
  3350. if (item(idx).match(mask))
  3351. return true;
  3352. idx++;
  3353. }
  3354. return false;
  3355. }
  3356. virtual bool next()
  3357. {
  3358. idx++;
  3359. while (idx<ordinality()) {
  3360. if (item(idx).match(mask))
  3361. return true;
  3362. idx++;
  3363. }
  3364. return false;
  3365. }
  3366. virtual bool isValid()
  3367. {
  3368. return (idx<ordinality());
  3369. }
  3370. virtual IFile & query()
  3371. {
  3372. if (isValid())
  3373. return *item(idx).file;
  3374. return *(IFile *) NULL;
  3375. }
  3376. virtual StringBuffer &getName(StringBuffer &buf)
  3377. {
  3378. if (isValid())
  3379. buf.append(item(idx).name);
  3380. return buf;
  3381. }
  3382. virtual bool isDir()
  3383. {
  3384. if (isValid())
  3385. return item(idx).isdir;
  3386. return false;
  3387. }
  3388. __int64 getFileSize()
  3389. {
  3390. if (isValid())
  3391. return item(idx).isdir?0:item(idx).size;
  3392. return 0;
  3393. }
  3394. virtual bool getModifiedTime(CDateTime &ret)
  3395. {
  3396. if (isValid()) {
  3397. ret.set(item(idx).modified);
  3398. }
  3399. return false;
  3400. }
  3401. virtual void setMask(unsigned _mask)
  3402. {
  3403. mask = (byte)_mask;
  3404. }
  3405. virtual unsigned getFlags()
  3406. {
  3407. if (isValid())
  3408. return item(idx).flags;
  3409. return 0;
  3410. }
  3411. unsigned numChanges()
  3412. {
  3413. unsigned ret=0;
  3414. ForEachItemIn(i,*this) {
  3415. byte f = item(i).flags;
  3416. if ((f!=0)&&(f!=IDDIunchanged))
  3417. ret++;
  3418. }
  3419. return ret;
  3420. }
  3421. };
  3422. IDirectoryDifferenceIterator *CFile::monitorDirectory(IDirectoryIterator *_prev, // in
  3423. const char *mask,
  3424. bool sub,
  3425. bool includedirs,
  3426. unsigned checkinterval,
  3427. unsigned timeout,
  3428. Semaphore *abortsem)
  3429. {
  3430. Linked<IDirectoryIterator> prev;
  3431. if (_prev)
  3432. prev.set(_prev);
  3433. else
  3434. prev.setown(directoryFiles(mask,sub,includedirs));
  3435. if (!prev)
  3436. return NULL;
  3437. Owned<CDirectoryDifferenceIterator> base = new CDirectoryDifferenceIterator(prev,NULL);
  3438. prev.clear(); // not needed now
  3439. unsigned start=msTick();
  3440. loop {
  3441. if (abortsem) {
  3442. if (abortsem->wait(checkinterval))
  3443. break;
  3444. }
  3445. else
  3446. Sleep(checkinterval);
  3447. Owned<IDirectoryIterator> current = directoryFiles(mask,sub,includedirs);
  3448. if (!current)
  3449. break;
  3450. Owned<CDirectoryDifferenceIterator> cmp = new CDirectoryDifferenceIterator(current,base);
  3451. current.clear();
  3452. if (cmp->numChanges())
  3453. return cmp.getClear();
  3454. if (msTick()-start>timeout)
  3455. break;
  3456. }
  3457. return NULL; // timed out
  3458. }
  3459. //---------------------------------------------------------------------------
  3460. class FixedPasswordProvider : public CInterface, implements IPasswordProvider
  3461. {
  3462. public:
  3463. FixedPasswordProvider(const char * _username, const char * _password) { username.set(_username); password.set(_password); }
  3464. IMPLEMENT_IINTERFACE;
  3465. virtual bool getPassword(const IpAddress & ip, StringBuffer & _username, StringBuffer & _password)
  3466. {
  3467. _username.append(username.get());
  3468. _password.append(password.get());
  3469. return true;
  3470. }
  3471. protected:
  3472. StringAttr username;
  3473. StringAttr password;
  3474. };
  3475. IPasswordProvider * queryPasswordProvider()
  3476. {
  3477. return passwordProvider;
  3478. }
  3479. void setPasswordProvider(IPasswordProvider * provider)
  3480. {
  3481. passwordProvider.set(provider);
  3482. }
  3483. void setDefaultUser(const char * username,const char *password)
  3484. {
  3485. Owned<IPasswordProvider> provider = new FixedPasswordProvider(username, password);
  3486. setPasswordProvider(provider);
  3487. }
  3488. //---------------------------------------------------------------------------
  3489. bool recursiveCreateDirectory(const char * path)
  3490. {
  3491. Owned<IFile> file = createIFile(path);
  3492. return file->createDirectory();
  3493. }
  3494. bool recursiveCreateDirectoryForFile(const char *fullFileName)
  3495. {
  3496. StringBuffer path;
  3497. splitFilename(fullFileName, &path, &path, NULL, NULL);
  3498. return recursiveCreateDirectory(path.str());
  3499. }
  3500. //---------------------------------------------------------------------------
  3501. size32_t DirectBufferI::read(offset_t pos, size32_t len, void * data)
  3502. {
  3503. if (pos + len > buffLen)
  3504. {
  3505. if (pos > buffLen)
  3506. pos = buffLen;
  3507. len = (size32_t)(buffLen - pos);
  3508. }
  3509. memcpy(data, buffer+pos, len);
  3510. return len;
  3511. }
  3512. size32_t DirectBufferI::write(offset_t pos, size32_t len, const void * data)
  3513. {
  3514. UNIMPLEMENTED;
  3515. }
  3516. size32_t DirectBufferIO::write(offset_t pos, size32_t len, const void * data)
  3517. {
  3518. if (pos + len > buffLen)
  3519. {
  3520. if (pos > buffLen)
  3521. pos = buffLen;
  3522. len = (size32_t)(buffLen - pos);
  3523. }
  3524. memcpy(buffer+pos, data, len);
  3525. return len;
  3526. }
  3527. //---------------------------------------------------------------------------
  3528. IFile * createIFile(const char * filename)
  3529. {
  3530. if (!filename)
  3531. return NULL;
  3532. IFile *ret = createContainedIFileByHook(filename);
  3533. if (ret)
  3534. return ret;
  3535. bool linremote=(memcmp(filename,"//",2)==0);
  3536. if (!linremote&&(memcmp(filename,"\\\\",2)!=0)) // see if remote looking
  3537. return new CFile(filename);
  3538. RemoteFilename rfn;
  3539. rfn.setRemotePath(filename);
  3540. if (rfn.isNull())
  3541. throw MakeStringException(-1, "CreateIFile cannot resolve %s", filename);
  3542. if (rfn.isLocal()) { // ignore dafilesrv request if local and standard port
  3543. StringBuffer tmplocal;
  3544. return new CFile(rfn.getLocalPath(tmplocal).str());
  3545. }
  3546. #ifdef _WIN32
  3547. StringBuffer tmplocal;
  3548. if (linremote||(rfn.queryEndpoint().port!=0)) {
  3549. ret = createIFileByHook(rfn); // use daliservix in preference
  3550. if (ret)
  3551. return ret;
  3552. while (*filename) { // no daliservix so swap '/' for '\' and hope for best
  3553. if (*filename=='/')
  3554. tmplocal.append('\\');
  3555. else
  3556. tmplocal.append(*filename);
  3557. filename++;
  3558. }
  3559. filename =tmplocal.str();
  3560. }
  3561. return new CWindowsRemoteFile(filename);
  3562. #else
  3563. #ifdef USE_SAMBA
  3564. if(strncmp(filename, "smb://", 6) == 0)
  3565. return new CSambaRemoteFile(filename);
  3566. if(memcmp(filename, "//", 2) == 0) {
  3567. StringBuffer smbfile("smb:");
  3568. smbfile.append(filename);
  3569. return new CSambaRemoteFile(smbfile.str());
  3570. }
  3571. if(memcmp(filename, "\\\\", 2) == 0) {
  3572. StringBuffer smbfile("smb:");
  3573. int i = 0;
  3574. while(filename[i]) {
  3575. if(filename[i] == '\\')
  3576. smbfile.append('/');
  3577. else
  3578. smbfile.append(filename[i]);
  3579. i++;
  3580. }
  3581. return new CSambaRemoteFile(smbfile.str());
  3582. }
  3583. #else
  3584. if (memcmp(filename,"smb://",6)==0) // don't support samba - try remote
  3585. return createIFile(filename+4);
  3586. ret = createIFileByHook(rfn);
  3587. if (!ret)
  3588. throw MakeStringException(-1, "CreateIFile::cannot attach to %s. (remote.so not linked?)", filename);
  3589. return ret;
  3590. #endif
  3591. return new CFile(filename);
  3592. #endif
  3593. }
  3594. IFileIOStream * createIOStream(IFileIO * file)
  3595. {
  3596. return new CFileIOStream(file);
  3597. }
  3598. IFileIO * createIORange(IFileIO * io, offset_t header, offset_t length)
  3599. {
  3600. return new CFileRangeIO(io, header, length);
  3601. }
  3602. IFileIOStream * createBufferedIOStream(IFileIO * io, unsigned bufsize)
  3603. {
  3604. if (bufsize == (unsigned)-1)
  3605. bufsize = DEFAULT_BUFFER_SIZE;
  3606. return new CBufferedFileIOStream(io, bufsize);
  3607. }
  3608. IFileIOStream * createBufferedAsyncIOStream(IFileAsyncIO * io, unsigned bufsize)
  3609. {
  3610. if (bufsize == (unsigned)-1)
  3611. bufsize = DEFAULT_BUFFER_SIZE*2;
  3612. return new CBufferedAsyncIOStream(io, bufsize);
  3613. }
  3614. IReadSeq *createReadSeq(IFileIOStream * stream, offset_t offset, size32_t size)
  3615. {
  3616. return new CIOStreamReadWriteSeq(stream, offset, size);
  3617. }
  3618. IWriteSeq *createWriteSeq(IFileIOStream * stream, size32_t size)
  3619. {
  3620. return new CIOStreamReadWriteSeq(stream, 0, size);
  3621. }
  3622. extern jlib_decl offset_t filesize(const char *name)
  3623. {
  3624. CFile f(name);
  3625. return f.size();
  3626. }
  3627. extern jlib_decl offset_t getFreeSpace(const char* name)
  3628. {
  3629. offset_t freeBytesToCaller;
  3630. #ifdef _WIN32
  3631. offset_t totalBytes;
  3632. offset_t freeBytes;
  3633. int fResult = GetDiskFreeSpaceEx (name,
  3634. (PULARGE_INTEGER)&freeBytesToCaller,
  3635. (PULARGE_INTEGER)&totalBytes,
  3636. (PULARGE_INTEGER)&freeBytes);
  3637. if (fResult == 0) // error
  3638. {
  3639. return 0;
  3640. }
  3641. #elif defined (__linux__)
  3642. struct statfs buf;
  3643. int fResult = statfs(name, &buf);
  3644. if (fResult == -1) // error
  3645. {
  3646. return 0;
  3647. }
  3648. // from "man statfs the def of f_bavail and f_bsize...
  3649. // mult "free blocks avail to non-superuser" by "optimal transfer block size" to get the available size
  3650. freeBytesToCaller = (offset_t)buf.f_bavail * buf.f_bsize;
  3651. #else
  3652. UNIMPLEMENTED;
  3653. #endif
  3654. return freeBytesToCaller;
  3655. }
  3656. extern jlib_decl void createHardLink(const char* fileName, const char* existingFileName)
  3657. {
  3658. #ifdef _WIN32
  3659. // requirements...need to make sure that the directory already exists
  3660. // and the 2 directories need to be on the same drive
  3661. if (!CreateHardLink(fileName, existingFileName, NULL))
  3662. {
  3663. LPVOID lpMsgBuf;
  3664. FormatMessage(
  3665. FORMAT_MESSAGE_ALLOCATE_BUFFER |
  3666. FORMAT_MESSAGE_FROM_SYSTEM |
  3667. FORMAT_MESSAGE_IGNORE_INSERTS,
  3668. NULL,
  3669. GetLastError(),
  3670. MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), // Default language
  3671. (LPTSTR) &lpMsgBuf,
  3672. 0,
  3673. NULL
  3674. );
  3675. StringBuffer err;
  3676. err.appendf("Failed to create log alias %s for %s: %s", fileName, existingFileName, lpMsgBuf);
  3677. LocalFree( lpMsgBuf );
  3678. throw MakeStringException(-1, "createHardLink:: %s.",err.str());
  3679. }
  3680. #else
  3681. if (link(existingFileName, fileName) != 0) // error
  3682. throw MakeStringException(-1, "Failed to create log alias %s for %s: error code = %d", fileName, existingFileName, errno);
  3683. #endif
  3684. }
  3685. #if 0
  3686. void testDirectory()
  3687. {
  3688. //setDefaultUser("administrator", "password");
  3689. Owned<IFile> dir = createIFile("\\test1\\gavin\\home\\base");
  3690. dir->createDirectory();
  3691. Owned<IFile> dir2 = createIFile("\\test1\\gavin\\home\\base2\\");
  3692. dir2->createDirectory();
  3693. dir2->createDirectory();
  3694. Owned<IFile> dir3 = createIFile("c:\\test1\\gavin\\home2\\base");
  3695. dir3->createDirectory();
  3696. Owned<IFile> dir4 = createIFile("\\\\ghalliday\\c$\\test1\\gavin\\home3\\base");
  3697. dir4->createDirectory();
  3698. Owned<IFile> dir5 = createIFile("\\\\192.168.1.3\\d$\\test2\\gavin\\home3\\base");
  3699. dir5->createDirectory();
  3700. StringBuffer x;
  3701. x.clear();
  3702. splitUNCFilename("\\\\127.0.0.1\\gavin\\abc.ext", &x, &x, &x, &x);
  3703. x.clear();
  3704. splitUNCFilename("\\\\127.0.0.1\\gavin\\abc.ext\\etc", &x, &x, &x, &x);
  3705. x.clear();
  3706. splitUNCFilename("\\\\127.0.0.1\\", &x, &x, &x, &x);
  3707. }
  3708. #endif
  3709. // -- RemoteFilename class (file location encapsulation)
  3710. //#define _TRACERFN
  3711. void RemoteFilename::badFilename(const char * filename)
  3712. {
  3713. throw MakeStringException(-1, "Badly formatted file entry %s", filename);
  3714. }
  3715. bool RemoteFilename::equals(const RemoteFilename & other) const
  3716. {
  3717. if (isNull())
  3718. return other.isNull();
  3719. if (other.isNull())
  3720. return false;
  3721. if (!ep.ipequals(other.ep)) // only use ip for compare
  3722. return false;
  3723. if (isUnixPath()) { // case sensitive
  3724. if (localhead.length()&&other.localhead.length()) {
  3725. if (strcmp(localhead.get(),other.localhead.get())!=0)
  3726. return false;
  3727. }
  3728. else if (sharehead.length()&&other.sharehead.length()) {
  3729. if (strcmp(sharehead.get(),other.sharehead.get())!=0)
  3730. return false;
  3731. }
  3732. else if (sharehead.length()!=other.sharehead.length())
  3733. return false;
  3734. if (strcmp(tailpath.get(),other.tailpath.get())!=0)
  3735. return false;
  3736. }
  3737. else {
  3738. if (localhead.length()&&other.localhead.length()) {
  3739. if (stricmp(localhead.get(),other.localhead.get())!=0)
  3740. return false;
  3741. }
  3742. else if (sharehead.length()&&other.sharehead.length()) {
  3743. if (stricmp(sharehead.get(),other.sharehead.get())!=0)
  3744. return false;
  3745. }
  3746. else if (sharehead.length()!=other.sharehead.length())
  3747. return false;
  3748. if (stricmp(tailpath.get(),other.tailpath.get())!=0)
  3749. return false;
  3750. }
  3751. return true;
  3752. }
  3753. bool RemoteFilename::isNull() const
  3754. {
  3755. return (ep.isNull()||(tailpath.length()==0));
  3756. }
  3757. void RemoteFilename::clear()
  3758. {
  3759. ep.set(NULL, 0);
  3760. localhead.clear();
  3761. sharehead.clear();
  3762. tailpath.clear();
  3763. }
  3764. void RemoteFilename::serialize(MemoryBuffer & out)
  3765. {
  3766. ep.serialize(out);
  3767. out.append(localhead);
  3768. out.append(sharehead);
  3769. out.append(tailpath);
  3770. }
  3771. void RemoteFilename::deserialize(MemoryBuffer & in)
  3772. {
  3773. ep.deserialize(in);
  3774. in.read(localhead);
  3775. in.read(sharehead);
  3776. in.read(tailpath);
  3777. }
  3778. StringBuffer & RemoteFilename::getTail(StringBuffer &name) const
  3779. {
  3780. split(NULL, NULL, &name, &name);
  3781. return name;
  3782. }
  3783. StringBuffer & RemoteFilename::getPath(StringBuffer & name) const
  3784. {
  3785. if (isLocal())
  3786. return getLocalPath(name);
  3787. else
  3788. return getRemotePath(name);
  3789. }
  3790. bool RemoteFilename::isUnixPath() const // bit arbitrary
  3791. {
  3792. if (tailpath.length()!=0) {
  3793. char c = tailpath[0];
  3794. if (c=='/')
  3795. return true;
  3796. if (c=='\\')
  3797. return false;
  3798. }
  3799. if (sharehead.length()!=0) {
  3800. char c = sharehead[0];
  3801. if (c=='/')
  3802. return true;
  3803. if (c=='\\')
  3804. return false;
  3805. }
  3806. if (localhead.length()!=0) {
  3807. const char *s=localhead;
  3808. if (*s=='/')
  3809. return true;
  3810. if ((s[1]==':')&&(s[2]=='/'))
  3811. return false;
  3812. // those are the expected cases, otherwise look for first separator (legacy)
  3813. while (*s) {
  3814. if (*s=='\\')
  3815. return false;
  3816. s++;
  3817. if (*s=='/')
  3818. return true;
  3819. }
  3820. }
  3821. #ifdef _WIN32 // assume local on starting machine
  3822. return false;
  3823. #else
  3824. return true;
  3825. #endif
  3826. }
  3827. char RemoteFilename::getPathSeparator() const
  3828. {
  3829. return isUnixPath()?'/':'\\';
  3830. }
  3831. StringBuffer & RemoteFilename::getLocalPath(StringBuffer & out) const
  3832. {
  3833. if (tailpath.length()!=0) {
  3834. out.append(localhead).append(tailpath);
  3835. }
  3836. #ifdef _TRACERFN
  3837. StringBuffer eps;
  3838. PROGLOG("getLocalPath '%s' '%s' '%s' -> '%s'",sharehead.get()?sharehead.get():"NULL",localhead.get()?localhead.get():"NULL",tailpath.get()?tailpath.get():"NULL",out.str());
  3839. #endif
  3840. return out;
  3841. }
  3842. StringBuffer & RemoteFilename::getRemotePath(StringBuffer & out) const
  3843. { // this creates a name that can be used by windows or linux
  3844. // note - no longer loses port
  3845. char c=getPathSeparator();
  3846. out.append(c).append(c);
  3847. ep.getUrlStr(out);
  3848. const char *fn;
  3849. StringBuffer loc;
  3850. if (sharehead.length())
  3851. fn = loc.append(sharehead).append(tailpath).str();
  3852. else // try and guess from just tail (may likely fail other than for windows)
  3853. fn=getLocalPath(loc).str();
  3854. if ((c=='\\')&&(fn[1]==':')) { // windows \\d$
  3855. out.append((char)tolower(c)).append(*fn).append(getShareChar());
  3856. fn+=2;
  3857. }
  3858. out.append(fn);
  3859. #ifdef _TRACERFN
  3860. StringBuffer eps;
  3861. PROGLOG("getRemotePath '%s' '%s' '%s' -> '%s'",sharehead.get()?sharehead.get():"NULL",localhead.get()?localhead.get():"NULL",tailpath.get()?tailpath.get():"NULL",out.str());
  3862. #endif
  3863. return out;
  3864. }
  3865. bool RemoteFilename::isLocal() const
  3866. {
  3867. if (ep.port&&(ep.port!=DAFILESRV_PORT && ep.port!=SECURE_DAFILESRV_PORT))
  3868. return false; // treat non-dafilesrv port as remote
  3869. return ep.isLocal() || ep.isNull();
  3870. }
  3871. void RemoteFilename::set(const RemoteFilename & other)
  3872. {
  3873. ep.set(other.ep);
  3874. localhead.set(other.localhead);
  3875. sharehead.set(other.sharehead);
  3876. tailpath.set(other.tailpath);
  3877. }
  3878. void RemoteFilename::setExtension(const char * newext)
  3879. {
  3880. const char * dot = NULL;
  3881. const char * start = tailpath;
  3882. const char * cur = start;
  3883. while (*cur)
  3884. {
  3885. if (*cur == '.')
  3886. dot = cur;
  3887. cur++;
  3888. }
  3889. StringBuffer newtail;
  3890. if (dot)
  3891. newtail.append(dot-start,start);
  3892. else
  3893. newtail.append(start);
  3894. if (newext)
  3895. {
  3896. if (newext[0] != '.')
  3897. newtail.append('.');
  3898. newtail.append(newext);
  3899. }
  3900. tailpath.set(newtail);
  3901. }
  3902. void RemoteFilename::setPath(const SocketEndpoint & _ep, const char * _filename)
  3903. {
  3904. const char * filename=_filename;
  3905. StringBuffer full;
  3906. ep.set(_ep);
  3907. localhead.clear();
  3908. sharehead.clear();
  3909. if (filename&&*filename) {
  3910. if (isSpecialPath(filename)) {
  3911. tailpath.set(filename);
  3912. return;
  3913. }
  3914. if (isLocal()&&!isAbsolutePath(filename)) {
  3915. char dir[_MAX_PATH];
  3916. if (!GetCurrentDirectory(sizeof(dir), dir)) {
  3917. ERRLOG("RemoteFilename::setPath: Current directory path too big, bailing out");
  3918. throwUnexpected();
  3919. }
  3920. if (*filename==PATHSEPCHAR) {
  3921. #ifdef _WIN32
  3922. if (*dir && (dir[1]==':'))
  3923. dir[2] = 0;
  3924. #endif
  3925. filename++;
  3926. }
  3927. full.append(dir);
  3928. if (full.charAt(full.length()-1) != PATHSEPCHAR)
  3929. full.append(PATHSEPCHAR);
  3930. full.append(filename);
  3931. filename = full.str();
  3932. }
  3933. bool isunix;
  3934. if (filename[0]=='/')
  3935. isunix=true;
  3936. else if (filename[0]=='\\')
  3937. isunix=false;
  3938. else if ((filename[1]==':')&&(filename[2]=='\\'))
  3939. isunix = false;
  3940. else // invalid call really as supposed to be full local path!
  3941. #ifdef _WIN32
  3942. isunix = false;
  3943. #else
  3944. isunix = true;
  3945. #endif
  3946. StringBuffer tmpxlat;
  3947. char sepchar = isunix?'/':'\\';
  3948. char altchar = isunix?'\\':'/';
  3949. // fix mixed separator path
  3950. if (strchr(filename,altchar)) {
  3951. while (*filename) {
  3952. if (*filename==altchar)
  3953. tmpxlat.append(sepchar);
  3954. else
  3955. tmpxlat.append(*filename);
  3956. filename++;
  3957. }
  3958. filename = tmpxlat.str();
  3959. }
  3960. if (isunix) {
  3961. if (filename[0]=='/') {
  3962. const char *tail;
  3963. tail=strchr(filename+1,'/');
  3964. if (tail) {
  3965. sharehead.set(filename,tail-filename); // we don't know share so guess same as leading dir
  3966. localhead.set(filename,tail-filename);
  3967. filename = tail;
  3968. }
  3969. }
  3970. }
  3971. else { // windows can guess anonymous share
  3972. StringBuffer sharestr;
  3973. if ((filename[1]==':')&&(filename[2]=='\\')) { // this should be always true if true full windows path
  3974. localhead.set(filename,2);
  3975. sharestr.append('\\').append((char)tolower(filename[0])).append(getShareChar());
  3976. filename += 2;
  3977. }
  3978. else if (filename[0]=='\\') {
  3979. const char *tail = strchr(filename+1,'\\');
  3980. if (tail) {
  3981. sharestr.append(tail-filename,filename);
  3982. localhead.set(filename,tail-filename);
  3983. filename = tail;
  3984. }
  3985. else {
  3986. localhead.set("c:");
  3987. sharestr.append("\\c").append(getShareChar());
  3988. }
  3989. }
  3990. sharehead.set(sharestr);
  3991. }
  3992. tailpath.set(filename);
  3993. }
  3994. else
  3995. tailpath.clear();
  3996. #ifdef _TRACERFN
  3997. StringBuffer eps;
  3998. PROGLOG("setPath (%s,%s) -> '%s' '%s' '%s'",ep.getUrlStr(eps).str(),_filename?_filename:"NULL",sharehead.get()?sharehead.get():"NULL",localhead.get()?localhead.get():"NULL",tailpath.get()?tailpath.get():"NULL");
  3999. #endif
  4000. }
  4001. void RemoteFilename::setRemotePath(const char * _url,const char *localpath)
  4002. {
  4003. // url should be full (share) path including ep
  4004. const char *url=_url;
  4005. // if ep not included then assume local (bit weird though)
  4006. char sep = url?url[0]:0;
  4007. if (isPathSepChar(sep)&&(sep==url[1])) {
  4008. url+=2;
  4009. const char *end = findPathSepChar(url);
  4010. if (end) {
  4011. StringBuffer eps(end-url,url);
  4012. ep.set(eps.str());
  4013. url = end;
  4014. }
  4015. else {
  4016. ep.set(NULL,0);
  4017. url--; // don't know what is going on!
  4018. }
  4019. if (isPathSepChar(*url)||!*url)
  4020. sep = *url;
  4021. }
  4022. else
  4023. ep.setLocalHost(0);
  4024. if (localpath&&*localpath)
  4025. setPath(ep,localpath);
  4026. else {
  4027. localhead.clear();
  4028. sharehead.clear();
  4029. tailpath.clear();
  4030. }
  4031. if (sep&&*url) {
  4032. // url should point to the share now
  4033. const char *tail=findPathSepChar(url+1);
  4034. if (tail) { // hopefully must be!
  4035. sharehead.set(url,tail-url);
  4036. url = tail;
  4037. }
  4038. if (localhead.length()==0) { // we don't know so guess
  4039. if (sharehead.length()!=0) {
  4040. const char *share=sharehead.get();
  4041. if (sep=='\\') {
  4042. StringBuffer locstr;
  4043. if (sharehead.length()>2) {
  4044. if (isShareChar(share[2])) {
  4045. locstr.append(share[1]).append(':');
  4046. share+=3;
  4047. }
  4048. else // we haven't a clue!
  4049. locstr.append("c:");
  4050. }
  4051. else if (sharehead[1]!='$') // data format
  4052. locstr.append("c:");
  4053. locstr.append(share);
  4054. localhead.set(locstr);
  4055. }
  4056. else { // we haven't a clue so assume same as share
  4057. localhead.set(sharehead);
  4058. }
  4059. }
  4060. }
  4061. }
  4062. if (tailpath.length()==0)
  4063. tailpath.set(url);
  4064. #ifdef _TRACERFN
  4065. StringBuffer eps;
  4066. PROGLOG("setRemotePath ('%s','%s') -> '%s' '%s' '%s'",_url,localpath?localpath:"NULL",sharehead.get()?sharehead.get():"NULL",localhead.get()?localhead.get():"NULL",tailpath.get()?tailpath.get():"NULL");
  4067. #endif
  4068. }
  4069. void RemoteFilename::setLocalPath(const char *filename)
  4070. {
  4071. SocketEndpoint localEP;
  4072. localEP.setLocalHost(0);
  4073. setPath(localEP, filename);
  4074. }
  4075. void RemoteFilename::split(StringBuffer * drive, StringBuffer * path, StringBuffer * tail, StringBuffer * ext) const
  4076. {
  4077. StringBuffer localname;
  4078. const char *s = getLocalPath(localname).str();
  4079. char sep;
  4080. if (isUnixPath())
  4081. sep = '/';
  4082. else {
  4083. sep = '\\';
  4084. if (s[1]==':') {
  4085. if (drive)
  4086. drive->append(2,s);
  4087. s+=2;
  4088. }
  4089. }
  4090. const char *t=NULL;
  4091. const char *e=s;
  4092. while (*e) {
  4093. if (*e==sep)
  4094. t = e+1;
  4095. e++;
  4096. }
  4097. if (t) {
  4098. if (path)
  4099. path->append(t-s, s);
  4100. s=t;
  4101. }
  4102. if (!tail&&!ext)
  4103. return;
  4104. t = NULL;
  4105. e=s;
  4106. while (*e) {
  4107. if (*e=='.') // NB final extension
  4108. t = e;
  4109. e++;
  4110. }
  4111. if (t) {
  4112. if (tail)
  4113. tail->append(t-s,s);
  4114. if (ext)
  4115. ext->append(e-t,t);
  4116. }
  4117. else if (tail)
  4118. tail->append(e-s,s);
  4119. }
  4120. void RemoteMultiFilename::append(const char *mpath,const char *defaultdir)
  4121. {
  4122. StringArray paths;
  4123. expand(mpath, paths);
  4124. StringBuffer fullpath;
  4125. StringBuffer deftmp;
  4126. if (paths.ordinality()&&(defaultdir==NULL)&&isAbsolutePath(paths.item(0))) {
  4127. splitDirTail(paths.item(0),deftmp);
  4128. defaultdir = deftmp.str();
  4129. }
  4130. ForEachItemIn(i,paths) {
  4131. const char *filename = paths.item(i);
  4132. RemoteFilename rfn;
  4133. if (isPathSepChar(*filename)&&isPathSepChar(filename[1])) // full URL
  4134. rfn.setRemotePath(filename);
  4135. else {
  4136. if (defaultdir&&!isAbsolutePath(filename))
  4137. filename = addPathSepChar(fullpath.clear().append(defaultdir)).append(filename).str();
  4138. rfn.setPath(ep,filename);
  4139. }
  4140. append(rfn);
  4141. }
  4142. }
  4143. void RemoteMultiFilename::append(const RemoteFilename &inrfn)
  4144. {
  4145. RemoteFilename rfn;
  4146. rfn.set(inrfn);
  4147. if (ep.isNull())
  4148. ep = rfn.queryEndpoint();
  4149. else if (rfn.queryIP().isNull()) {
  4150. rfn.setEp(ep);
  4151. }
  4152. else if (!rfn.queryIP().ipequals(ep)) {
  4153. StringBuffer path;
  4154. rfn.getRemotePath(path);
  4155. throw MakeStringException(-1, "Component file IP does not match: %s", path.str());
  4156. }
  4157. RemoteFilenameArray::append(rfn);
  4158. }
  4159. void RemoteMultiFilename::deserialize(MemoryBuffer & in)
  4160. {
  4161. clear();
  4162. ep.deserialize(in);
  4163. unsigned n;
  4164. in.read(n);
  4165. StringBuffer last;
  4166. StringBuffer filename;
  4167. RemoteFilename rfn;
  4168. for (unsigned i=0;i<n;i++) {
  4169. byte l;
  4170. in.read(l);
  4171. filename.clear();
  4172. if (l)
  4173. filename.append((size32_t)l,last.str());
  4174. StringAttr s;
  4175. in.read(s);
  4176. filename.append(s);
  4177. rfn.setPath(ep,filename.str());
  4178. RemoteFilenameArray::append(rfn);
  4179. last.swapWith(filename);
  4180. }
  4181. }
  4182. void RemoteMultiFilename::serialize(MemoryBuffer & out)
  4183. {
  4184. ep.serialize(out);
  4185. unsigned n=ordinality();
  4186. out.append(n);
  4187. // do simple compression
  4188. StringBuffer last;
  4189. StringBuffer filename;
  4190. for (unsigned i=0;i<n;i++) {
  4191. item(i).getLocalPath(filename.clear());
  4192. const char *s1 = last.str();
  4193. const char *s2 = filename.str();
  4194. byte l=0;
  4195. while ((l<255)&&*s1&&*s2&&(*s1==*s2)) {
  4196. s1++;
  4197. s2++;
  4198. l++;
  4199. }
  4200. out.append(l);
  4201. out.append(s2);
  4202. last.swapWith(filename);
  4203. }
  4204. }
  4205. bool RemoteMultiFilename::isWild(unsigned idx) const
  4206. {
  4207. if (idx==(unsigned)-1) {
  4208. ForEachItem(i)
  4209. if (isWild(i))
  4210. return true;
  4211. }
  4212. else {
  4213. StringBuffer s;
  4214. item(idx).getTail(s);
  4215. if (containsFileWildcard(s.str()))
  4216. return true;
  4217. }
  4218. return false;
  4219. }
  4220. void RemoteMultiFilename::expandWild()
  4221. {
  4222. bool anywild = false;
  4223. BoolArray iswild;
  4224. ForEachItem(i1) {
  4225. if (isWild(i1)) {
  4226. anywild = true;
  4227. iswild.append(true);
  4228. }
  4229. else
  4230. iswild.append(false);
  4231. }
  4232. if (!anywild)
  4233. return; // nothing to do
  4234. // first cache old values (bit long winded but want to preserve order)
  4235. RemoteFilenameArray tmpa;
  4236. Int64Array tmpsz;
  4237. ForEachItem(i2) {
  4238. RemoteFilename rfn(item(i2));
  4239. tmpa.append(rfn);
  4240. if (i2<sizescache.ordinality())
  4241. tmpsz.append(sizescache.item(i2));
  4242. else
  4243. tmpsz.append(-1);
  4244. }
  4245. RemoteFilenameArray::kill();
  4246. sizescache.kill();
  4247. ForEachItemIn(i3,tmpa) {
  4248. RemoteFilename rfn(tmpa.item(i3));
  4249. if (iswild.item(i3)) {
  4250. StringBuffer name;
  4251. rfn.getLocalPath(name);
  4252. const char *s=name.str();
  4253. const char *t=s;
  4254. loop {
  4255. const char *sep=findPathSepChar(t);
  4256. if (!sep)
  4257. break;
  4258. t = sep+1;
  4259. }
  4260. StringAttr tail(t);
  4261. name.setLength(t-s);
  4262. rfn.setPath(rfn.queryEndpoint(),name);
  4263. Owned<IFile> dir = createIFile(rfn);
  4264. Owned<IDirectoryIterator> iter = dir->directoryFiles(tail.get());
  4265. ForEach(*iter) {
  4266. append(iter->query().queryFilename());
  4267. sizescache.append(iter->getFileSize());
  4268. }
  4269. }
  4270. else {
  4271. append(rfn);
  4272. sizescache.append(tmpsz.item(i3));
  4273. }
  4274. }
  4275. }
  4276. offset_t RemoteMultiFilename::getSize(unsigned i)
  4277. {
  4278. __int64 ret = (i<sizescache.ordinality())?sizescache.item(i):-1;
  4279. if (ret==-1) {
  4280. RemoteFilename rfn(item(i));
  4281. Owned<IFile> file = createIFile(rfn);
  4282. ret = file->size();
  4283. }
  4284. return (offset_t)ret;
  4285. }
  4286. void RemoteMultiFilename::setIp(const IpAddress & ip)
  4287. {
  4288. ep.ipset(ip);
  4289. ForEachItem(i)
  4290. element(i).setIp(ip);
  4291. }
  4292. void RemoteMultiFilename::setEp(const SocketEndpoint & _ep)
  4293. {
  4294. ep.set(_ep);
  4295. ForEachItem(i)
  4296. element(i).setEp(_ep);
  4297. }
  4298. void RemoteMultiFilename::setPort(unsigned short port)
  4299. {
  4300. ep.port = port;
  4301. ForEachItem(i)
  4302. element(i).setPort(port);
  4303. }
  4304. void RemoteMultiFilename::set(const RemoteMultiFilename & other)
  4305. {
  4306. clear();
  4307. ep.set(other.ep);
  4308. ForEachItemIn(i,other) {
  4309. append(other.item(i));
  4310. if (i<other.sizescache.ordinality())
  4311. sizescache.append(other.sizescache.item(i));
  4312. }
  4313. }
  4314. bool RemoteMultiFilename::equals(const RemoteMultiFilename & other)
  4315. {
  4316. if (!ep.equals(other.ep))
  4317. return false;
  4318. if (ordinality()!=other.ordinality())
  4319. return false;
  4320. ForEachItem(i)
  4321. if (!item(i).equals(other.item(i)))
  4322. return false;
  4323. return true;
  4324. }
  4325. void RemoteMultiFilename::expand(const char *mpath, StringArray &array)
  4326. {
  4327. StringBuffer path;
  4328. StringBuffer fullpath;
  4329. loop {
  4330. while (isspace(*mpath))
  4331. mpath++;
  4332. if (!*mpath)
  4333. break;
  4334. bool inquote=false;
  4335. while (*mpath) {
  4336. if (*mpath=='"') {
  4337. mpath++;
  4338. if (inquote) {
  4339. if (*mpath!='"') {
  4340. inquote = false;
  4341. continue;
  4342. }
  4343. }
  4344. else {
  4345. inquote = true;
  4346. continue;
  4347. }
  4348. }
  4349. if ((*mpath==',')&&!inquote) {
  4350. mpath++;
  4351. break;
  4352. }
  4353. path.append(*mpath);
  4354. mpath++;
  4355. }
  4356. path.clip();
  4357. if (path.length()) {
  4358. array.append(path.str());
  4359. path.clear();
  4360. }
  4361. }
  4362. }
  4363. void RemoteMultiFilename::tostr(StringArray &array,StringBuffer &out)
  4364. {
  4365. ForEachItemIn(i,array) {
  4366. const char *s = array.item(i);
  4367. if (!s||!*s)
  4368. continue;
  4369. if (i!=0)
  4370. out.append(',');
  4371. bool needquote=false;
  4372. for (const char *e=s;*e;e++)
  4373. if (isspace(*e)||(*e==',')) {
  4374. needquote = true;
  4375. break;
  4376. }
  4377. if (needquote)
  4378. out.append('"');
  4379. out.append(s);
  4380. if (needquote)
  4381. out.append('"');
  4382. }
  4383. }
  4384. //===================================================================================================
  4385. static IArrayOf<IContainedFileHook> containedFileHooks;
  4386. static ReadWriteLock containedFileHookLock;
  4387. void addContainedFileHook(IContainedFileHook *hook)
  4388. {
  4389. if (hook)
  4390. {
  4391. hook->Link();
  4392. WriteLockBlock block(containedFileHookLock);
  4393. containedFileHooks.append(*hook);
  4394. }
  4395. }
  4396. void removeContainedFileHook(IContainedFileHook *hook)
  4397. {
  4398. WriteLockBlock block(containedFileHookLock);
  4399. containedFileHooks.zap(*hook);
  4400. }
  4401. static IFile *createContainedIFileByHook(const char *filename)
  4402. {
  4403. ReadLockBlock block(containedFileHookLock);
  4404. ForEachItemIn(i, containedFileHooks)
  4405. {
  4406. IFile * ret = containedFileHooks.item(i).createIFile(filename);
  4407. if (ret)
  4408. return ret;
  4409. }
  4410. return NULL;
  4411. }
  4412. static IArrayOf<IRemoteFileCreateHook> remoteFileHooks;
  4413. static ReadWriteLock remoteFileHookLock;
  4414. void addIFileCreateHook(IRemoteFileCreateHook *hook)
  4415. {
  4416. if (hook)
  4417. {
  4418. hook->Link();
  4419. WriteLockBlock block(remoteFileHookLock);
  4420. remoteFileHooks.append(*hook);
  4421. }
  4422. }
  4423. void removeIFileCreateHook(IRemoteFileCreateHook *hook)
  4424. {
  4425. WriteLockBlock block(remoteFileHookLock);
  4426. remoteFileHooks.zap(*hook);
  4427. }
  4428. static IFile *createIFileByHook(const RemoteFilename & filename)
  4429. {
  4430. ReadLockBlock block(remoteFileHookLock);
  4431. ForEachItemIn(i, remoteFileHooks)
  4432. {
  4433. IFile * ret = remoteFileHooks.item(i).createIFile(filename);
  4434. if (ret)
  4435. return ret;
  4436. }
  4437. return NULL;
  4438. }
  4439. IFile * createIFile(const RemoteFilename & filename)
  4440. {
  4441. IFile * ret = createIFileByHook(filename);
  4442. if (ret)
  4443. return ret;
  4444. StringBuffer name;
  4445. return createIFile(getLocalOrRemoteName(name,filename).str());
  4446. }
  4447. StringBuffer &makePathUniversal(const char *path, StringBuffer &out)
  4448. {
  4449. if (!path||!*path)
  4450. return out;
  4451. if (path[1]==':')
  4452. {
  4453. out.append('/').append(*path);
  4454. path+=2;
  4455. }
  4456. for (; *path; path++)
  4457. out.append(isPathSepChar(*path) ? '/' : *path);
  4458. return out;
  4459. }
  4460. StringBuffer &makeAbsolutePath(const char *relpath,StringBuffer &out, bool mustExist)
  4461. {
  4462. if (isPathSepChar(relpath[0])&&(relpath[0]==relpath[1]))
  4463. {
  4464. if (mustExist)
  4465. {
  4466. OwnedIFile iFile = createIFile(relpath);
  4467. if (!iFile->exists())
  4468. throw makeStringExceptionV(-1, "makeAbsolutePath: could not resolve absolute path for %s", relpath);
  4469. }
  4470. return out.append(relpath); // if remote then already should be absolute
  4471. }
  4472. #ifdef _WIN32
  4473. char rPath[MAX_PATH];
  4474. char *filepart;
  4475. if (!relpath || '\0' == *relpath)
  4476. relpath = ".";
  4477. DWORD res = GetFullPathName(relpath, sizeof(rPath), rPath, &filepart);
  4478. if (0 == res)
  4479. throw makeOsExceptionV(GetLastError(), "makeAbsolutePath: could not resolve absolute path for %s", relpath);
  4480. else if (mustExist)
  4481. {
  4482. OwnedIFile iFile = createIFile(rPath);
  4483. if (!iFile->exists())
  4484. throw makeStringExceptionV(-1, "makeAbsolutePath: could not resolve absolute path for %s", rPath);
  4485. }
  4486. out.append(rPath);
  4487. #else
  4488. char rPath[PATH_MAX];
  4489. if (mustExist)
  4490. {
  4491. if (!realpath(relpath, rPath))
  4492. throw makeErrnoExceptionV(errno, "makeAbsolutePath: could not resolve absolute path for %s", relpath);
  4493. out.append(rPath);
  4494. }
  4495. else
  4496. {
  4497. // no error, will attempt to resolve(realpath) as much of relpath as possible and append rest
  4498. if (strlen(relpath))
  4499. {
  4500. const char *end = relpath+strlen(relpath);
  4501. const char *path = relpath;
  4502. const char *tail = end;
  4503. StringBuffer head;
  4504. loop
  4505. {
  4506. if (realpath(path,rPath))
  4507. {
  4508. out.append(rPath);
  4509. if (tail != end)
  4510. out.append(tail);
  4511. return removeTrailingPathSepChar(out);
  4512. }
  4513. // mark next tail
  4514. loop
  4515. {
  4516. --tail;
  4517. if (tail == relpath)
  4518. break;
  4519. else if ('/' == *tail)
  4520. break;
  4521. }
  4522. if (tail == relpath)
  4523. break; // bail out and guess
  4524. head.clear().append(tail-relpath, relpath);
  4525. path = head.str();
  4526. }
  4527. }
  4528. if (isAbsolutePath(relpath))
  4529. out.append(relpath);
  4530. else
  4531. {
  4532. appendCurrentDirectory(out, true);
  4533. if (strlen(relpath))
  4534. addPathSepChar(out).append(relpath);
  4535. }
  4536. }
  4537. #endif
  4538. return removeTrailingPathSepChar(out);
  4539. }
  4540. StringBuffer &makeAbsolutePath(StringBuffer &relpath,bool mustExist)
  4541. {
  4542. StringBuffer out;
  4543. makeAbsolutePath(relpath.str(),out,mustExist);
  4544. relpath.swapWith(out);
  4545. return relpath;
  4546. }
  4547. StringBuffer &makeAbsolutePath(const char *relpath, const char *basedir, StringBuffer &out)
  4548. {
  4549. StringBuffer combined;
  4550. if (basedir && !isAbsolutePath(relpath))
  4551. relpath = combined.append(basedir).append(relpath);
  4552. return makeAbsolutePath(relpath, out);
  4553. }
  4554. const char *splitRelativePath(const char *full,const char *basedir,StringBuffer &reldir)
  4555. {
  4556. if (basedir&&*basedir) {
  4557. size32_t bl = strlen(basedir);
  4558. if (isPathSepChar(basedir[bl-1]))
  4559. bl--;
  4560. if ((memicmp(full,basedir,bl)==0)&&isPathSepChar(full[bl]))
  4561. full += bl+1;
  4562. }
  4563. const char *t = full;
  4564. loop {
  4565. const char *n = findPathSepChar(t);
  4566. if (!n)
  4567. break;
  4568. t = n+1;
  4569. }
  4570. if (t!=full)
  4571. reldir.append(t-full,full);
  4572. return t;
  4573. }
  4574. const char *splitDirMultiTail(const char *multipath,StringBuffer &dir,StringBuffer &tail)
  4575. {
  4576. // the first directory is the significant one
  4577. // others only removed if same
  4578. dir.clear();
  4579. StringArray files;
  4580. RemoteMultiFilename::expand(multipath,files);
  4581. StringBuffer reldir;
  4582. ForEachItemIn(i,files) {
  4583. const char *s = files.item(i);
  4584. if (i==0) {
  4585. if (isAbsolutePath(s)) {
  4586. StringAttr tail(splitDirTail(s,dir));
  4587. if (dir.length())
  4588. files.replace(tail,i);
  4589. }
  4590. }
  4591. else if (dir.length()) {
  4592. s= splitRelativePath(s,dir.str(),reldir.clear());
  4593. if (reldir.length()) {
  4594. reldir.append(s);
  4595. files.replace(reldir.str(),i);
  4596. }
  4597. }
  4598. }
  4599. RemoteMultiFilename::tostr(files,tail);
  4600. return tail.str();
  4601. }
  4602. StringBuffer &mergeDirMultiTail(const char *dir,const char *tail, StringBuffer &multipath)
  4603. {
  4604. StringArray files;
  4605. RemoteMultiFilename::expand(tail,files);
  4606. StringBuffer reldir;
  4607. if (dir && *dir) {
  4608. ForEachItemIn(i,files) {
  4609. const char *s = files.item(i);
  4610. if (!isAbsolutePath(s)) {
  4611. reldir.clear().append(dir);
  4612. addPathSepChar(reldir).append(s);
  4613. files.replace(reldir.str(),i);
  4614. }
  4615. }
  4616. }
  4617. RemoteMultiFilename::tostr(files,multipath);
  4618. return multipath;
  4619. }
  4620. StringBuffer &removeRelativeMultiPath(const char *full,const char *reldir,StringBuffer &res)
  4621. {
  4622. StringArray files;
  4623. RemoteMultiFilename::expand(full,files);
  4624. StringBuffer tmp1;
  4625. StringBuffer tmp2;
  4626. StringBuffer dir;
  4627. ForEachItemIn(i,files) {
  4628. const char *s = files.item(i);
  4629. if (isAbsolutePath(s)) {
  4630. if (!dir.length())
  4631. splitDirTail(s,dir);
  4632. }
  4633. else if (dir.length()) {
  4634. tmp1.clear().append(dir);
  4635. addPathSepChar(tmp1).append(s);
  4636. s = tmp1.str();
  4637. }
  4638. s = splitRelativePath(s,reldir,tmp2.clear());
  4639. tmp2.append(s);
  4640. files.replace(tmp2.str(),i);
  4641. }
  4642. RemoteMultiFilename::tostr(files,res);
  4643. return res;
  4644. }
  4645. // removes basedir if matches, returns relative multipath
  4646. //===================================================================================================
  4647. ExtractedBlobInfo::ExtractedBlobInfo(const char * _filename, offset_t _length, offset_t _offset)
  4648. {
  4649. filename.set(_filename);
  4650. length = _length;
  4651. offset = _offset;
  4652. }
  4653. void ExtractedBlobInfo::serialize(MemoryBuffer & buffer)
  4654. {
  4655. ::serialize(buffer, filename.get());
  4656. buffer.append(length).append(offset);
  4657. }
  4658. void ExtractedBlobInfo::deserialize(MemoryBuffer & buffer)
  4659. {
  4660. ::deserialize(buffer, filename);
  4661. buffer.read(length).read(offset);
  4662. }
  4663. //----------------------------------------------------------------------------
  4664. #define DFTERR_InvalidSplitPrefixFormat 8091
  4665. #define DFTERR_InvalidSplitPrefixFormat_Text "Cannot process file %s using the splitprefix supplied"
  4666. static void * readLength(MemoryBuffer & buffer, IFileIOStream * in, size_t len, const char * filenameText)
  4667. {
  4668. void * ptr = buffer.clear().reserve(len);
  4669. if (in->read(len, ptr) != len)
  4670. throwError1(DFTERR_InvalidSplitPrefixFormat, filenameText);
  4671. return ptr;
  4672. }
  4673. void extractBlobElements(const char * prefix, const RemoteFilename &filename, ExtractedBlobArray & extracted)
  4674. {
  4675. StringBuffer filenameText;
  4676. filename.getPath(filenameText);
  4677. Owned<IFile> file = createIFile(filename);
  4678. Owned<IFileIO> inIO = file->open(IFOread);
  4679. if (!inIO)
  4680. throw MakeStringException(-1, "extractBlobElements: file '%s' not found", filenameText.str());
  4681. Owned<IFileIOStream> in = createIOStream(inIO);
  4682. MemoryBuffer buffer;
  4683. offset_t endOffset = in->size();
  4684. while (in->tell() != endOffset)
  4685. {
  4686. StringAttr blobFilename;
  4687. offset_t blobLength = (offset_t)-1;
  4688. const char * finger = prefix;
  4689. while (finger)
  4690. {
  4691. StringAttr command;
  4692. const char * comma = strchr(finger, ',');
  4693. if (comma)
  4694. {
  4695. command.set(finger, comma-finger);
  4696. finger = comma+1;
  4697. }
  4698. else
  4699. {
  4700. command.set(finger);
  4701. finger = NULL;
  4702. }
  4703. command.toUpperCase();
  4704. if (memcmp(command, "FILENAME", 8) == 0)
  4705. {
  4706. if (command[8] == ':')
  4707. {
  4708. unsigned maxLen = atoi(command+9);
  4709. const char * nameptr = (const char *)readLength(buffer, in, maxLen, filenameText.str());
  4710. blobFilename.set(nameptr, maxLen);
  4711. }
  4712. else
  4713. {
  4714. unsigned * lenptr = (unsigned *)readLength(buffer, in, sizeof(unsigned), filenameText.str());
  4715. #if __BYTE_ORDER != __LITTLE_ENDIAN
  4716. _rev(sizeof(*lenptr), lenptr);
  4717. #endif
  4718. unsigned filenamelen = *lenptr;
  4719. const char * nameptr = (const char *)readLength(buffer, in, filenamelen, filenameText.str());
  4720. blobFilename.set(nameptr, filenamelen);
  4721. }
  4722. }
  4723. else if ((memcmp(command, "FILESIZE", 8) == 0) || (command.length() == 2))
  4724. {
  4725. const char * format = command;
  4726. if (memcmp(format, "FILESIZE", 8) == 0)
  4727. {
  4728. if (format[8] == ':')
  4729. format = format+9;
  4730. else
  4731. format = "L4";
  4732. }
  4733. bool bigEndian;
  4734. char c = format[0];
  4735. if (c == 'B')
  4736. bigEndian = true;
  4737. else if (c == 'L')
  4738. bigEndian = false;
  4739. else
  4740. throwError1(DFTERR_InvalidSplitPrefixFormat, format);
  4741. c = format[1];
  4742. if ((c <= '0') || (c > '8'))
  4743. throwError1(DFTERR_InvalidSplitPrefixFormat, format);
  4744. unsigned length = (c - '0');
  4745. byte * lenptr = (byte *)readLength(buffer, in, length, filenameText.str());
  4746. if (!bigEndian)
  4747. _rev(length, lenptr);
  4748. blobLength = 0;
  4749. for (unsigned i=0; i<length; i++)
  4750. {
  4751. blobLength = (blobLength << 8) | lenptr[i];
  4752. }
  4753. }
  4754. else if (memcmp(command, "SKIP:", 5) == 0)
  4755. {
  4756. unsigned skipLen = atoi(command+5);
  4757. in->seek(in->tell()+skipLen, IFSbegin);
  4758. }
  4759. else if (memcmp(command, "SEQ:", 4) == 0)
  4760. {
  4761. unsigned skipLen = atoi(command+4);
  4762. in->seek(in->tell()+skipLen, IFSbegin);
  4763. }
  4764. else
  4765. throwError1(DFTERR_InvalidSplitPrefixFormat, command.get());
  4766. }
  4767. if ((blobLength == (offset_t)-1) || !blobFilename.get())
  4768. throwError1(DFTERR_InvalidSplitPrefixFormat, filenameText.str());
  4769. offset_t blobOffset = in->tell();
  4770. extracted.append(* new ExtractedBlobInfo(blobFilename, blobLength, blobOffset));
  4771. in->seek(blobOffset + blobLength, IFSbegin);
  4772. }
  4773. }
  4774. bool mountDrive(const char *drv,const RemoteFilename &rfn)
  4775. {
  4776. #ifdef _WIN32
  4777. return false;
  4778. #else
  4779. unmountDrive(drv);
  4780. localCreateDirectory(drv);
  4781. int ret;
  4782. for (unsigned vtry=0;vtry<2;vtry++) {
  4783. StringBuffer cmd;
  4784. cmd.append("mount ");
  4785. rfn.queryIP().getIpText(cmd);
  4786. cmd.append(':');
  4787. rfn.getLocalPath(cmd);
  4788. cmd.append(' ').append(drv).append(" -t nfs ");
  4789. if (vtry==0)
  4790. cmd.append("-o nfsvers=v3 "); // prefer v3
  4791. cmd.append("2> /dev/null");
  4792. ret = system(cmd.str());
  4793. if (ret==0)
  4794. break;
  4795. }
  4796. return (ret==0);
  4797. #endif
  4798. }
  4799. bool unmountDrive(const char *drv)
  4800. {
  4801. #ifdef _WIN32
  4802. return false;
  4803. #else
  4804. StringBuffer cmd;
  4805. cmd.append("umount ").append(drv).append(" 2> /dev/null");
  4806. int ret = system(cmd.str());
  4807. return (ret==0);
  4808. #endif
  4809. }
  4810. IFileIO *createUniqueFile(const char *dir, const char *prefix, const char *ext, StringBuffer &filename)
  4811. {
  4812. CDateTime dt;
  4813. dt.setNow();
  4814. unsigned t = (unsigned)dt.getSimple();
  4815. if (dir)
  4816. {
  4817. filename.append(dir);
  4818. addPathSepChar(filename);
  4819. }
  4820. if (prefix && *prefix)
  4821. filename.append(prefix);
  4822. else
  4823. filename.append("uniq");
  4824. if (!ext || !*ext)
  4825. ext = "tmp";
  4826. filename.appendf("_%" I64F "x.%x.%x.%s", (__int64)GetCurrentThreadId(), (unsigned)GetCurrentProcessId(), t, ext);
  4827. OwnedIFile iFile = createIFile(filename.str());
  4828. IFileIO *iFileIO = NULL;
  4829. unsigned attempts = 5; // max attempts
  4830. loop
  4831. {
  4832. if (!iFile->exists())
  4833. {
  4834. try { return iFile->openShared(IFOcreate, IFSHnone); } // NB: could be null if path not found
  4835. catch (IException *e)
  4836. {
  4837. EXCLOG(e, "createUniqueFile");
  4838. e->Release();
  4839. }
  4840. }
  4841. if (0 == --attempts)
  4842. break;
  4843. t += getRandom();
  4844. filename.clear().appendf("uniq_%" I64F "x.%x.%x.%s", (__int64)GetCurrentThreadId(), (unsigned)GetCurrentProcessId(), t, ext);
  4845. iFile.setown(createIFile(filename.str()));
  4846. }
  4847. return NULL;
  4848. }
  4849. unsigned sortDirectory( CIArrayOf<CDirectoryEntry> &sortedfiles,
  4850. IDirectoryIterator &iter,
  4851. SortDirectoryMode mode,
  4852. bool rev,
  4853. bool includedirs
  4854. )
  4855. {
  4856. sortedfiles.kill();
  4857. StringBuffer name;
  4858. ForEach(iter) {
  4859. if (!iter.isDir()||includedirs)
  4860. sortedfiles.append(*new CDirectoryEntry(iter));
  4861. }
  4862. if (mode!=SD_nosort) {
  4863. struct icmp: implements ICompare
  4864. {
  4865. SortDirectoryMode mode;
  4866. bool rev;
  4867. int docompare(const void *l,const void *r) const
  4868. {
  4869. int ret=0;
  4870. const CDirectoryEntry *dl = (const CDirectoryEntry *)l;
  4871. const CDirectoryEntry *dr = (const CDirectoryEntry *)r;
  4872. switch (mode) {
  4873. case SD_byname:
  4874. ret = strcmp(dl->name,dr->name);
  4875. break;
  4876. case SD_bynameNC:
  4877. ret = stricmp(dl->name,dr->name);
  4878. break;
  4879. case SD_bydate:
  4880. ret = dl->modifiedTime.compare(dr->modifiedTime);
  4881. break;
  4882. case SD_bysize:
  4883. ret = (dl->size>dr->size)?1:((dl->size<dr->size)?-1:0);
  4884. break;
  4885. }
  4886. if (rev)
  4887. ret = -ret;
  4888. return ret;
  4889. }
  4890. } cmp;
  4891. cmp.mode = mode;
  4892. cmp.rev = rev;
  4893. qsortvec((void **)sortedfiles.getArray(), sortedfiles.ordinality(), cmp);
  4894. }
  4895. return sortedfiles.ordinality();
  4896. }
  4897. class CReplicatedFile : public CInterface, implements IReplicatedFile
  4898. {
  4899. RemoteFilenameArray copies;
  4900. public:
  4901. IMPLEMENT_IINTERFACE;
  4902. RemoteFilenameArray &queryCopies()
  4903. {
  4904. return copies;
  4905. }
  4906. IFile *open()
  4907. {
  4908. StringBuffer locations;
  4909. Owned<IException> exc;
  4910. ForEachItemIn(copy,copies) {
  4911. const RemoteFilename &rfn = copies.item(copy);
  4912. try {
  4913. OwnedIFile iFile = createIFile(rfn);
  4914. if (iFile->exists())
  4915. return iFile.getClear();
  4916. if (locations.length())
  4917. locations.append(", ");
  4918. rfn.getRemotePath(locations);
  4919. }
  4920. catch(IException *e) {
  4921. EXCLOG(e,"CReplicatedFile::open");
  4922. if (exc)
  4923. e->Release();
  4924. else
  4925. exc.setown(e);
  4926. }
  4927. }
  4928. if (exc.get())
  4929. throw exc.getClear();
  4930. throw MakeStringException(0, "%s: Failed to open part file at any of the following locations: ", locations.str());
  4931. }
  4932. };
  4933. IReplicatedFile *createReplicatedFile()
  4934. {
  4935. return new CReplicatedFile;
  4936. }
  4937. // ---------------------------------------------------------------------------------
  4938. class CSerialStreamBase : public CInterface, implements ISerialStream
  4939. {
  4940. private:
  4941. size32_t bufsize;
  4942. size32_t bufpos;
  4943. size32_t bufmax;
  4944. offset_t bufbase;
  4945. offset_t endpos; // -1 if not known
  4946. MemoryAttr ma;
  4947. byte *buf;
  4948. bool eoinput;
  4949. IFileSerialStreamCallback *tally;
  4950. inline size32_t doread(offset_t pos, size32_t max_size, void *ptr)
  4951. {
  4952. if (endpos!=(offset_t)-1) {
  4953. if (pos>=endpos)
  4954. return 0;
  4955. if (endpos-pos<max_size)
  4956. max_size = (size32_t)(endpos-pos);
  4957. }
  4958. size32_t size_read = rawread(pos, max_size, ptr);
  4959. if (tally)
  4960. tally->process(pos,size_read,ptr);
  4961. return size_read;
  4962. }
  4963. const void * dopeek(size32_t sz, size32_t &got)
  4964. {
  4965. loop
  4966. {
  4967. size32_t left = bufmax-bufpos;
  4968. got = left;
  4969. if (left>=sz)
  4970. return buf+bufpos;
  4971. if (eoinput)
  4972. return left?(buf+bufpos):NULL;
  4973. size32_t reqsz = sz+bufsize; // NB not sz-left as want some slack
  4974. if (ma.length()<reqsz) {
  4975. MemoryAttr ma2;
  4976. void *nb = ma2.allocate(reqsz);
  4977. memcpy(nb,buf+bufpos,left);
  4978. ma.setOwn(reqsz,ma2.detach());
  4979. buf = (byte *)nb;
  4980. }
  4981. else
  4982. memmove(buf,buf+bufpos,left);
  4983. bufbase += bufpos;
  4984. size32_t rd = doread(bufbase+left,bufsize,buf+left);
  4985. if (!rd)
  4986. eoinput = true;
  4987. bufmax = rd+left;
  4988. bufpos = 0;
  4989. }
  4990. }
  4991. protected:
  4992. virtual size32_t rawread(offset_t pos, size32_t max_size, void *ptr) = 0;
  4993. public:
  4994. IMPLEMENT_IINTERFACE;
  4995. CSerialStreamBase(offset_t _offset, offset_t _len, size32_t _bufsize, IFileSerialStreamCallback *_tally)
  4996. {
  4997. tally = _tally;
  4998. bufsize = _bufsize;
  4999. if (bufsize==(size32_t)-1)
  5000. bufsize = DEFAULT_STREAM_BUFFER_SIZE;
  5001. if (bufsize<4096)
  5002. bufsize = 4096;
  5003. else
  5004. bufsize = ((bufsize+4095)/4096)*4096;
  5005. buf = (byte *)ma.allocate(bufsize+4096); // 4K initial slack
  5006. bufpos = 0;
  5007. bufmax = 0;
  5008. bufbase = _offset;
  5009. if (_len==(offset_t)-1)
  5010. endpos = (offset_t)-1;
  5011. else
  5012. endpos = _offset+_len;
  5013. eoinput = (_len==0);
  5014. }
  5015. void reset(offset_t _offset, offset_t _len)
  5016. {
  5017. bufpos = 0;
  5018. bufmax = 0;
  5019. bufbase = _offset;
  5020. if (_len==(offset_t)-1)
  5021. endpos = (offset_t)-1;
  5022. else
  5023. endpos = _offset+_len;
  5024. eoinput = (_len==0);
  5025. }
  5026. const void *peek(size32_t sz,size32_t &got)
  5027. {
  5028. return dopeek(sz, got);
  5029. }
  5030. void get(size32_t len, void * ptr)
  5031. {
  5032. size32_t cpy = bufmax-bufpos;
  5033. if (cpy>len)
  5034. cpy = len;
  5035. memcpy(ptr,(const byte *)buf+bufpos,cpy);
  5036. len -= cpy;
  5037. if (len==0) {
  5038. bufpos += cpy;
  5039. return;
  5040. }
  5041. bufbase += bufmax;
  5042. bufpos = 0;
  5043. bufmax = 0;
  5044. size32_t rd = 0;
  5045. if (!eoinput) {
  5046. ptr = (byte *)ptr+cpy;
  5047. if (len>=bufsize) {
  5048. size32_t tord = (len/bufsize)*bufsize;
  5049. rd = doread(bufbase,tord,ptr); // copy directly if large
  5050. bufbase += rd;
  5051. if (rd!=tord) {
  5052. eoinput = true;
  5053. PrintStackReport();
  5054. ERRLOG("CFileSerialStream::get read past end of stream.1 (%u,%u) %s",rd,tord,eoinput?"eoinput":"");
  5055. throw MakeStringException(-1,"CFileSerialStream::get read past end of stream");
  5056. }
  5057. cpy += rd;
  5058. len -= rd;
  5059. if (!len)
  5060. return;
  5061. ptr = (byte *)ptr+rd;
  5062. }
  5063. const void *p = dopeek(len,rd);
  5064. if (len<=rd) {
  5065. memcpy(ptr,p,len);
  5066. bufpos += len;
  5067. return;
  5068. }
  5069. }
  5070. PrintStackReport();
  5071. ERRLOG("CFileSerialStream::get read past end of stream.2 (%u,%u) %s",len,rd,eoinput?"eoinput":"");
  5072. throw MakeStringException(-1,"CFileSerialStream::get read past end of stream");
  5073. }
  5074. bool eos()
  5075. {
  5076. if (bufmax-bufpos)
  5077. return false;
  5078. size32_t rd;
  5079. return dopeek(1,rd)==NULL;
  5080. }
  5081. void skip(size32_t len)
  5082. {
  5083. size32_t left = bufmax-bufpos;
  5084. if (left>=len) {
  5085. bufpos += len;
  5086. return;
  5087. }
  5088. len -= left;
  5089. bufbase += bufmax;
  5090. bufpos = 0;
  5091. bufmax = 0;
  5092. if (!eoinput) {
  5093. while (len>=bufsize) {
  5094. size32_t rd;
  5095. if (tally) {
  5096. rd = doread(bufbase,bufsize,buf);
  5097. if (rd!=bufsize) {
  5098. eoinput = true;
  5099. throw MakeStringException(-1,"CFileSerialStream::skip read past end of stream");
  5100. }
  5101. }
  5102. else {
  5103. rd = (len/bufsize)*bufsize;
  5104. //rd=doskip(bufbase,bufsize,buf); to cope with reading from sockets etc?
  5105. }
  5106. bufbase += rd;
  5107. len -= bufsize;
  5108. }
  5109. if (len==0)
  5110. return;
  5111. size32_t got;
  5112. dopeek(len,got);
  5113. if (len<=got) {
  5114. bufpos += got;
  5115. return;
  5116. }
  5117. }
  5118. throw MakeStringException(-1,"CFileSerialStream::skip read past end of stream");
  5119. }
  5120. offset_t tell()
  5121. {
  5122. return bufbase+bufpos;
  5123. }
  5124. };
  5125. class CFileSerialStream: public CSerialStreamBase
  5126. {
  5127. Linked<IFileIO> fileio;
  5128. virtual size32_t rawread(offset_t pos, size32_t max_size, void *ptr)
  5129. {
  5130. return fileio->read(pos,max_size,ptr);
  5131. }
  5132. public:
  5133. CFileSerialStream(IFileIO *_fileio,offset_t _offset, offset_t _len, size32_t _bufsize, IFileSerialStreamCallback *_tally)
  5134. : CSerialStreamBase(_offset, _len, _bufsize, _tally), fileio(_fileio)
  5135. {
  5136. }
  5137. };
  5138. ISerialStream *createFileSerialStream(IFileIO *fileio,offset_t ofs, offset_t flen, size32_t bufsize,IFileSerialStreamCallback *callback)
  5139. {
  5140. if (!fileio)
  5141. return NULL;
  5142. return new CFileSerialStream(fileio,ofs,flen,bufsize,callback);
  5143. }
  5144. class CIoSerialStream: public CSerialStreamBase
  5145. {
  5146. Linked<IFileIOStream> io;
  5147. offset_t lastpos;
  5148. virtual size32_t rawread(offset_t pos, size32_t max_size, void *ptr)
  5149. {
  5150. if (lastpos!=pos)
  5151. throw MakeStringException(-1,"CIoSerialStream: non-sequential read (%" I64F "d,%" I64F "d)",lastpos,pos);
  5152. size32_t rd = io->read(max_size,ptr);
  5153. lastpos = pos+rd;
  5154. return rd;
  5155. }
  5156. public:
  5157. CIoSerialStream(IFileIOStream * _io,offset_t _offset, size32_t _bufsize, IFileSerialStreamCallback *_tally)
  5158. : CSerialStreamBase(_offset, (offset_t)-1, _bufsize, _tally), io(_io)
  5159. {
  5160. lastpos = _offset;
  5161. }
  5162. };
  5163. ISerialStream *createFileSerialStream(IFileIOStream *io,size32_t bufsize,IFileSerialStreamCallback *callback)
  5164. {
  5165. if (!io)
  5166. return NULL;
  5167. return new CIoSerialStream(io,0,bufsize,callback);
  5168. }
  5169. class CSocketSerialStream: public CSerialStreamBase
  5170. {
  5171. Linked<ISocket> socket;
  5172. unsigned timeout;
  5173. offset_t lastpos;
  5174. virtual size32_t rawread(offset_t pos, size32_t max_size, void *ptr)
  5175. {
  5176. if (lastpos!=pos)
  5177. throw MakeStringException(-1,"CSocketSerialStream: non-sequential read (%" I64F "d,%" I64F "d)",lastpos,pos);
  5178. size32_t size_read;
  5179. socket->readtms(ptr, 0, max_size, size_read, timeout);
  5180. lastpos = pos+size_read;
  5181. return size_read;
  5182. }
  5183. public:
  5184. CSocketSerialStream(ISocket * _socket, unsigned _timeout, offset_t _offset, size32_t _bufsize, IFileSerialStreamCallback *_tally)
  5185. : CSerialStreamBase(_offset, (offset_t)-1, _bufsize, _tally), socket(_socket), timeout(_timeout)
  5186. {
  5187. lastpos = _offset;
  5188. }
  5189. };
  5190. ISerialStream *createSocketSerialStream(ISocket * socket, unsigned timeoutms, size32_t bufsize, IFileSerialStreamCallback *callback)
  5191. {
  5192. if (!socket)
  5193. return NULL;
  5194. return new CSocketSerialStream(socket,timeoutms,0,bufsize,callback);
  5195. }
  5196. class CSimpleReadSerialStream: public CSerialStreamBase
  5197. {
  5198. Linked<ISimpleReadStream> input;
  5199. offset_t lastpos;
  5200. virtual size32_t rawread(offset_t pos, size32_t max_size, void *ptr)
  5201. {
  5202. if (lastpos!=pos)
  5203. throw MakeStringException(-1,"CSimpleReadSerialStream: non-sequential read (%" I64F "d,%" I64F "d)",lastpos,pos);
  5204. size32_t rd = input->read(max_size, ptr);
  5205. lastpos += rd;
  5206. return rd;
  5207. }
  5208. public:
  5209. CSimpleReadSerialStream(ISimpleReadStream * _input, offset_t _offset, size32_t _bufsize, IFileSerialStreamCallback *_tally)
  5210. : CSerialStreamBase(_offset, (offset_t)-1, _bufsize, _tally), input(_input)
  5211. {
  5212. lastpos = _offset;
  5213. }
  5214. void reset(offset_t _ofs, offset_t _len)
  5215. {
  5216. // assume knows what doing
  5217. lastpos = _ofs;
  5218. CSerialStreamBase::reset(_ofs,_len);
  5219. }
  5220. };
  5221. ISerialStream *createSimpleSerialStream(ISimpleReadStream * input, size32_t bufsize, IFileSerialStreamCallback *callback)
  5222. {
  5223. if (!input)
  5224. return NULL;
  5225. return new CSimpleReadSerialStream(input,0,bufsize,callback);
  5226. }
  5227. class CMemoryMappedSerialStream: public CInterface, implements ISerialStream
  5228. {
  5229. Linked<IMemoryMappedFile> mmfile;
  5230. const byte *mmbase;
  5231. memsize_t mmsize;
  5232. memsize_t mmofs;
  5233. bool eoinput;
  5234. IFileSerialStreamCallback *tally;
  5235. public:
  5236. IMPLEMENT_IINTERFACE;
  5237. CMemoryMappedSerialStream(IMemoryMappedFile *_mmfile, offset_t ofs, offset_t flen, IFileSerialStreamCallback *_tally)
  5238. : mmfile(_mmfile)
  5239. {
  5240. tally = _tally;
  5241. offset_t fs = mmfile->fileSize();
  5242. if ((memsize_t)fs!=fs)
  5243. throw MakeStringException(-1,"CMemoryMappedSerialStream file too big to be mapped");
  5244. if ((flen!=(offset_t)-1)&&(fs>flen))
  5245. fs = flen;
  5246. mmsize = (memsize_t)fs;
  5247. mmofs = (memsize_t)((ofs<fs)?ofs:fs);
  5248. mmbase = (const byte *)mmfile->base();
  5249. eoinput = false;
  5250. }
  5251. void reset(offset_t _ofs, offset_t _len)
  5252. {
  5253. offset_t fs = mmfile->fileSize();
  5254. if ((_len!=(offset_t)-1)&&(fs>_len))
  5255. fs = _len;
  5256. mmsize = (memsize_t)fs;
  5257. mmofs = (memsize_t)((_ofs<fs)?_ofs:fs);
  5258. mmsize = (memsize_t)fs;
  5259. }
  5260. CMemoryMappedSerialStream(const void *buf, memsize_t len, IFileSerialStreamCallback *_tally)
  5261. {
  5262. tally = _tally;
  5263. mmsize = len;
  5264. mmofs = 0;
  5265. mmbase = (const byte *)buf;
  5266. eoinput = false;
  5267. }
  5268. const void *peek(size32_t sz,size32_t &got)
  5269. {
  5270. memsize_t left = mmsize-mmofs;
  5271. if (sz>left)
  5272. sz = (size32_t)left;
  5273. else if (left>=UINT_MAX)
  5274. got = UINT_MAX-1;
  5275. else
  5276. got = (size32_t)left;
  5277. return mmbase+mmofs;
  5278. }
  5279. void get(size32_t len, void * ptr)
  5280. {
  5281. memsize_t left = mmsize-mmofs;
  5282. if (len>left) {
  5283. PrintStackReport();
  5284. ERRLOG("CFileSerialStream::get read past end of stream.3 (%u,%u)",(unsigned)len,(unsigned)left);
  5285. throw MakeStringException(-1,"CMemoryMappedSerialStream::get read past end of stream (%u,%u)",(unsigned)len,(unsigned)left);
  5286. }
  5287. if (tally)
  5288. tally->process(mmofs,len,mmbase+mmofs);
  5289. memcpy(ptr,mmbase+mmofs,len);
  5290. mmofs += len;
  5291. }
  5292. bool eos()
  5293. {
  5294. return (mmsize<=mmofs);
  5295. }
  5296. void skip(size32_t len)
  5297. {
  5298. memsize_t left = mmsize-mmofs;
  5299. if (len>left)
  5300. throw MakeStringException(-1,"CMemoryMappedSerialStream::skip read past end of stream (%u,%u)",(unsigned)len,(unsigned)left);
  5301. if (tally)
  5302. tally->process(mmofs,len,mmbase+mmofs);
  5303. mmofs += len;
  5304. }
  5305. offset_t tell()
  5306. {
  5307. return mmofs;
  5308. }
  5309. virtual void reset(offset_t _offset)
  5310. {
  5311. mmofs = (memsize_t)((_offset<mmsize)?_offset:mmsize);
  5312. eoinput = false;
  5313. }
  5314. };
  5315. ISerialStream *createFileSerialStream(IMemoryMappedFile *mmfile, offset_t ofs, offset_t flen, IFileSerialStreamCallback *callback)
  5316. {
  5317. return new CMemoryMappedSerialStream(mmfile,ofs,flen,callback);
  5318. }
  5319. ISerialStream *createMemorySerialStream(const void *buffer, memsize_t len, IFileSerialStreamCallback *callback)
  5320. {
  5321. return new CMemoryMappedSerialStream(buffer,len,callback);
  5322. }
  5323. class CMemoryBufferSerialStream: public CInterface, implements ISerialStream
  5324. {
  5325. MemoryBuffer & buffer;
  5326. IFileSerialStreamCallback *tally;
  5327. public:
  5328. IMPLEMENT_IINTERFACE;
  5329. CMemoryBufferSerialStream(MemoryBuffer & _buffer, IFileSerialStreamCallback * _tally)
  5330. : buffer(_buffer), tally(_tally)
  5331. {
  5332. }
  5333. virtual const void *peek(size32_t sz,size32_t &got)
  5334. {
  5335. got = buffer.remaining();
  5336. return buffer.readDirect(0);
  5337. }
  5338. virtual void get(size32_t len, void * ptr)
  5339. {
  5340. if (len>buffer.remaining()) {
  5341. ERRLOG("CMemoryBufferSerialStream::get read past end of stream.4(%u,%u)",(unsigned)len,(unsigned)buffer.remaining());
  5342. throw MakeStringException(-1,"CMemoryBufferSerialStream::get read past end of stream (%u,%u)",(unsigned)len,(unsigned)buffer.remaining());
  5343. }
  5344. const void * data = buffer.readDirect(len);
  5345. if (tally)
  5346. tally->process(buffer.getPos()-len,len,data);
  5347. memcpy(ptr,data,len);
  5348. }
  5349. virtual bool eos()
  5350. {
  5351. return buffer.remaining() == 0;
  5352. }
  5353. virtual void skip(size32_t len)
  5354. {
  5355. if (len>buffer.remaining())
  5356. throw MakeStringException(-1,"CMemoryBufferSerialStream::skip read past end of stream (%u,%u)",(unsigned)len,(unsigned)buffer.remaining());
  5357. const void * data = buffer.readDirect(len);
  5358. if (tally)
  5359. tally->process(buffer.getPos()-len,len,data);
  5360. }
  5361. virtual offset_t tell()
  5362. {
  5363. return buffer.getPos();
  5364. }
  5365. virtual void reset(offset_t _offset,offset_t _len)
  5366. {
  5367. size32_t ofs = (size32_t)_offset;
  5368. assertex(ofs==_offset);
  5369. assertex((_len==(offset_t)-1)||(_len>=buffer.length())); // don't support len on memory buffer
  5370. buffer.reset(ofs);
  5371. }
  5372. };
  5373. ISerialStream *createMemoryBufferSerialStream(MemoryBuffer & buffer, IFileSerialStreamCallback *callback)
  5374. {
  5375. return new CMemoryBufferSerialStream(buffer,callback);
  5376. }
  5377. // Memory Mapped Files
  5378. #define MEMORYMAP_PAGESIZE (0x1000) // could be different but won't ever be!
  5379. class CMemoryMappedFile: public CInterface, implements IMemoryMappedFile
  5380. {
  5381. byte *ptr; // base
  5382. offset_t ofs;
  5383. offset_t realofs; // rounded down to page size
  5384. offset_t filesize;
  5385. memsize_t size;
  5386. bool writeaccess;
  5387. memsize_t windowsize;
  5388. HANDLE hfile;
  5389. #ifdef _WIN32
  5390. static size32_t pagesize;
  5391. HANDLE hmap;
  5392. #endif
  5393. Linked<IFile> file;
  5394. inline offset_t pageround(offset_t o)
  5395. {
  5396. #ifdef _WIN32
  5397. return o-(o%pagesize);
  5398. #else
  5399. return o-(o%MEMORYMAP_PAGESIZE);
  5400. #endif
  5401. }
  5402. inline void * realptr()
  5403. {
  5404. return ptr?(ptr-(ofs-realofs)):NULL;
  5405. }
  5406. inline memsize_t realsize()
  5407. {
  5408. return (memsize_t)(size+(ofs-realofs));
  5409. }
  5410. public:
  5411. IMPLEMENT_IINTERFACE;
  5412. CMemoryMappedFile(HANDLE _hfile, offset_t _filesize, offset_t _ofs, memsize_t _size, bool _writeaccess)
  5413. {
  5414. hfile = _hfile;
  5415. #ifdef _WIN32
  5416. if (pagesize==0) {
  5417. SYSTEM_INFO sysinfo;
  5418. GetSystemInfo(&sysinfo);
  5419. pagesize = sysinfo.dwAllocationGranularity;
  5420. }
  5421. hmap = NULLFILE;
  5422. #endif
  5423. ptr = NULL;
  5424. filesize = _filesize;
  5425. windowsize = _size; // first open size assumed to be window size
  5426. reinit(_ofs, _size, _writeaccess);
  5427. }
  5428. ~CMemoryMappedFile()
  5429. {
  5430. #ifdef _WIN32
  5431. if (hmap!=NULLFILE)
  5432. CloseHandle(hmap);
  5433. CloseHandle(hfile);
  5434. #else
  5435. close(hfile);
  5436. #endif
  5437. }
  5438. byte *base() { return ptr; }
  5439. offset_t offset() { return ofs; }
  5440. virtual memsize_t length() { return size; }
  5441. virtual offset_t fileSize() { return filesize; }
  5442. virtual int compareWithin(const void *p)
  5443. {
  5444. if (p<ptr)
  5445. return -1;
  5446. if (p>=size+ptr)
  5447. return 1;
  5448. return 0;
  5449. }
  5450. bool writeAccess() { return writeaccess; }
  5451. void flush()
  5452. {
  5453. if (ptr) {
  5454. #ifdef _WIN32
  5455. FlushViewOfFile(realptr(),0);
  5456. #elif defined(__linux__)
  5457. msync(realptr(),realsize(),MS_SYNC);
  5458. #else
  5459. UNIMPLEMENTED;
  5460. #endif
  5461. }
  5462. }
  5463. byte *nextPtr(const void *p,offset_t skip, memsize_t req, memsize_t &got)
  5464. {
  5465. // for scanning in sequence
  5466. if (p==NULL)
  5467. p = ptr;
  5468. else if ((p<ptr)||(p>ptr+size))
  5469. throw MakeStringException(-1,"CMemoryMappedFile::nextPtr - outside map");
  5470. memsize_t d = (byte *)p-ptr;
  5471. offset_t o = ofs+d+skip;
  5472. if (o>=filesize) {
  5473. got = 0;
  5474. return NULL;
  5475. }
  5476. offset_t left = filesize-o;
  5477. if (left<req)
  5478. req = (memsize_t)left;
  5479. if (o+req-ofs>size) {
  5480. reinit(o,windowsize,writeaccess);
  5481. assertex(o==ofs);
  5482. got = windowsize;
  5483. if (left<got)
  5484. got = (memsize_t)left;
  5485. return ptr;
  5486. }
  5487. got = (memsize_t)(size+o-ofs);
  5488. if (left<got)
  5489. got = (memsize_t)left;
  5490. return (byte *)p+skip;
  5491. }
  5492. virtual void reinit(offset_t _ofs, memsize_t _size, bool _writeaccess)
  5493. {
  5494. writeaccess = _writeaccess;
  5495. if (ptr) {
  5496. #ifdef _WIN32
  5497. // no need to unmap view
  5498. if (hmap != INVALID_HANDLE_VALUE) {
  5499. CloseHandle(hmap);
  5500. hmap = INVALID_HANDLE_VALUE;
  5501. }
  5502. #elif defined(__linux__)
  5503. munmap(realptr(),realsize());
  5504. // error checking TBD
  5505. #else
  5506. UNIMPLEMENTED;
  5507. #endif
  5508. ptr = NULL;
  5509. }
  5510. if (_ofs>filesize)
  5511. ofs = filesize;
  5512. else
  5513. ofs = _ofs;
  5514. realofs = pageround(ofs);
  5515. if (_size==(memsize_t)-1) {
  5516. size = (memsize_t)(filesize-_ofs);
  5517. if (size!=(filesize-_ofs))
  5518. throw MakeStringException(-1,"CMemoryMappedFile::reinit file too big");
  5519. }
  5520. else
  5521. size = _size;
  5522. memsize_t mapsz = realsize();
  5523. if (filesize-realofs<mapsz)
  5524. mapsz = (memsize_t)(filesize-realofs);
  5525. #ifdef _WIN32
  5526. LARGE_INTEGER li;
  5527. if (hmap == INVALID_HANDLE_VALUE) {
  5528. hmap = CreateFileMapping(hfile,NULL,writeaccess?PAGE_READWRITE:PAGE_READONLY,0, 0, NULL);
  5529. if (!hmap) {
  5530. DWORD err = GetLastError();
  5531. throw makeOsException(err,"CMemoryMappedFile::reinit");
  5532. }
  5533. }
  5534. li.QuadPart = realofs;
  5535. ptr = (byte *) MapViewOfFile(hmap, writeaccess?(FILE_MAP_READ|FILE_MAP_WRITE):FILE_MAP_READ, li.HighPart, li.LowPart, mapsz);
  5536. if (!ptr) {
  5537. DWORD err = GetLastError();
  5538. throw makeOsException(err,"CMemoryMappedFile::reinit");
  5539. }
  5540. #elif defined (__linux__)
  5541. ptr = (byte *) mmap(NULL, mapsz, writeaccess?(PROT_READ|PROT_WRITE):PROT_READ, MAP_SHARED|MAP_NORESERVE, hfile, realofs);
  5542. // error checking TBD
  5543. #else
  5544. UNIMPLEMENTED;
  5545. #endif
  5546. if (ptr)
  5547. ptr += (ofs-realofs);
  5548. }
  5549. };
  5550. #ifdef _WIN32
  5551. size32_t CMemoryMappedFile::pagesize=0;
  5552. #endif
  5553. // TBD MADV_SEQUENTIAL & MADV_WILLNEED?
  5554. IMemoryMappedFile *CFile::openMemoryMapped(offset_t ofs, memsize_t len, bool write)
  5555. {
  5556. HANDLE hfile = openHandle(write?IFOcreaterw:IFOread, IFSHread, false);
  5557. if (hfile == NULLFILE)
  5558. return NULL;
  5559. return new CMemoryMappedFile(hfile,size(),ofs,len,write);
  5560. }
  5561. bool isSpecialPath(const char *path)
  5562. {
  5563. // used for remote queries
  5564. if (!path)
  5565. return false;
  5566. if (isPathSepChar(path[0])&&(path[0]==path[1])) {
  5567. path += 2;
  5568. loop {
  5569. if (*path=='/')
  5570. return (path[1]=='>');
  5571. if (!*path||(*path=='\\'))
  5572. return false;
  5573. path++;
  5574. }
  5575. }
  5576. return (path&&(*path=='/')&&(path[1]=='>'));
  5577. }
  5578. size32_t SendFile(ISocket *target, IFileIO *fileio,offset_t start,size32_t len)
  5579. {
  5580. assertex(target);
  5581. assertex(fileio);
  5582. offset_t fsz = fileio->size();
  5583. if (start>=fsz)
  5584. return 0;
  5585. if (start+len>fsz)
  5586. len = (size32_t)(fsz-start);
  5587. if (!len)
  5588. return 0;
  5589. CFileIO *cfile = QUERYINTERFACE(fileio,CFileIO);
  5590. if (cfile) {
  5591. HANDLE fh = cfile->queryHandle();
  5592. if ((fh!=(HANDLE)0)&&(fh!=(HANDLE)-1)) {
  5593. unsigned sh = target->OShandle();
  5594. if ((sh!=(unsigned)0)&&(sh!=(unsigned)-1)) {
  5595. #ifdef _WIN32
  5596. // MORE - should there be something here?
  5597. #elif defined(__linux__)
  5598. // TransmitFile not in std DLLs so don't bother with
  5599. off_t ofs = start;
  5600. ssize_t sent = ::sendfile(sh,fh,&ofs,len);
  5601. if (sent!=(ssize_t)-1)
  5602. return (size32_t)sent;
  5603. int err = errno;
  5604. if ((err!=EINVAL)&&(err!=ENOSYS))
  5605. throw makeOsException(err, "sendfile");
  5606. #else
  5607. UNIMPLEMENTED;
  5608. #endif
  5609. }
  5610. }
  5611. }
  5612. #ifdef _DEBUG
  5613. #ifndef _WIN32
  5614. WARNLOG("SendFile falling back");
  5615. #endif
  5616. #endif
  5617. // fallback
  5618. MemoryAttr ma;
  5619. void *buf = ma.allocate(len);
  5620. size32_t rd = fileio->read(start,len,buf);
  5621. target->write(buf,rd);
  5622. return rd;
  5623. }
  5624. void asyncClose(IFileIO *io)
  5625. {
  5626. if (!io)
  5627. return;
  5628. static CriticalSection ADsect;
  5629. CriticalBlock block(ADsect);
  5630. static Owned<IWorkQueueThread> adwp = createWorkQueueThread();
  5631. class cWQI: public CInterface,implements IWorkQueueItem
  5632. {
  5633. Owned<IFileIO> io;
  5634. public:
  5635. IMPLEMENT_IINTERFACE;
  5636. cWQI(IFileIO *_io)
  5637. : io(_io)
  5638. {
  5639. }
  5640. void execute()
  5641. {
  5642. io.clear();
  5643. }
  5644. };
  5645. adwp->post(new cWQI(io));
  5646. };
  5647. int stdIoHandle(const char *path)
  5648. {
  5649. if (strcmp(path,"stdin:")==0)
  5650. return 0;
  5651. if (strcmp(path,"stdout:")==0)
  5652. return 1;
  5653. if (strcmp(path,"stderr:")==0)
  5654. return 2;
  5655. return -1;
  5656. }
  5657. extern jlib_decl bool containsFileWildcard(const char * path)
  5658. {
  5659. if (!path)
  5660. return false;
  5661. return strchr(path, '*') || strchr(path, '?');
  5662. }
  5663. extern jlib_decl bool isDirectory(const char * path)
  5664. {
  5665. Owned<IFile> file = createIFile(path);
  5666. return file->isDirectory() == foundYes;
  5667. }
  5668. // IFileIOCache
  5669. class CLazyFileIOCache;
  5670. class CCachedFileIO: public CInterface, implements IFileIO
  5671. {
  5672. CLazyFileIOCache &owner;
  5673. RemoteFilename filename;
  5674. CriticalSection &sect;
  5675. IFOmode mode;
  5676. void writeNotSupported(const char *s)
  5677. {
  5678. StringBuffer tmp;
  5679. filename.getRemotePath(tmp);
  5680. throw MakeStringException(-1, "CCachedFileIO(%s) %s not supported", tmp.str(), s);
  5681. }
  5682. public:
  5683. unsigned accesst;
  5684. Owned<IFileIO> cachedio;
  5685. CCachedFileIO(CLazyFileIOCache &_owner, CriticalSection &_sect, RemoteFilename &_filename, IFOmode _mode)
  5686. : owner(_owner), sect(_sect)
  5687. {
  5688. filename.set(_filename);
  5689. mode = _mode;
  5690. }
  5691. virtual void Link(void) const { CInterface::Link(); } \
  5692. virtual bool Release(void) const;
  5693. IFileIO *open();
  5694. size32_t read(offset_t pos, size32_t len, void * data)
  5695. {
  5696. CriticalBlock block(sect);
  5697. Owned<IFileIO> io = open();
  5698. return io->read(pos,len,data);
  5699. }
  5700. offset_t size()
  5701. {
  5702. CriticalBlock block(sect);
  5703. Owned<IFileIO> io = open();
  5704. return io->size();
  5705. }
  5706. size32_t write(offset_t pos, size32_t len, const void * data)
  5707. {
  5708. CriticalBlock block(sect);
  5709. Owned<IFileIO> io = open();
  5710. return io->write(pos,len,data);
  5711. }
  5712. virtual void flush()
  5713. {
  5714. CriticalBlock block(sect);
  5715. if (cachedio)
  5716. cachedio->flush();
  5717. }
  5718. virtual void close()
  5719. {
  5720. CriticalBlock block(sect);
  5721. if (cachedio)
  5722. {
  5723. cachedio->close();
  5724. cachedio.clear();
  5725. }
  5726. }
  5727. offset_t appendFile(IFile *file,offset_t pos,offset_t len)
  5728. {
  5729. CriticalBlock block(sect);
  5730. Owned<IFileIO> io = open();
  5731. return io->appendFile(file,pos,len);
  5732. }
  5733. void setSize(offset_t size)
  5734. {
  5735. CriticalBlock block(sect);
  5736. Owned<IFileIO> io = open();
  5737. io->setSize(size);
  5738. }
  5739. };
  5740. class CLazyFileIOCache: public CInterface, implements IFileIOCache
  5741. {
  5742. CriticalSection sect;
  5743. unsigned max;
  5744. IPointerArrayOf<CCachedFileIO> cache;
  5745. public:
  5746. IMPLEMENT_IINTERFACE;
  5747. CLazyFileIOCache(unsigned _max)
  5748. {
  5749. max = _max;
  5750. }
  5751. CCachedFileIO *addFile( RemoteFilename &filename, IFOmode mode )
  5752. {
  5753. // check exists
  5754. unsigned i = cache.ordinality();
  5755. ForEachItemIn(i2,cache) {
  5756. if (!cache.item(i2)) {
  5757. i = i2;
  5758. break;
  5759. }
  5760. }
  5761. Linked<CCachedFileIO> ret = new CCachedFileIO(*this,sect,filename,mode);
  5762. if (i<cache.ordinality())
  5763. cache.replace(ret,i);
  5764. else
  5765. cache.append(ret);
  5766. return ret.getClear();
  5767. }
  5768. void checkCache(unsigned wanted=1)
  5769. {
  5770. // called in sect
  5771. loop {
  5772. CCachedFileIO *oldest = NULL;
  5773. unsigned oldestt = 0;
  5774. unsigned t = msTick();
  5775. unsigned n = wanted;
  5776. ForEachItemIn(i1,cache) {
  5777. CCachedFileIO *cio = cache.item(i1);
  5778. if (cio&&cio->cachedio.get()) {
  5779. if (t-cio->accesst>=oldestt) {
  5780. oldest = cio;
  5781. oldestt = t-cio->accesst;
  5782. }
  5783. n++;
  5784. }
  5785. }
  5786. if (n<max)
  5787. break;
  5788. if (!oldest)
  5789. break;
  5790. oldest->cachedio.clear();
  5791. //If previously had max ios then we now have space.
  5792. if (n == max)
  5793. break;
  5794. }
  5795. }
  5796. const CCachedFileIO *removeFile(const CCachedFileIO *io)
  5797. {
  5798. CriticalBlock block(sect);
  5799. ForEachItemIn(idx, cache)
  5800. {
  5801. if (io == cache.item(idx))
  5802. {
  5803. cache.replace(NULL, idx, true);
  5804. break;
  5805. }
  5806. }
  5807. return io;
  5808. }
  5809. };
  5810. bool CCachedFileIO::Release(void) const
  5811. {
  5812. if (CInterface::Release())
  5813. return true;
  5814. if (!CInterface::IsShared())
  5815. return owner.removeFile(this)->Release();
  5816. return false;
  5817. }
  5818. IFileIO *CCachedFileIO::open()
  5819. {
  5820. // called in sect
  5821. if (!cachedio) {
  5822. owner.checkCache();
  5823. Owned<IFile> inFile = createIFile(filename);
  5824. cachedio.setown(inFile->open(mode));
  5825. if (!cachedio.get()) {
  5826. StringBuffer tmp;
  5827. filename.getRemotePath(tmp);
  5828. throw MakeStringException(-1, "CCachedFileIO::open(%d) '%s' failed", (int)mode, tmp.str());
  5829. }
  5830. }
  5831. accesst = msTick();
  5832. return cachedio.getLink();
  5833. }
  5834. IFileIOCache* createFileIOCache(unsigned max)
  5835. {
  5836. return new CLazyFileIOCache(max);
  5837. }
  5838. extern jlib_decl IFile * createSentinelTarget()
  5839. {
  5840. const char * sentinelFilename = getenv("SENTINEL");
  5841. if (sentinelFilename)
  5842. return createIFile(sentinelFilename);
  5843. else
  5844. return NULL;
  5845. }
  5846. extern jlib_decl void removeSentinelFile(IFile * sentinelFile)
  5847. {
  5848. if (sentinelFile)
  5849. {
  5850. if(sentinelFile->exists() && !sentinelFile->isDirectory())
  5851. {
  5852. DBGLOG("Removing sentinel file %s", sentinelFile->queryFilename());
  5853. try
  5854. {
  5855. sentinelFile->remove();
  5856. }
  5857. catch(IException *E)
  5858. {
  5859. StringBuffer s;
  5860. EXCLOG(E, s.appendf("Failed to remove sentinel file %s", sentinelFile->queryFilename()).str());
  5861. E->Release();
  5862. throw makeOsException(errno, "removeSentinelFile - file cannot be removed.");
  5863. }
  5864. }
  5865. }
  5866. }
  5867. extern jlib_decl void writeSentinelFile(IFile * sentinelFile)
  5868. {
  5869. if ( sentinelFile )
  5870. {
  5871. DBGLOG("Creating sentinel file %s for rerun from script", sentinelFile->queryFilename());
  5872. try
  5873. {
  5874. Owned<IFileIO> sentinel = sentinelFile->open(IFOcreate);
  5875. sentinel->write(0, 5, "rerun");
  5876. }
  5877. catch(IException *E)
  5878. {
  5879. StringBuffer s;
  5880. EXCLOG(E, s.appendf("Failed to create sentinel file %s for rerun from script", sentinelFile->queryFilename()).str());
  5881. E->Release();
  5882. throw makeOsException(errno, "writeSentinelFile - file not created.");
  5883. }
  5884. }
  5885. }
  5886. jlib_decl StringBuffer & appendCurrentDirectory(StringBuffer & target, bool blankIfFails)
  5887. {
  5888. char temp[_MAX_PATH+1];
  5889. if (!getcwd(temp,sizeof(temp)))
  5890. {
  5891. if (blankIfFails)
  5892. return target;
  5893. throw MakeStringException(JLIBERR_InternalError, "getcwd failed (%d)", errno);
  5894. }
  5895. return target.append(temp);
  5896. }