jfile.cpp 169 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341
  1. /*##############################################################################
  2. Copyright (C) 2011 HPCC Systems.
  3. All rights reserved. This program is free software: you can redistribute it and/or modify
  4. it under the terms of the GNU Affero General Public License as
  5. published by the Free Software Foundation, either version 3 of the
  6. License, or (at your option) any later version.
  7. This program is distributed in the hope that it will be useful,
  8. but WITHOUT ANY WARRANTY; without even the implied warranty of
  9. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  10. GNU Affero General Public License for more details.
  11. You should have received a copy of the GNU Affero General Public License
  12. along with this program. If not, see <http://www.gnu.org/licenses/>.
  13. ############################################################################## */
  14. #ifdef _WIN32
  15. #define _WIN32_WINNT 0x0500 // for CreateHardLink
  16. #include <errno.h>
  17. //#include <winsock.h> // for TransmitFile
  18. #endif
  19. #include "platform.h"
  20. #include <sys/types.h>
  21. #include <sys/stat.h>
  22. #include <algorithm>
  23. #if defined (__linux__)
  24. #include <time.h>
  25. #include <dirent.h>
  26. #include <utime.h>
  27. #include <sys/vfs.h>
  28. #include <sys/mman.h>
  29. #include <sys/sendfile.h>
  30. #endif
  31. #include "time.h"
  32. #include "jerror.hpp"
  33. #include "jlib.hpp"
  34. #include "jio.hpp"
  35. #include "jmisc.hpp"
  36. #include "jsort.hpp"
  37. #include "jmutex.hpp"
  38. #include "jfile.hpp"
  39. #include "jfile.ipp"
  40. #include <limits.h>
  41. #include "jexcept.hpp"
  42. #include "jsocket.hpp" // for IpAddress
  43. #include "jregexp.hpp"
  44. #include "portlist.h"
  45. // #define REMOTE_DISCONNECT_ON_DESTRUCTOR // enable to disconnect on IFile destructor
  46. // this should not be enabled in WindowRemoteDirectory used
  47. #ifdef _DEBUG
  48. #define ASSERTEX(e) assertex(e);
  49. #else
  50. #define ASSERTEX(e)
  51. #endif
  52. #ifdef __64BIT__
  53. #define DEFAULT_STREAM_BUFFER_SIZE 0x100000
  54. #else
  55. #define DEFAULT_STREAM_BUFFER_SIZE 0x10000
  56. #endif
  57. #ifdef _WIN32
  58. #define NULLFILE INVALID_HANDLE_VALUE
  59. #else
  60. #define NULLFILE -1
  61. #endif
  62. static IFile *createIFileByHook(const RemoteFilename & filename);
  63. static char ShareChar='$';
  64. bool isShareChar(char c)
  65. {
  66. return (c==ShareChar)||(c=='$');
  67. }
  68. char getShareChar()
  69. {
  70. return ShareChar;
  71. }
  72. void setShareChar(char c)
  73. {
  74. ShareChar = c;
  75. }
  76. StringBuffer &setPathDrive(StringBuffer &filename,unsigned drvnum)
  77. {
  78. return swapPathDrive(filename,(unsigned)-1,drvnum);
  79. }
  80. unsigned getPathDrive(const char *s)
  81. {
  82. char c = *s;
  83. if (isPathSepChar(c)&&(s[1]==c)) {
  84. s = strchr(s+2,c);
  85. if (!s)
  86. return 0;
  87. }
  88. if (isPathSepChar(c))
  89. s++;
  90. if (*s&&((s[1]==':')||(isShareChar(s[1]))))
  91. return *s-'c';
  92. return 0;
  93. }
  94. StringBuffer &swapPathDrive(StringBuffer &filename,unsigned fromdrvnum,unsigned todrvnum,const char *frommask,const char *tomask)
  95. {
  96. const char *s = filename.str();
  97. char c = *s;
  98. if (isPathSepChar(c)&&(s[1]==c)) {
  99. s = strchr(s+2,c);
  100. if (!s)
  101. return filename;
  102. }
  103. if (isPathSepChar(c))
  104. s++;
  105. if (*s&&((fromdrvnum==(unsigned)-1)||(*s==(fromdrvnum+'c')))&&((s[1]==':')||(isShareChar(s[1]))))
  106. filename.setCharAt((size32_t)(s-filename.str()),todrvnum+'c');
  107. else if (frommask&&*frommask) { // OSS
  108. StringBuffer tmp;
  109. if (replaceConfigurationDirectoryEntry(filename.str(),frommask,tomask, tmp))
  110. tmp.swapWith(filename);
  111. }
  112. return filename;
  113. }
  114. bool checkFileExists(const char * filename)
  115. {
  116. #ifdef _WIN32
  117. for (unsigned i=0;i<10;i++) {
  118. DWORD ret = (DWORD)GetFileAttributes(filename);
  119. if (ret!=(DWORD)-1)
  120. return true;
  121. DWORD err = GetLastError();
  122. if (err!=ERROR_IO_PENDING)
  123. break;
  124. Sleep(100*i);
  125. }
  126. return false;
  127. #else
  128. struct stat info;
  129. return (stat(filename, &info) == 0);
  130. #endif
  131. }
  132. bool checkDirExists(const char * filename)
  133. {
  134. #ifdef _WIN32
  135. DWORD attr = GetFileAttributes(filename);
  136. return (attr != (DWORD)-1)&&(attr & FILE_ATTRIBUTE_DIRECTORY);
  137. #else
  138. struct stat info;
  139. if (stat(filename, &info) != 0)
  140. return false;
  141. return S_ISDIR(info.st_mode);
  142. #endif
  143. }
  144. static void set_inherit(HANDLE handle, bool inherit)
  145. {
  146. #ifndef _WIN32
  147. long flag = fcntl(handle, F_GETFD);
  148. if(inherit)
  149. flag &= ~FD_CLOEXEC;
  150. else
  151. flag |= FD_CLOEXEC;
  152. fcntl(handle, F_SETFD, flag);
  153. #endif
  154. }
  155. static StringBuffer &getLocalOrRemoteName(StringBuffer &name,const RemoteFilename & filename)
  156. {
  157. if (filename.isLocal()&&!filename.isNull()) {
  158. filename.getLocalPath(name);
  159. #ifdef _WIN32
  160. // kludge to allow local linux paths on windows
  161. const char *n=name.str();
  162. if (n[0]=='/') {
  163. StringBuffer tmp;
  164. if (isShareChar(n[2])) {
  165. tmp.append(n[1]).append(':');
  166. n+=3;
  167. }
  168. while (*n) {
  169. if (*n=='/')
  170. tmp.append('\\');
  171. else
  172. tmp.append(*n);
  173. n++;
  174. }
  175. name.clear().append(tmp);
  176. }
  177. #endif
  178. }
  179. else
  180. filename.getRemotePath(name);
  181. return name;
  182. }
  183. CFile::CFile(const char * _filename)
  184. {
  185. filename.set(_filename);
  186. flags = ((unsigned)IFSHread)|((S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH)<<16);
  187. }
  188. void CFile::setCreateFlags(unsigned cflags)
  189. {
  190. flags |= (cflags<<16);
  191. }
  192. void CFile::setShareMode(IFSHmode shmode)
  193. {
  194. flags &= ~(IFSHfull|IFSHread);
  195. flags |= (unsigned)(shmode&(IFSHfull|IFSHread));
  196. }
  197. bool CFile::exists()
  198. {
  199. if (stdIoHandle(filename)>=0)
  200. return true;
  201. return checkFileExists(filename);
  202. }
  203. #ifdef _WIN32
  204. bool WindowsCreateDirectory(const char * path)
  205. {
  206. unsigned retry = 0;
  207. loop {
  208. if (CreateDirectory(path, NULL))
  209. return true;
  210. DWORD err = GetLastError();
  211. if ((err==ERROR_FILE_NOT_FOUND) || (err==ERROR_PATH_NOT_FOUND) || (err==ERROR_FILE_EXISTS) || (err==ERROR_CANNOT_MAKE))
  212. break;
  213. else if (err==ERROR_ALREADY_EXISTS) {
  214. DWORD attr = GetFileAttributes(path);
  215. if ((attr != -1)&&( attr & FILE_ATTRIBUTE_DIRECTORY))
  216. return true;
  217. return false;
  218. }
  219. if ((retry++==10)|| // some or all of the following can occur when the domain controller gets busy
  220. // retrying improves chance of success
  221. ((err!=ERROR_NETNAME_DELETED)&&(err!=ERROR_DEV_NOT_EXIST)&&(err!=ERROR_GEN_FAILURE)&&(err!=ERROR_NETWORK_BUSY)&&(err!=ERROR_BAD_NET_NAME)))
  222. throw MakeOsException(err,"WindowsCreateDirectory %s", path);
  223. //PROGLOG("Retrying(%d) WindowsCreateDirectory %s, err=%d",retry,filename,err);
  224. Sleep(retry*100);
  225. }
  226. return false;
  227. }
  228. #else
  229. bool LinuxCreateDirectory(const char * path)
  230. {
  231. if (!path)
  232. return false;
  233. if (CreateDirectory(path, NULL))
  234. return true;
  235. else
  236. {
  237. if (EEXIST == errno)
  238. {
  239. struct stat info;
  240. if (stat(path, &info) != 0)
  241. return false;
  242. return S_ISDIR(info.st_mode);
  243. }
  244. }
  245. return false;
  246. }
  247. #endif
  248. bool localCreateDirectory(const char *name)
  249. {
  250. if (!name)
  251. return false;
  252. size32_t l = (size32_t)strlen(name);
  253. if (l==0)
  254. return true;
  255. if (isPathSepChar(name[0])&&((l==1)||(isPathSepChar(name[1])&&!containsPathSepChar(name+2))))
  256. return true;
  257. #ifdef _WIN32
  258. if (name[1]==':') {
  259. if ((l==2)||((l==3)&&isPathSepChar(name[2])))
  260. return true;
  261. }
  262. #endif
  263. if (checkDirExists(name))
  264. return true;
  265. #ifdef _WIN32
  266. if (WindowsCreateDirectory(name))
  267. #else
  268. if (LinuxCreateDirectory(name))
  269. #endif
  270. return true;
  271. if (isPathSepChar(name[l-1])) l--;
  272. while (l&&!isPathSepChar(name[l-1]))
  273. l--;
  274. if (l<=1)
  275. return true;
  276. StringAttr parent(name,l-1);
  277. if (!localCreateDirectory(parent.get()))
  278. return false;
  279. #ifdef _WIN32
  280. return (WindowsCreateDirectory(name));
  281. #else
  282. return (LinuxCreateDirectory(name));
  283. #endif
  284. }
  285. bool CFile::createDirectory()
  286. {
  287. return localCreateDirectory(filename);
  288. }
  289. #ifdef _WIN32
  290. union TimeIntegerUnion
  291. {
  292. FILETIME ft;
  293. ULARGE_INTEGER l;
  294. };
  295. void FILETIMEtoIDateTime(CDateTime * target, const FILETIME & ft)
  296. {
  297. if (target)
  298. {
  299. TimeIntegerUnion u;
  300. memcpy(&u, &ft, sizeof(ft));
  301. unsigned __int64 hundredNanoseconds = (u.l.QuadPart % 10000000);
  302. u.l.QuadPart -= hundredNanoseconds;
  303. SYSTEMTIME systime;
  304. FileTimeToSystemTime(&u.ft, &systime);
  305. target->set(systime.wYear, systime.wMonth, systime.wDay, systime.wHour, systime.wMinute, systime.wSecond, (unsigned)(hundredNanoseconds*100));
  306. }
  307. }
  308. FILETIME * IDateTimetoFILETIME(FILETIME & ft, const CDateTime * dt)
  309. {
  310. if (!dt)
  311. return NULL;
  312. SYSTEMTIME systime;
  313. TimeIntegerUnion u;
  314. unsigned wYear, wMonth, wDay, wHour, wMinute, wSecond, nanoSeconds;
  315. dt->getDate(wYear, wMonth, wDay);
  316. dt->getTime(wHour, wMinute, wSecond, nanoSeconds);
  317. systime.wYear = wYear;
  318. systime.wMonth = wMonth;
  319. systime.wDay = wDay;
  320. systime.wHour = wHour;
  321. systime.wMinute = wMinute;
  322. systime.wSecond = wSecond;
  323. systime.wMilliseconds = 0;
  324. SystemTimeToFileTime(&systime, &u.ft);
  325. //Adjust the fractions of a second ourselves because the function above is only accurate to milli seconds.
  326. u.l.QuadPart += (nanoSeconds / 100);
  327. memcpy(&ft, &u, sizeof(ft));
  328. return &ft;
  329. }
  330. #endif
  331. bool CFile::getTime(CDateTime * createTime, CDateTime * modifiedTime, CDateTime * accessedTime)
  332. {
  333. #ifdef _WIN32
  334. //MORE could use GetFileAttributesEx() if we were allowed...
  335. FILETIME timeCreated, timeModified, timeAccessed;
  336. HANDLE handle = CreateFile(filename, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, 0, NULL);
  337. if (handle==(HANDLE)-1)
  338. return false;
  339. GetFileTime(handle, &timeCreated, &timeAccessed, &timeModified);
  340. CloseHandle(handle);
  341. FILETIMEtoIDateTime(createTime, timeCreated);
  342. FILETIMEtoIDateTime(modifiedTime, timeModified);
  343. FILETIMEtoIDateTime(accessedTime, timeAccessed);
  344. #else
  345. struct stat info;
  346. if (stat(filename, &info) != 0)
  347. return false;
  348. timetToIDateTime(accessedTime, info.st_atime);
  349. timetToIDateTime(createTime, info.st_ctime);
  350. timetToIDateTime(modifiedTime, info.st_mtime);
  351. #endif
  352. return true;
  353. }
  354. bool CFile::setTime(const CDateTime * createTime, const CDateTime * modifiedTime, const CDateTime * accessedTime)
  355. {
  356. #ifdef _WIN32
  357. FILETIME timeCreated, timeModified, timeAccessed;
  358. FILETIME *pTimeCreated, *pTimeModified, *pTimeAccessed;
  359. pTimeCreated = IDateTimetoFILETIME(timeCreated, createTime);
  360. pTimeModified = IDateTimetoFILETIME(timeModified, modifiedTime);
  361. pTimeAccessed = IDateTimetoFILETIME(timeAccessed, accessedTime);
  362. HANDLE handle = CreateFile(filename, GENERIC_WRITE, FILE_SHARE_READ, NULL, OPEN_EXISTING, 0, NULL);
  363. if (!handle)
  364. return false;
  365. SetFileTime(handle, pTimeCreated, pTimeAccessed, pTimeModified);
  366. CloseHandle(handle);
  367. return true;
  368. #elif defined (__linux__)
  369. struct utimbuf am;
  370. if (!accessedTime||!modifiedTime) {
  371. struct stat info;
  372. if (stat(filename, &info) != 0)
  373. return false;
  374. am.actime = info.st_atime;
  375. am.modtime = info.st_mtime;
  376. }
  377. if (accessedTime)
  378. am.actime = timetFromIDateTime (accessedTime);
  379. if (modifiedTime)
  380. am.modtime = timetFromIDateTime (modifiedTime);
  381. return (utime(filename, &am)==0);
  382. #else
  383. UNIMPLEMENTED;
  384. #endif
  385. }
  386. fileBool CFile::isDirectory()
  387. {
  388. #ifdef _WIN32
  389. DWORD attr = GetFileAttributes(filename);
  390. if (attr == -1)
  391. return notFound;
  392. return ( attr & FILE_ATTRIBUTE_DIRECTORY) ? foundYes : foundNo;
  393. #else
  394. struct stat info;
  395. if (stat(filename, &info) != 0)
  396. return notFound;
  397. return S_ISDIR(info.st_mode) ? foundYes : foundNo;
  398. #endif
  399. }
  400. fileBool CFile::isFile()
  401. {
  402. if (stdIoHandle(filename)>=0)
  403. return foundYes;
  404. #ifdef _WIN32
  405. DWORD attr = GetFileAttributes(filename);
  406. if (attr == -1)
  407. return notFound;
  408. return ( attr & FILE_ATTRIBUTE_DIRECTORY) ? foundNo : foundYes;
  409. #else
  410. struct stat info;
  411. if (stat(filename, &info) != 0)
  412. return notFound;
  413. return S_ISREG(info.st_mode) ? foundYes : foundNo;
  414. #endif
  415. }
  416. fileBool CFile::isReadOnly()
  417. {
  418. #ifdef _WIN32
  419. DWORD attr = GetFileAttributes(filename);
  420. if (attr == -1)
  421. return notFound;
  422. return ( attr & FILE_ATTRIBUTE_READONLY) ? foundYes : foundNo;
  423. #else
  424. struct stat info;
  425. if (stat(filename, &info) != 0)
  426. return notFound;
  427. //MORE: I think this is correct, but someone with better unix knowledge should check!
  428. return (info.st_mode & (S_IWUSR|S_IWGRP|S_IWOTH)) ? foundNo : foundYes;
  429. #endif
  430. }
  431. #ifndef _WIN32
  432. static bool setShareLock(int fd,IFSHmode share)
  433. {
  434. struct flock fl;
  435. do {
  436. memset(&fl,0,sizeof(fl));
  437. if (share==IFSHnone)
  438. fl.l_type = F_WRLCK;
  439. else if (share==IFSHread)
  440. fl.l_type = F_RDLCK;
  441. else
  442. fl.l_type = F_UNLCK;
  443. fl.l_whence = SEEK_SET;
  444. fl.l_pid = getpid();
  445. if (fcntl(fd, F_SETLK, &fl) != -1)
  446. return true;
  447. } while (errno==EINTR);
  448. if ((errno!=EAGAIN)&&(errno!=EACCES))
  449. ERRLOG("setShareLock failure %d, mode %d", errno,(int)share);
  450. return (share==IFSHfull); // always return true for full
  451. }
  452. #endif
  453. HANDLE CFile::openHandle(IFOmode mode, IFSHmode sharemode, bool async, int stdh)
  454. {
  455. HANDLE handle = NULLFILE;
  456. #ifdef _WIN32
  457. if (stdh>=0) {
  458. DWORD mode;
  459. switch (stdh) {
  460. case 0: mode = STD_INPUT_HANDLE; break;
  461. case 1: mode = STD_OUTPUT_HANDLE; break;
  462. case 2: mode = STD_ERROR_HANDLE; break;
  463. default:
  464. return handle;
  465. }
  466. DuplicateHandle(GetCurrentProcess(), GetStdHandle(mode), GetCurrentProcess(), &handle , 0, FALSE, DUPLICATE_SAME_ACCESS);
  467. return handle;
  468. }
  469. DWORD share = 0;
  470. switch (sharemode) {
  471. case (IFSHfull|IFSHread):
  472. case IFSHfull:
  473. share |= FILE_SHARE_WRITE;
  474. // fall through
  475. case IFSHread:
  476. share |= FILE_SHARE_READ;
  477. }
  478. DWORD fflags = async?FILE_FLAG_OVERLAPPED:0;
  479. if (async&&(mode==IFOread))
  480. fflags |= FILE_FLAG_SEQUENTIAL_SCAN;
  481. switch (mode) {
  482. case IFOcreate:
  483. handle = CreateFile(filename, GENERIC_WRITE, share, NULL, CREATE_ALWAYS, fflags, NULL);
  484. break;
  485. case IFOread:
  486. handle = CreateFile(filename, GENERIC_READ, share, NULL, OPEN_EXISTING, fflags, NULL);
  487. break;
  488. case IFOwrite:
  489. handle = CreateFile(filename, GENERIC_WRITE, share, NULL, OPEN_ALWAYS, fflags, NULL);
  490. break;
  491. case IFOcreaterw:
  492. handle = CreateFile(filename, GENERIC_WRITE|GENERIC_READ, share, NULL, CREATE_ALWAYS, fflags, NULL);
  493. break;
  494. case IFOreadwrite:
  495. handle = CreateFile(filename, GENERIC_WRITE|GENERIC_READ, share, NULL, OPEN_ALWAYS, fflags, NULL);
  496. break;
  497. }
  498. if (handle == NULLFILE)
  499. {
  500. DWORD err = GetLastError();
  501. if ((IFOread!=mode) || ((err!=ERROR_FILE_NOT_FOUND) && (err!=ERROR_PATH_NOT_FOUND)))
  502. throw MakeOsException(err,"CFile::open %s (%x, %x)", filename.get(), mode, share);
  503. return NULLFILE;
  504. }
  505. #else
  506. if (stdh>=0)
  507. return (HANDLE)dup(stdh);
  508. unsigned fileflags = (flags>>16) & (S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IWGRP|S_IXGRP|S_IROTH|S_IWOTH|S_IXOTH);
  509. unsigned openflags;
  510. switch (mode) {
  511. case IFOcreate:
  512. openflags = O_WRONLY | O_CREAT | O_TRUNC;
  513. break;
  514. case IFOread:
  515. openflags = O_RDONLY;
  516. break;
  517. case IFOwrite:
  518. openflags = O_WRONLY | O_CREAT;
  519. break;
  520. case IFOcreaterw:
  521. openflags = O_RDWR | O_CREAT | O_TRUNC;
  522. break;
  523. case IFOreadwrite:
  524. openflags = O_RDWR | O_CREAT;
  525. break;
  526. default:
  527. return NULLFILE;
  528. }
  529. handle = _lopen(filename.get(), openflags, fileflags);
  530. if (handle == -1)
  531. {
  532. if ((IFOread!=mode) || (errno != ENOENT))
  533. throw MakeErrnoException(errno, "CFile::open %s", filename.get());
  534. return NULLFILE;
  535. }
  536. // check not a directory (compatible with windows)
  537. struct stat info;
  538. if (fstat(handle, &info) == -1) {
  539. int err = errno;
  540. close(handle);
  541. handle = NULLFILE;
  542. throw MakeErrnoException(err, "CFile::open fstat %s", filename.get());
  543. }
  544. if (S_ISDIR(info.st_mode)) {
  545. close(handle);
  546. handle = NULLFILE;
  547. throw MakeErrnoException(EISDIR, "CFile::open %s", filename.get());
  548. }
  549. #endif
  550. return handle;
  551. }
  552. IFileIO * CFile::open(IFOmode mode)
  553. {
  554. // we may want mode dependant defaults later
  555. return openShared(mode,(IFSHmode)(flags&(IFSHfull|IFSHread)));
  556. }
  557. IFileAsyncIO * CFile::openAsync(IFOmode mode)
  558. {
  559. HANDLE handle = openHandle(mode,IFSHread,true); // I don't think we want shared write to an async file
  560. #ifndef _WIN32
  561. set_inherit(handle, false);
  562. #endif
  563. return new CFileAsyncIO(handle,IFSHread);
  564. }
  565. const char * CFile::queryFilename()
  566. {
  567. return filename;
  568. }
  569. bool CFile::remove()
  570. {
  571. #ifdef _WIN32
  572. unsigned retry = 0;
  573. loop {
  574. if (isDirectory()==foundYes) {
  575. if (RemoveDirectory(filename) != 0)
  576. return true;
  577. }
  578. else {
  579. if (DeleteFile(filename) != 0)
  580. return true;
  581. }
  582. DWORD err = GetLastError();
  583. if ( (err==ERROR_FILE_NOT_FOUND) || (err==ERROR_PATH_NOT_FOUND) )
  584. break;
  585. if ((retry++==10)|| // some or all of the following can occur when the domain controller gets busy
  586. // retrying improves chance of success
  587. ((err!=ERROR_NETNAME_DELETED)&&(err!=ERROR_DEV_NOT_EXIST)&&(err!=ERROR_GEN_FAILURE)&&(err!=ERROR_NETWORK_BUSY)&&(err!=ERROR_BAD_NET_NAME)))
  588. throw MakeOsException(err,"CFile::remove %s", filename.get());
  589. //PROGLOG("Retrying(%d) DeleteFile %s, err=%d",retry,filename,err);
  590. Sleep(retry*100);
  591. }
  592. return false;
  593. #else
  594. if (isDirectory()==foundYes) {
  595. if (rmdir(filename) == 0)
  596. return true;
  597. }
  598. else {
  599. if (unlink(filename) == 0)
  600. return true;
  601. }
  602. if (ENOENT!=errno)
  603. throw MakeErrnoException("CFile::remove %s", filename.get());
  604. return false;
  605. #endif
  606. }
  607. void CFile::rename(const char *newname)
  608. {
  609. // now hopefully newname is just file tail
  610. // however we do allow full paths and extract tail, warning if the directory appears to mismatch
  611. StringBuffer path;
  612. splitDirTail(filename,path);
  613. StringBuffer newdir;
  614. const char *tail = splitDirTail(newname,newdir);
  615. if (path.length()&&newdir.length()) {
  616. if (strcmp(newdir.str(),path.str())!=0)
  617. WARNLOG("CFile::rename '%s' to '%s' : directory mismatch",filename.get(),newname);
  618. }
  619. const char *dst = path.append(tail);
  620. if (isPathSepChar(dst[0])&&(dst[1]==dst[0])) { // hmm is share - convert to local path
  621. RemoteFilename rfn;
  622. rfn.setRemotePath(dst);
  623. if (rfn.isLocal())
  624. dst = rfn.getLocalPath(path.clear()).str();
  625. }
  626. if (-1 == ::rename(filename, dst))
  627. throw MakeErrnoException("CFile::rename(%s, %s)", filename.get(),dst);
  628. filename.set(path);
  629. }
  630. void CFile::move(const char *newname)
  631. {
  632. if (!newname||!*newname)
  633. return;
  634. StringBuffer path;
  635. if (isPathSepChar(newname[0])&&(newname[1]==newname[0])) { // hmm is share - convert to local path
  636. RemoteFilename rfn;
  637. rfn.setRemotePath(newname);
  638. if (rfn.isLocal())
  639. newname = rfn.getLocalPath(path.clear()).str();
  640. }
  641. #ifdef _WIN32
  642. unsigned retry = 0;
  643. loop {
  644. if (MoveFileEx(filename.get(),newname,0))
  645. return;
  646. DWORD err = GetLastError();
  647. if ((retry++==10)|| // some or all of the following can occur when the domain controller gets busy
  648. // retrying improves chance of success
  649. ((err!=ERROR_NETNAME_DELETED)&&(err!=ERROR_DEV_NOT_EXIST)&&(err!=ERROR_GEN_FAILURE)&&(err!=ERROR_NETWORK_BUSY)&&(err!=ERROR_BAD_NET_NAME)))
  650. throw MakeOsException(err,"CFile::move(%s, %s)", filename.get(), newname);
  651. Sleep(retry*100);
  652. }
  653. #else
  654. int ret=::rename(filename.get(),newname);
  655. if (ret==-1)
  656. throw MakeErrnoException("CFile::move(%s, %s)", filename.get(), newname);
  657. #endif
  658. filename.set(newname);
  659. }
  660. #ifdef _WIN32
  661. DWORD CALLBACK fastCopyProgressRoutine(
  662. LARGE_INTEGER TotalFileSize,
  663. LARGE_INTEGER TotalBytesTransferred,
  664. LARGE_INTEGER StreamSize,
  665. LARGE_INTEGER StreamBytesTransferred,
  666. DWORD dwStreamNumber,
  667. DWORD dwCallbackReason,
  668. HANDLE hSourceFile,
  669. HANDLE hDestinationFile,
  670. LPVOID lpData
  671. )
  672. {
  673. if (TotalBytesTransferred.QuadPart<=TotalFileSize.QuadPart)
  674. {
  675. CFPmode status = ((ICopyFileProgress *)lpData)->onProgress(TotalBytesTransferred.QuadPart,TotalFileSize.QuadPart);
  676. switch (status)
  677. {
  678. case CFPcontinue:
  679. return PROGRESS_CONTINUE;
  680. case CFPstop:
  681. return PROGRESS_STOP;
  682. default:
  683. return PROGRESS_CANCEL;
  684. }
  685. }
  686. return PROGRESS_CONTINUE;
  687. }
  688. #endif
  689. bool CFile::fastCopyFile(CFile &target, size32_t buffersize, ICopyFileProgress *progress)
  690. {
  691. // only currently supported for windows
  692. #ifdef _WIN32
  693. unsigned retry = 0;
  694. loop {
  695. BOOL cancel=FALSE;
  696. if (CopyFileEx(queryFilename(),target.queryFilename(),progress?fastCopyProgressRoutine:NULL,progress?progress:NULL,&cancel,0))
  697. break;
  698. DWORD err = GetLastError();
  699. if ( (err==ERROR_FILE_NOT_FOUND) || (err==ERROR_PATH_NOT_FOUND) )
  700. return false;
  701. if ((retry++==10)|| // some or all of the following can occur when the domain controller gets busy
  702. // retrying improves chance of success
  703. ((err!=ERROR_NETNAME_DELETED)&&(err!=ERROR_DEV_NOT_EXIST)&&(err!=ERROR_GEN_FAILURE)&&(err!=ERROR_NETWORK_BUSY)&&(err!=ERROR_BAD_NET_NAME)))
  704. return false;
  705. Sleep(retry*100);
  706. }
  707. return true;
  708. #else
  709. return false;
  710. #endif
  711. }
  712. void CFile::copySection(const RemoteFilename &dest, offset_t toOfs, offset_t fromOfs, offset_t size, ICopyFileProgress *progress)
  713. {
  714. // check to see if src and target are remote
  715. Owned<IFile> target = createIFile(dest);
  716. const size32_t buffersize = 0x100000;
  717. IFOmode omode = IFOwrite;
  718. if (toOfs==(offset_t)-1) {
  719. if (fromOfs==0) {
  720. copyFile(target,this,buffersize,progress);
  721. return;
  722. }
  723. omode = IFOcreate;
  724. toOfs = 0;
  725. }
  726. OwnedIFileIO targetIO = target->open(IFOwrite);
  727. if (!targetIO)
  728. throw MakeStringException(-1, "copyFile: target path '%s' could not be created", target->queryFilename());
  729. MemoryAttr mb;
  730. void * buffer = mb.allocate(buffersize);
  731. OwnedIFileIO sourceIO = open(IFOread);
  732. if (!sourceIO)
  733. throw MakeStringException(-1, "copySection: source '%s' not found", queryFilename());
  734. offset_t offset = 0;
  735. offset_t total;
  736. try
  737. {
  738. total = sourceIO->size();
  739. if (total<fromOfs)
  740. total = 0;
  741. else
  742. total -= fromOfs;
  743. if (total>size)
  744. total = size;
  745. while (offset<total)
  746. {
  747. size32_t got = sourceIO->read(fromOfs+offset, (buffersize>total-offset)?((size32_t)(total-offset)):buffersize, buffer);
  748. if (got == 0)
  749. break;
  750. targetIO->write(offset+toOfs, got, buffer);
  751. offset += got;
  752. if (progress && progress->onProgress(offset, total) != CFPcontinue)
  753. break;
  754. }
  755. }
  756. catch (IException *e)
  757. {
  758. StringBuffer s;
  759. s.append("copyFile target=").append(target->queryFilename()).append(" source=").append(queryFilename()).append("; read/write failure").append(": ");
  760. e->errorMessage(s);
  761. IException *e2 = MakeOsException(e->errorCode(), "%s", s.str());
  762. e->Release();
  763. throw e2;
  764. }
  765. }
  766. void CFile::copyTo(IFile *dest, size32_t buffersize, ICopyFileProgress *progress,bool usetmp)
  767. {
  768. doCopyFile(dest,this,buffersize,progress,NULL,usetmp);
  769. }
  770. void CFile::setReadOnly(bool ro)
  771. {
  772. #ifdef _WIN32
  773. if (!SetFileAttributes(filename, ro ? FILE_ATTRIBUTE_READONLY : FILE_ATTRIBUTE_NORMAL))
  774. {
  775. DWORD err = GetLastError();
  776. if ( (err!=ERROR_FILE_NOT_FOUND) && (err!=ERROR_PATH_NOT_FOUND) )
  777. throw MakeOsException(err, "CFile::setReadOnly %s", filename.get());
  778. }
  779. #else
  780. struct stat info;
  781. if (stat(filename, &info) != 0)
  782. throw MakeErrnoException("CFile::setReadOnly() %s", filename.get());
  783. // not sure correct but consistant with isReadOnly
  784. if (ro)
  785. info.st_mode &= ~(S_IWUSR|S_IWGRP|S_IWOTH);
  786. else
  787. info.st_mode |= (S_IWUSR|S_IWGRP|S_IWOTH);
  788. chmod(filename, info.st_mode);
  789. #endif
  790. }
  791. offset_t CFile::size()
  792. {
  793. if (stdIoHandle(filename)>=0)
  794. return 0; // dummy value
  795. #ifdef _WIN32
  796. WIN32_FILE_ATTRIBUTE_DATA info;
  797. if (GetFileAttributesEx(filename, GetFileExInfoStandard, &info) != 0) {
  798. LARGE_INTEGER x;
  799. x.LowPart = info.nFileSizeLow;
  800. x.HighPart = info.nFileSizeHigh;
  801. return (offset_t)x.QuadPart;
  802. }
  803. #else
  804. struct stat info;
  805. if (stat(filename, &info) == 0)
  806. return info.st_size;
  807. #endif
  808. #if 0
  809. // we could try opening but I don't think needed
  810. Owned<IFileIO>io = openShared(IFOread,IFSHfull);
  811. if (io)
  812. return io->size();
  813. #endif
  814. return (offset_t)-1;
  815. }
  816. bool CFile::setCompression(bool set)
  817. {
  818. #ifdef _WIN32
  819. DWORD attr=::GetFileAttributes(filename.get());
  820. if(attr==-1)
  821. throw MakeOsException(::GetLastError(), "CFile::setCompression %s", filename.get());
  822. if (((attr & FILE_ATTRIBUTE_COMPRESSED) != 0) == set)
  823. return true;
  824. HANDLE handle=::CreateFile(filename.get(),GENERIC_READ|GENERIC_WRITE,0,NULL,OPEN_EXISTING,FILE_FLAG_BACKUP_SEMANTICS,NULL);
  825. if(handle==INVALID_HANDLE_VALUE)
  826. throw MakeOsException(::GetLastError(), "CFile::setCompression %s", filename.get());
  827. USHORT compression=set ? COMPRESSION_FORMAT_DEFAULT : COMPRESSION_FORMAT_NONE;
  828. DWORD bytes;
  829. if(::DeviceIoControl(handle, FSCTL_SET_COMPRESSION, &compression, sizeof(compression), NULL, 0, &bytes, NULL))
  830. {
  831. ::CloseHandle(handle);
  832. return true;
  833. }
  834. DWORD err=::GetLastError();
  835. ::CloseHandle(handle);
  836. throw MakeOsException(err, "CFile::setCompression %s", filename.get());
  837. #else
  838. return false;
  839. #endif
  840. }
  841. offset_t CFile::compressedSize()
  842. {
  843. #ifdef _WIN32
  844. DWORD hi, lo=::GetCompressedFileSize(filename.get(),&hi), err;
  845. if(lo==INVALID_FILE_SIZE && (err=::GetLastError())!=NO_ERROR)
  846. {
  847. if ( (err!=ERROR_FILE_NOT_FOUND) && (err!=ERROR_PATH_NOT_FOUND) )
  848. throw MakeOsException(err,"CFile::compressedSize %s", filename.get());
  849. return -1;
  850. }
  851. return makeint64(hi,lo);
  852. #else
  853. return size();
  854. #endif
  855. }
  856. class CDiscretionaryFileLock: public CInterface, implements IDiscretionaryLock
  857. {
  858. bool locked;
  859. bool excllock;
  860. Linked<IFile> file;
  861. Linked<IFileIO> fileio;
  862. HANDLE handle;
  863. public:
  864. CDiscretionaryFileLock(IFile *_file)
  865. : file(_file)
  866. {
  867. excllock = false;
  868. locked = false;
  869. #ifdef _WIN32
  870. handle=::CreateFile(file->queryFilename(),GENERIC_READ,FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE,NULL,OPEN_EXISTING,FILE_FLAG_BACKUP_SEMANTICS,NULL);
  871. if(handle==INVALID_HANDLE_VALUE) {
  872. handle = NULLFILE;
  873. throw MakeOsException(GetLastError(),"CDiscretionaryFileLock::openhandle %s", file->queryFilename());
  874. }
  875. #else
  876. handle = _lopen(file->queryFilename(), O_RDONLY, 0);
  877. if (handle == -1) {
  878. handle = NULLFILE;
  879. throw MakeErrnoException(errno, "CDiscretionaryFileLock::openhandle %s",file->queryFilename());
  880. }
  881. #endif
  882. }
  883. CDiscretionaryFileLock(IFileIO *_fileio)
  884. : fileio(_fileio)
  885. {
  886. excllock = false;
  887. locked = false;
  888. handle = NULLFILE;
  889. CFileIO *cfileio = QUERYINTERFACE(_fileio,CFileIO);
  890. if (cfileio)
  891. handle = cfileio->queryHandle();
  892. if (handle==NULLFILE)
  893. throw MakeStringException(-1,"CDiscretionaryFileLock - invalid parameter");
  894. }
  895. ~CDiscretionaryFileLock()
  896. {
  897. if (locked)
  898. unlock();
  899. if ((handle!=NULLFILE)&&!fileio.get())
  900. #ifdef _WIN32
  901. CloseHandle(handle);
  902. #else
  903. _lclose(handle);
  904. #endif
  905. }
  906. IMPLEMENT_IINTERFACE;
  907. virtual bool isLocked() { return (handle!=NULLFILE); }
  908. virtual bool isExclusiveLocked() { return excllock; }
  909. bool lock(bool exclusive=true, unsigned timeout=INFINITE)
  910. {
  911. if (locked) {
  912. if (exclusive) {
  913. if (excllock)
  914. return true;
  915. unlock(); // have to unlock - unfortunate as means no window-less change mode
  916. }
  917. else {
  918. if (!excllock)
  919. return true;
  920. unlock(); // have to unlock - unfortunate as means no window-less change mode
  921. }
  922. }
  923. unsigned interval = 1;
  924. unsigned start = msTick();
  925. loop {
  926. #ifdef _WIN32
  927. OVERLAPPED overlapped;
  928. memset(&overlapped,0,sizeof(overlapped));
  929. if (LockFileEx(handle,
  930. (exclusive?LOCKFILE_EXCLUSIVE_LOCK:0)|((timeout==INFINITE)?0:LOCKFILE_FAIL_IMMEDIATELY),
  931. 0,1,0,&overlapped))
  932. break;
  933. DWORD err = ::GetLastError();
  934. if (err!=ERROR_LOCK_VIOLATION)
  935. throw MakeOsException(err, "CDiscretionaryFileLock::lock");
  936. #else
  937. if (setShareLock(handle,exclusive?IFSHnone:IFSHread))
  938. break;
  939. #endif
  940. unsigned now = msTick();
  941. if (now-start>=timeout)
  942. return false;
  943. if (interval>timeout-now+start)
  944. interval = timeout-now+start;
  945. Sleep(interval);
  946. interval *= 2;
  947. if (interval>1000)
  948. interval = 1000;
  949. }
  950. excllock = exclusive;
  951. locked = true;
  952. return true;
  953. }
  954. void unlock()
  955. {
  956. if (handle!=NULLFILE) {
  957. #ifdef _WIN32
  958. OVERLAPPED overlapped;
  959. memset(&overlapped,0,sizeof(overlapped));
  960. if (!UnlockFileEx(handle,0,1,0,&overlapped))
  961. throw MakeOsException(::GetLastError(), "CDiscretionaryFileLock::unlockhandle");
  962. #else
  963. setShareLock(handle,IFSHfull);
  964. #endif
  965. }
  966. }
  967. };
  968. IDiscretionaryLock *createDiscretionaryLock(IFile *file)
  969. {
  970. return new CDiscretionaryFileLock(file);
  971. }
  972. IDiscretionaryLock *createDiscretionaryLock(IFileIO *fileio)
  973. {
  974. return new CDiscretionaryFileLock(fileio);
  975. }
  976. unsigned CFile::getCRC()
  977. {
  978. if (stdIoHandle(filename)>=0)
  979. return 0; // dummy value
  980. unsigned crc=~0;
  981. MemoryAttr ma;
  982. void *buf = ma.allocate(0x10000);
  983. Owned<IFileIO> fileio = open(IFOread);
  984. if (fileio) {
  985. offset_t pos=0;
  986. loop {
  987. size32_t rd = fileio->read(pos,0x10000,buf);
  988. if (!rd)
  989. break;
  990. crc=crc32((const char *)buf,rd,crc);
  991. pos += rd;
  992. }
  993. }
  994. return ~crc;
  995. }
  996. //---------------------------------------------------------------------------
  997. static Linked<IPasswordProvider> passwordProvider;
  998. MODULE_INIT(INIT_PRIORITY_JFILE)
  999. {
  1000. return true;
  1001. }
  1002. MODULE_EXIT()
  1003. {
  1004. passwordProvider.clear();
  1005. }
  1006. static bool parseShare(const char *filename,IpAddress &machine,StringBuffer &share)
  1007. { // windows share parsing
  1008. if (!filename||!isPathSepChar(filename[0])||(filename[0]!=filename[1]))
  1009. return false;
  1010. const char *start = filename+2;
  1011. const char * end = start;
  1012. while (*end && !isPathSepChar(*end))
  1013. end++;
  1014. if (!*end)
  1015. return false;
  1016. StringBuffer ipText(end-start, start);
  1017. machine.ipset(ipText.str());
  1018. end++;
  1019. while (*end && !isPathSepChar(*end))
  1020. end++;
  1021. start = filename;
  1022. while (start!=end) {
  1023. if (*start=='/')
  1024. share.append('\\');
  1025. else
  1026. share.append(*start);
  1027. start++;
  1028. }
  1029. return true;
  1030. }
  1031. static CriticalSection connectcrit;
  1032. static bool connectToExternalDrive(const char * const filename)
  1033. {
  1034. CriticalBlock block(connectcrit);
  1035. if (!passwordProvider)
  1036. return false;
  1037. #ifdef _WIN32
  1038. StringBuffer share, username, password;
  1039. IpAddress ip;
  1040. if (!parseShare(filename,ip,share))
  1041. return false;
  1042. if (!passwordProvider->getPassword(ip, username, password))
  1043. return false;
  1044. // first see if connected
  1045. char buf[255];
  1046. buf[0] = 0;
  1047. DWORD len = sizeof(buf);
  1048. DWORD err = WNetGetUser(share.str(),buf,&len);
  1049. if ((err==0)&&(stricmp(username.str(),buf)==0)) {
  1050. // check can access share as well
  1051. for (unsigned i=0;i<10;i++) {
  1052. DWORD ret = (DWORD)GetFileAttributes(share.str());
  1053. DWORD err = (ret==(DWORD)-1)?GetLastError():0;
  1054. if (err!=ERROR_IO_INCOMPLETE)
  1055. break;
  1056. Sleep(100*i);
  1057. }
  1058. if (err==0)
  1059. return true;
  1060. }
  1061. for (unsigned retry=0;retry<5;retry++) {
  1062. NETRESOURCE res;
  1063. memset(&res, 0, sizeof(res));
  1064. res.dwType = RESOURCETYPE_DISK;
  1065. res.lpRemoteName = (char *)share.str();
  1066. err = WNetAddConnection2(&res, password.str(), username.str(), 0);
  1067. if (err) {
  1068. if (err==ERROR_SESSION_CREDENTIAL_CONFLICT) {
  1069. WNetCancelConnection2(res.lpRemoteName, 0, false);
  1070. err = WNetAddConnection2(&res, password.str(), username.str(), 0);
  1071. }
  1072. if (err)
  1073. ERRLOG("WNetAddConnection2(%d): connecting to %s, User: %s",err,res.lpRemoteName,username.str());
  1074. }
  1075. if (err==0)
  1076. return true;
  1077. Sleep(retry*100);
  1078. }
  1079. #endif
  1080. return false;
  1081. }
  1082. static void disconnectFromExternalDrive(const char * const filename)
  1083. {
  1084. #ifdef _WIN32
  1085. CriticalBlock block(connectcrit);
  1086. StringBuffer share;
  1087. IpAddress ip;
  1088. if (!parseShare(filename,ip,share))
  1089. return;
  1090. if (share.length()&&isShareChar(share.charAt(share.length())))
  1091. WNetCancelConnection2((char *)share.str(), 0, 0);
  1092. #else
  1093. //TODO: fill in for linux
  1094. #endif
  1095. }
  1096. #ifdef _WIN32
  1097. class CWindowsRemoteFile : public CInterface, implements IFile
  1098. {
  1099. IFile *ifile;
  1100. StringAttr filename;
  1101. bool connected;
  1102. inline bool connect()
  1103. {
  1104. if (!connected)
  1105. connected = connectToExternalDrive(filename);
  1106. return connected;
  1107. }
  1108. public:
  1109. CWindowsRemoteFile(const char * _filename)
  1110. : filename(_filename)
  1111. {
  1112. connected = false;
  1113. ifile = new CFile(_filename);
  1114. }
  1115. ~CWindowsRemoteFile()
  1116. {
  1117. ifile->Release();
  1118. #ifdef REMOTE_DISCONNECT_ON_DESTRUCTOR
  1119. if (connected)
  1120. disconnectFromExternalDrive(filename);
  1121. #endif
  1122. }
  1123. IMPLEMENT_IINTERFACE
  1124. virtual bool exists()
  1125. {
  1126. connect();
  1127. return ifile->exists();
  1128. }
  1129. virtual bool getTime(CDateTime * createTime, CDateTime * modifiedTime, CDateTime * accessedTime)
  1130. {
  1131. connect();
  1132. return ifile->getTime(createTime, modifiedTime, accessedTime);
  1133. }
  1134. virtual bool setTime(const CDateTime * createTime, const CDateTime * modifiedTime, const CDateTime * accessedTime)
  1135. {
  1136. connect();
  1137. if (ifile->setTime(createTime, modifiedTime, accessedTime))
  1138. return true;
  1139. if (connected||!connect())
  1140. return false;
  1141. return ifile->setTime(createTime, modifiedTime, accessedTime);
  1142. }
  1143. virtual fileBool isDirectory()
  1144. {
  1145. connect();
  1146. fileBool ok = ifile->isDirectory();
  1147. if (ok == notFound && !connected && connect())
  1148. ok = ifile->isDirectory();
  1149. return ok;
  1150. }
  1151. virtual fileBool isFile()
  1152. {
  1153. connect();
  1154. fileBool ok = ifile->isFile();
  1155. if (ok == notFound && !connected && connect())
  1156. ok = ifile->isFile();
  1157. return ok;
  1158. }
  1159. virtual fileBool isReadOnly()
  1160. {
  1161. connect();
  1162. fileBool ok = ifile->isReadOnly();
  1163. if (ok == notFound && !connected && connect())
  1164. ok = ifile->isFile();
  1165. return ok;
  1166. }
  1167. virtual IFileIO * open(IFOmode mode)
  1168. {
  1169. connect();
  1170. return ifile->open(mode);
  1171. }
  1172. virtual IFileIO * openShared(IFOmode mode,IFSHmode shared)
  1173. {
  1174. connect();
  1175. return ifile->openShared(mode,shared);
  1176. }
  1177. virtual IFileAsyncIO * openAsync(IFOmode mode)
  1178. {
  1179. connect();
  1180. return ifile->openAsync(mode);
  1181. }
  1182. virtual const char * queryFilename()
  1183. {
  1184. return ifile->queryFilename();
  1185. }
  1186. virtual bool remove()
  1187. {
  1188. connect();
  1189. unsigned attempt=0; \
  1190. return ifile->remove();
  1191. }
  1192. virtual void rename(const char *newname)
  1193. {
  1194. connect();
  1195. StringBuffer path;
  1196. splitDirTail(filename,path);
  1197. StringBuffer newdir;
  1198. const char *tail = splitDirTail(newname,newdir);
  1199. if ((newname[0]=='\\')&&(newname[1]=='\\')) { // rename to remote
  1200. if (path.length()&&newdir.length()) {
  1201. if (strcmp(newdir.str(),path.str())!=0)
  1202. WARNLOG("CWindowsRemoteFile '%s' to '%s' : directory mismatch",filename.get(),newname);
  1203. }
  1204. newname = tail; // just rename using tail
  1205. }
  1206. ifile->rename(newname);
  1207. path.append(tail);
  1208. filename.set(path);
  1209. }
  1210. virtual void move(const char *newName)
  1211. {
  1212. connect();
  1213. ifile->move(newName);
  1214. filename.set(ifile->queryFilename());
  1215. }
  1216. virtual void setReadOnly(bool ro)
  1217. {
  1218. connect();
  1219. ifile->setReadOnly(ro);
  1220. }
  1221. virtual offset_t size()
  1222. {
  1223. connect();
  1224. offset_t ret=ifile->size();
  1225. if ((ret==(offset_t)-1)&&!connected&&connect())
  1226. ret=ifile->size();
  1227. return ret;
  1228. }
  1229. virtual bool setCompression(bool set)
  1230. {
  1231. connect();
  1232. return ifile->setCompression(set);
  1233. }
  1234. virtual offset_t compressedSize()
  1235. {
  1236. connect();
  1237. return ifile->compressedSize();
  1238. }
  1239. bool fastCopyFile(CWindowsRemoteFile &target, size32_t buffersize, ICopyFileProgress *progress)
  1240. {
  1241. #ifdef _WIN32
  1242. CFile *src = QUERYINTERFACE(ifile,CFile);
  1243. if (!src)
  1244. return false;
  1245. CFile *dst = QUERYINTERFACE(target.ifile,CFile);
  1246. if (!dst)
  1247. return false;
  1248. target.isFile(); // encourage connect on target
  1249. connect();
  1250. return src->fastCopyFile(*dst, buffersize, progress);
  1251. #else
  1252. return false;
  1253. #endif
  1254. }
  1255. bool fastCopyFile(CFile &target, size32_t buffersize, ICopyFileProgress *progress)
  1256. {
  1257. #ifdef _WIN32
  1258. CFile *src = QUERYINTERFACE(ifile,CFile);
  1259. if (!src)
  1260. return false;
  1261. connect();
  1262. return src->fastCopyFile(target, buffersize, progress);
  1263. #else
  1264. return false;
  1265. #endif
  1266. }
  1267. bool fastCopyFileRev(CFile &src, size32_t buffersize, ICopyFileProgress *progress)
  1268. {
  1269. #ifdef _WIN32
  1270. CFile *dst = QUERYINTERFACE(ifile,CFile);
  1271. if (!dst)
  1272. return false;
  1273. connect();
  1274. return src.fastCopyFile(*dst, buffersize, progress);
  1275. #else
  1276. return false;
  1277. #endif
  1278. }
  1279. void copyTo(IFile *dest, size32_t buffersize, ICopyFileProgress *progress, bool usetmp)
  1280. {
  1281. doCopyFile(dest,this,buffersize,progress,NULL,usetmp);
  1282. }
  1283. bool createDirectory()
  1284. {
  1285. connect();
  1286. return localCreateDirectory(filename);
  1287. }
  1288. IDirectoryIterator *directoryFiles(const char *mask,bool sub,bool includedirs)
  1289. {
  1290. connect();
  1291. return ifile->directoryFiles(mask,sub,includedirs);
  1292. }
  1293. IDirectoryDifferenceIterator *monitorDirectory(
  1294. IDirectoryIterator *prev=NULL, // in (NULL means use current as baseline)
  1295. const char *mask=NULL,
  1296. bool sub=false,
  1297. bool includedirs=false,
  1298. unsigned checkinterval=60*1000,
  1299. unsigned timeout=(unsigned)-1,
  1300. Semaphore *abortsem=NULL) // returns NULL if timed out or abortsem signalled
  1301. {
  1302. connect();
  1303. return ifile->monitorDirectory(prev,mask,sub,includedirs,checkinterval,timeout,abortsem);
  1304. }
  1305. bool getInfo(bool &isdir,offset_t &size,CDateTime &modtime)
  1306. {
  1307. connect();
  1308. return ifile->getInfo(isdir,size,modtime);
  1309. }
  1310. unsigned getCRC()
  1311. {
  1312. connect();
  1313. return ifile->getCRC();
  1314. }
  1315. void setCreateFlags(unsigned cflags)
  1316. {
  1317. ifile->setCreateFlags(cflags);
  1318. }
  1319. void setShareMode(IFSHmode shmode)
  1320. {
  1321. ifile->setShareMode(shmode);
  1322. }
  1323. void copySection(const RemoteFilename &dest, offset_t toOfs, offset_t fromOfs, offset_t size, ICopyFileProgress *progress)
  1324. {
  1325. connect();
  1326. ifile->copySection(dest,toOfs,fromOfs,size,progress);
  1327. }
  1328. IMemoryMappedFile *openMemoryMapped(offset_t ofs, memsize_t len, bool write)
  1329. {
  1330. throw MakeStringException(-1,"Remote file cannot be memory mapped");
  1331. return NULL;
  1332. }
  1333. void treeCopyTo(IFile *dest,IpSubNet &subnet,IpAddress &resfrom, bool usetmp)
  1334. {
  1335. // no special action for windows
  1336. GetHostIp(resfrom);
  1337. copyTo(dest,0x100000,NULL,usetmp);
  1338. }
  1339. };
  1340. #endif
  1341. IFileIO *_createIFileIO(const void *buffer, unsigned sz, bool readOnly)
  1342. {
  1343. class CMemoryBufferIO : public CInterface, implements IFileIO
  1344. {
  1345. MemoryBuffer mb;
  1346. void *buffer;
  1347. size32_t sz;
  1348. bool readOnly;
  1349. public:
  1350. IMPLEMENT_IINTERFACE;
  1351. CMemoryBufferIO(const void *_buffer, unsigned _sz, bool _readOnly) : readOnly(_readOnly)
  1352. {
  1353. // JCSMORE - should probably have copy as option
  1354. mb.append(_sz, _buffer);
  1355. buffer = (void *)mb.toByteArray();
  1356. sz = mb.length();
  1357. }
  1358. virtual size32_t read(offset_t pos, size32_t len, void * data)
  1359. {
  1360. if (pos>sz)
  1361. throw MakeStringException(-1, "CMemoryBufferIO: read beyond end of buffer pos=%"I64F"d, len=%d, buffer length=%d", pos, len, mb.length());
  1362. if (pos+len > sz)
  1363. len = (size32_t)(sz-pos);
  1364. memcpy(data, (byte *)buffer+pos, len);
  1365. return len;
  1366. }
  1367. virtual offset_t size() { return sz; }
  1368. virtual size32_t write(offset_t pos, size32_t len, const void * data)
  1369. {
  1370. assertex(!readOnly);
  1371. if (pos+len>sz)
  1372. throw MakeStringException(-1, "CMemoryBufferIO: UNIMPLEMENTED, writing beyond buffer, pos=%"I64F"d, len=%d, buffer length=%d", pos, len, mb.length());
  1373. memcpy((byte *)buffer+pos, data, len);
  1374. return len;
  1375. }
  1376. virtual void setSize(offset_t size)
  1377. {
  1378. if (size > mb.length())
  1379. throw MakeStringException(-1, "CMemoryBufferIO: UNIMPLEMENTED, setting size %"I64F"d beyond end of buffer, buffer length=%d", size, mb.length());
  1380. mb.setLength((size32_t)size);
  1381. }
  1382. offset_t appendFile(IFile *file,offset_t pos,offset_t len)
  1383. {
  1384. if (!file)
  1385. return 0;
  1386. const size32_t buffsize = 0x10000;
  1387. void * buffer = mb.reserve(buffsize);
  1388. Owned<IFileIO> fileio = file->open(IFOread);
  1389. offset_t ret=0;
  1390. while (len) {
  1391. size32_t toread = ((len>=buffsize)||(len<0))?buffsize:(size32_t)len;
  1392. size32_t read = fileio->read(pos,toread,buffer);
  1393. if (read<buffsize)
  1394. mb.setLength(mb.length()+read-buffsize);
  1395. if (read==0)
  1396. break;
  1397. pos += read;
  1398. len -= read;
  1399. ret += read;
  1400. }
  1401. return ret;
  1402. }
  1403. };
  1404. return new CMemoryBufferIO(buffer, sz, readOnly);
  1405. }
  1406. IFileIO * createIFileI(unsigned len, const void * buffer)
  1407. {
  1408. return _createIFileIO((void *)buffer, len, true);
  1409. }
  1410. IFileIO * createIFileIO(unsigned len, void * buffer)
  1411. {
  1412. return _createIFileIO(buffer, len, false);
  1413. }
  1414. IFileIO * createIFileIO(StringBuffer & buffer)
  1415. {
  1416. return _createIFileIO((void *)buffer.toCharArray(), buffer.length(), true);
  1417. }
  1418. IFileIO * createIFileIO(MemoryBuffer & buffer)
  1419. {
  1420. return _createIFileIO(buffer.toByteArray(), buffer.length(), false);
  1421. }
  1422. //---------------------------------------------------------------------------
  1423. class jlib_decl CSequentialFileIO : public CFileIO
  1424. {
  1425. offset_t pos;
  1426. void checkPos(const char *fn,offset_t _pos)
  1427. {
  1428. if (_pos!=pos)
  1429. throw MakeStringException(-1, "CSequentialFileIO %s out of sequence (%"I64F"d,%"I64F"d)",fn,pos,_pos);
  1430. }
  1431. public:
  1432. CSequentialFileIO(HANDLE h,IFSHmode _sharemode)
  1433. : CFileIO(h,_sharemode)
  1434. {
  1435. pos = 0;
  1436. }
  1437. ~CSequentialFileIO()
  1438. {
  1439. }
  1440. IMPLEMENT_IINTERFACE
  1441. size32_t read(offset_t _pos, size32_t len, void * data)
  1442. {
  1443. checkPos("read",_pos);
  1444. #ifdef _WIN32
  1445. // Can't use checked_read because don't have the c fileno for it
  1446. DWORD numRead;
  1447. if (ReadFile(file,data,len,&numRead,NULL) == 0) {
  1448. DWORD err = GetLastError();
  1449. if (err==ERROR_BROKEN_PIPE) // windows returns this at end of pipe
  1450. return 0;
  1451. throw MakeOsException(GetLastError(),"CSequentialFileIO::read");
  1452. }
  1453. size32_t ret = (size32_t)numRead;
  1454. #else
  1455. size32_t ret = checked_read(file, data, len);
  1456. #endif
  1457. pos += ret;
  1458. return ret;
  1459. }
  1460. virtual size32_t write(offset_t _pos, size32_t len, const void * data)
  1461. {
  1462. checkPos("write",_pos);
  1463. size32_t ret;
  1464. #ifdef _WIN32
  1465. DWORD numWritten;
  1466. if (!WriteFile(file,data,len,&numWritten,NULL))
  1467. throw MakeOsException(GetLastError(),"CSequentialFileIO::write");
  1468. if (numWritten != len)
  1469. throw MakeOsException(DISK_FULL_EXCEPTION_CODE,"CSequentialFileIO::write");
  1470. ret = (size32_t)numWritten;
  1471. #else
  1472. ret = ::write(file,data,len);
  1473. if (ret==(size32_t)-1) {
  1474. PrintStackReport();
  1475. ERRLOG("errno(%d): %"I64F"d %u",errno,pos,len);
  1476. throw MakeErrnoException(errno,"CFileIO::write");
  1477. }
  1478. if (ret<len)
  1479. throw MakeOsException(DISK_FULL_EXCEPTION_CODE,"CSequentialFileIO::write");
  1480. #endif
  1481. pos += ret;
  1482. return ret;
  1483. }
  1484. };
  1485. IFileIO * CFile::openShared(IFOmode mode,IFSHmode share)
  1486. {
  1487. int stdh = stdIoHandle(filename);
  1488. HANDLE handle = openHandle(mode,share,false, stdh);
  1489. if (handle==NULLFILE)
  1490. return NULL;
  1491. #ifndef _WIN32
  1492. set_inherit(handle, false);
  1493. #endif
  1494. if (stdh>=0)
  1495. return new CSequentialFileIO(handle,share);
  1496. return new CFileIO(handle,share);
  1497. }
  1498. //---------------------------------------------------------------------------
  1499. extern jlib_decl IFileIO *createIFileIO(HANDLE handle)
  1500. {
  1501. return new CFileIO(handle,IFSHfull);
  1502. }
  1503. offset_t CFileIO::appendFile(IFile *file,offset_t pos,offset_t len)
  1504. {
  1505. if (!file)
  1506. return 0;
  1507. CriticalBlock procedure(cs);
  1508. MemoryAttr mb;
  1509. const size32_t buffsize = 0x10000;
  1510. void * buffer = mb.allocate(buffsize);
  1511. Owned<IFileIO> fileio = file->open(IFOread);
  1512. offset_t ret=0;
  1513. offset_t outp = size();
  1514. while (len) {
  1515. size32_t toread = ((len>=buffsize)||(len<0))?buffsize:(size32_t)len;
  1516. size32_t read = fileio->read(pos,toread,buffer);
  1517. if (read==0)
  1518. break;
  1519. size32_t wr = write(outp,read,buffer);
  1520. pos += read;
  1521. outp += wr;
  1522. len -= read;
  1523. ret += wr;
  1524. if (wr!=read)
  1525. break;
  1526. }
  1527. return ret;
  1528. }
  1529. #ifdef _WIN32
  1530. //-- Windows implementation -------------------------------------------------
  1531. CFileIO::CFileIO(HANDLE handle, IFSHmode _sharemode)
  1532. {
  1533. assertex(handle != NULLFILE);
  1534. throwOnError = false;
  1535. file = handle;
  1536. sharemode = _sharemode;
  1537. }
  1538. CFileIO::~CFileIO()
  1539. {
  1540. if (file != NULLFILE) CloseHandle(file);
  1541. file = NULLFILE;
  1542. }
  1543. offset_t CFileIO::size()
  1544. {
  1545. LARGE_INTEGER pos;
  1546. pos.LowPart = GetFileSize(file, (unsigned long *)&pos.HighPart);
  1547. if (pos.LowPart==-1) {
  1548. DWORD err = GetLastError();
  1549. if (err!=0)
  1550. throw MakeOsException(err,"CFileIO::size");
  1551. }
  1552. return pos.QuadPart;
  1553. }
  1554. size32_t CFileIO::read(offset_t pos, size32_t len, void * data)
  1555. {
  1556. CriticalBlock procedure(cs);
  1557. DWORD numRead;
  1558. setPos(pos);
  1559. if (ReadFile(file,data,len,&numRead,NULL) == 0)
  1560. throw MakeOsException(GetLastError(),"CFileIO::read");
  1561. return (size32_t)numRead;
  1562. }
  1563. void CFileIO::setPos(offset_t newPos)
  1564. {
  1565. LARGE_INTEGER tempPos;
  1566. tempPos.QuadPart = newPos;
  1567. tempPos.LowPart = SetFilePointer(file, tempPos.LowPart, &tempPos.HighPart, FILE_BEGIN);
  1568. }
  1569. size32_t CFileIO::write(offset_t pos, size32_t len, const void * data)
  1570. {
  1571. CriticalBlock procedure(cs);
  1572. DWORD numWritten;
  1573. setPos(pos);
  1574. if (!WriteFile(file,data,len,&numWritten,NULL))
  1575. throw MakeOsException(GetLastError(),"CFileIO::write");
  1576. if (numWritten != len)
  1577. throw MakeOsException(DISK_FULL_EXCEPTION_CODE,"CFileIO::write");
  1578. return (size32_t)numWritten;
  1579. }
  1580. void CFileIO::setSize(offset_t pos)
  1581. {
  1582. CriticalBlock procedure(cs);
  1583. setPos(pos);
  1584. if (!SetEndOfFile(file))
  1585. throw MakeOsException(GetLastError(), "CFileIO::setSize");
  1586. }
  1587. #else
  1588. //-- Unix implementation ----------------------------------------------------
  1589. // More errorno checking TBD
  1590. CFileIO::CFileIO(HANDLE handle, IFSHmode _sharemode)
  1591. {
  1592. assertex(handle != NULLFILE);
  1593. throwOnError = false;
  1594. file = handle;
  1595. sharemode = _sharemode;
  1596. }
  1597. CFileIO::~CFileIO()
  1598. {
  1599. if (file != NULLFILE) {
  1600. close(file);
  1601. file=NULLFILE;
  1602. }
  1603. }
  1604. offset_t CFileIO::size()
  1605. {
  1606. CriticalBlock procedure(cs);
  1607. offset_t savedPos = lseek(file,0,SEEK_CUR);
  1608. offset_t length = lseek(file,0,SEEK_END);
  1609. setPos(savedPos);
  1610. return length;
  1611. }
  1612. size32_t CFileIO::read(offset_t pos, size32_t len, void * data)
  1613. {
  1614. if (0==len) return 0;
  1615. return checked_pread(file, data, len, pos);
  1616. }
  1617. void CFileIO::setPos(offset_t newPos)
  1618. {
  1619. if (file != NULLFILE)
  1620. _llseek(file,newPos,SEEK_SET);
  1621. }
  1622. size32_t CFileIO::write(offset_t pos, size32_t len, const void * data)
  1623. {
  1624. size32_t ret = pwrite(file,data,len,pos);
  1625. if (ret==(size32_t)-1) {
  1626. throw MakeErrnoException(errno,"CFileIO::write");
  1627. }
  1628. if (ret<len)
  1629. throw MakeOsException(DISK_FULL_EXCEPTION_CODE,"CFileIO::write");
  1630. return ret;
  1631. }
  1632. void CFileIO::setSize(offset_t pos)
  1633. {
  1634. if (0 != ftruncate(file, pos))
  1635. throw MakeErrnoException(errno, "CFileIO::setSize");
  1636. }
  1637. #endif
  1638. //---------------------------------------------------------------------------
  1639. CFileRangeIO::CFileRangeIO(IFileIO * _io, offset_t _headerSize, offset_t _maxLength)
  1640. {
  1641. io.set(_io);
  1642. headerSize = _headerSize;
  1643. maxLength = _maxLength;
  1644. }
  1645. size32_t CFileRangeIO::read(offset_t pos, size32_t len, void * data)
  1646. {
  1647. if (pos + len > maxLength)
  1648. {
  1649. if (pos > maxLength)
  1650. pos = maxLength;
  1651. len = (size32_t)(maxLength - pos);
  1652. }
  1653. return io->read(pos+headerSize, len, data);
  1654. }
  1655. offset_t CFileRangeIO::size()
  1656. {
  1657. return maxLength;
  1658. }
  1659. size32_t CFileRangeIO::write(offset_t pos, size32_t len, const void * data)
  1660. {
  1661. if (pos + len > maxLength)
  1662. {
  1663. if (pos > maxLength)
  1664. pos = maxLength;
  1665. len = (size32_t)(maxLength - pos);
  1666. }
  1667. return io->write(pos+headerSize, len, data);
  1668. }
  1669. //--------------------------------------------------------------------------
  1670. offset_t CFileAsyncIO::appendFile(IFile *file,offset_t pos,offset_t len)
  1671. {
  1672. // will implemented if needed
  1673. UNIMPLEMENTED;
  1674. }
  1675. #ifdef _WIN32
  1676. //-- Windows implementation -------------------------------------------------
  1677. class CFileAsyncResult: public CInterface, implements IFileAsyncResult
  1678. {
  1679. protected: friend class CFileAsyncIO;
  1680. OVERLAPPED overlapped;
  1681. DWORD value;
  1682. DWORD wrsize;
  1683. CFileAsyncIO *parent;
  1684. public:
  1685. IMPLEMENT_IINTERFACE;
  1686. CFileAsyncResult(offset_t offset,size32_t _wrsize)
  1687. {
  1688. parent = NULL;
  1689. memset(&overlapped,0,sizeof(overlapped));
  1690. overlapped.hEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
  1691. overlapped.Offset = (DWORD)offset;
  1692. overlapped.OffsetHigh = (DWORD)(offset>>32);
  1693. value = (size32_t)-1;
  1694. wrsize = _wrsize;
  1695. }
  1696. ~CFileAsyncResult()
  1697. {
  1698. size32_t dummy;
  1699. getResult(dummy,true);
  1700. CloseHandle(overlapped.hEvent);
  1701. }
  1702. bool getResult(size32_t &ret,bool wait)
  1703. {
  1704. if (value==(size32_t)-1) {
  1705. if (parent) {
  1706. if (GetOverlappedResult(parent->file,&overlapped,&value,wait)==0) {
  1707. int err = GetLastError();
  1708. if (err==ERROR_IO_INCOMPLETE)
  1709. return false;
  1710. if (err!=ERROR_HANDLE_EOF) {
  1711. CriticalBlock block(parent->cs);
  1712. parent->results.zap(*this,true); // don't delete as array does not own
  1713. parent = NULL;
  1714. throw MakeOsException(err,"CFileAsyncResult::getResult");
  1715. }
  1716. value = 0;
  1717. }
  1718. CriticalBlock block(parent->cs);
  1719. parent->results.zap(*this,true); // don't delete as array does not own
  1720. parent = NULL;
  1721. }
  1722. else
  1723. return false;
  1724. }
  1725. ret = value;
  1726. if (value<wrsize)
  1727. throw MakeOsException(DISK_FULL_EXCEPTION_CODE,"CFileAsyncResult::getResult");
  1728. return true;
  1729. }
  1730. };
  1731. CFileAsyncIO::CFileAsyncIO(HANDLE handle, IFSHmode _sharemode)
  1732. {
  1733. assertex(handle != NULLFILE);
  1734. throwOnError = false;
  1735. file = handle;
  1736. sharemode = _sharemode;
  1737. }
  1738. CFileAsyncIO::~CFileAsyncIO()
  1739. {
  1740. CriticalBlock block(cs);
  1741. ForEachItemInRev(i,results) {
  1742. size32_t dummy;
  1743. results.item(i).getResult(dummy,true);
  1744. }
  1745. // wait for all outstanding results
  1746. if (file != NULLFILE) CloseHandle(file);
  1747. file = NULLFILE;
  1748. }
  1749. offset_t CFileAsyncIO::size()
  1750. {
  1751. LARGE_INTEGER pos;
  1752. pos.LowPart = GetFileSize(file, (unsigned long *)&pos.HighPart);
  1753. if (pos.LowPart==-1) {
  1754. DWORD err = GetLastError();
  1755. if (err!=0)
  1756. throw MakeOsException(GetLastError(),"CFileAsyncIO::size");
  1757. }
  1758. return pos.QuadPart;
  1759. }
  1760. size32_t CFileAsyncIO::read(offset_t pos, size32_t len, void * data)
  1761. {
  1762. Owned<IFileAsyncResult> res = readAsync(pos,len,data);
  1763. size32_t ret;
  1764. res->getResult(ret,true);
  1765. return ret;
  1766. }
  1767. size32_t CFileAsyncIO::write(offset_t pos, size32_t len, const void * data)
  1768. {
  1769. Owned<IFileAsyncResult> res = writeAsync(pos,len,data);
  1770. size32_t ret;
  1771. res->getResult(ret,true);
  1772. return ret;
  1773. }
  1774. void CFileAsyncIO::setSize(offset_t size)
  1775. {
  1776. LARGE_INTEGER tempPos;
  1777. tempPos.QuadPart = size;
  1778. tempPos.LowPart = SetFilePointer(file, tempPos.LowPart, &tempPos.HighPart, FILE_BEGIN);
  1779. if (!SetEndOfFile(file))
  1780. throw MakeOsException(GetLastError(), "CFileIO::setSize");
  1781. }
  1782. IFileAsyncResult *CFileAsyncIO::readAsync(offset_t pos, size32_t len, void * data)
  1783. {
  1784. CFileAsyncResult *res = new CFileAsyncResult(pos,0);
  1785. DWORD val;
  1786. if (ReadFile(file,data,len,&val,&res->overlapped) == 0) {
  1787. int err = GetLastError();
  1788. if (err == ERROR_HANDLE_EOF) { // bit weird
  1789. res->value = 0; // won't need to wait
  1790. }
  1791. else if (err == ERROR_IO_PENDING) {
  1792. CriticalBlock block(cs);
  1793. res->parent = this;
  1794. results.append(*res);
  1795. }
  1796. else
  1797. throw MakeOsException(GetLastError(),"CFileIO::readAsync");
  1798. }
  1799. else {
  1800. res->value = val; // won't need to wait
  1801. }
  1802. return res;
  1803. }
  1804. IFileAsyncResult *CFileAsyncIO::writeAsync(offset_t pos, size32_t len, const void * data)
  1805. {
  1806. CFileAsyncResult *res = new CFileAsyncResult(pos,len);
  1807. DWORD val;
  1808. if (WriteFile(file,data,len,&val,&res->overlapped) == 0) {
  1809. int err = GetLastError();
  1810. if (err != ERROR_IO_PENDING)
  1811. throw MakeOsException(GetLastError(),"CFileIO::writeAsync");
  1812. CriticalBlock block(cs);
  1813. res->parent = this;
  1814. results.append(*res);
  1815. }
  1816. else {
  1817. res->value = val; // wont need to wait
  1818. }
  1819. return res;
  1820. }
  1821. #else
  1822. //-- Unix implementation ----------------------------------------------------
  1823. class CFileAsyncResult: public CInterface, implements IFileAsyncResult
  1824. {
  1825. protected:
  1826. friend class CFileAsyncIO;
  1827. DWORD value;
  1828. DWORD wrsize;
  1829. aiocb cb;
  1830. public:
  1831. IMPLEMENT_IINTERFACE;
  1832. CFileAsyncResult(offset_t offset,size32_t _wrsize)
  1833. {
  1834. value = (size32_t)-1;
  1835. wrsize = _wrsize;
  1836. }
  1837. ~CFileAsyncResult()
  1838. {
  1839. size32_t dummy;
  1840. getResult(dummy,true);
  1841. }
  1842. bool getResult(size32_t &ret,bool wait)
  1843. {
  1844. if (value==(size32_t)-1) {
  1845. loop {
  1846. int aio_errno = aio_error(&cb);
  1847. if (aio_errno==ECANCELED)
  1848. return false;
  1849. if (aio_errno != EINPROGRESS)
  1850. {
  1851. if (aio_errno)
  1852. throw MakeErrnoException(aio_errno,"CFileAsyncResult::getResult");
  1853. value = aio_return(&cb);
  1854. if (value<wrsize)
  1855. throw MakeOsException(DISK_FULL_EXCEPTION_CODE,"CFileAsyncResult::getResult");
  1856. break;
  1857. }
  1858. if (!wait)
  1859. return false;
  1860. loop {
  1861. struct timespec timeout;
  1862. timeout.tv_sec = 60*60*24; // a long time
  1863. timeout.tv_nsec = 0;
  1864. aiocb * cb_list[1];
  1865. cb_list[0] = &cb;
  1866. int rc = aio_suspend(cb_list, 1, &timeout);
  1867. if ((rc == 0)||(errno != EAGAIN))
  1868. break;
  1869. if (rc==ECANCELED)
  1870. return false;
  1871. }
  1872. }
  1873. }
  1874. ret = value;
  1875. return true;
  1876. }
  1877. };
  1878. CFileAsyncIO::CFileAsyncIO(HANDLE handle, IFSHmode _sharemode)
  1879. {
  1880. assertex(handle != NULLFILE);
  1881. throwOnError = false;
  1882. file = handle;
  1883. sharemode = _sharemode;
  1884. }
  1885. CFileAsyncIO::~CFileAsyncIO()
  1886. {
  1887. if (file != NULLFILE) {
  1888. aio_cancel(file,NULL);
  1889. _lclose(file);
  1890. }
  1891. file=NULLFILE;
  1892. }
  1893. offset_t CFileAsyncIO::size()
  1894. {
  1895. CriticalBlock procedure(cs);
  1896. offset_t savedPos = _llseek(file,0,SEEK_CUR);
  1897. offset_t length = _llseek(file,0,SEEK_END);
  1898. _llseek(file, savedPos, SEEK_SET);
  1899. return length;
  1900. }
  1901. size32_t CFileAsyncIO::read(offset_t pos, size32_t len, void * data)
  1902. {
  1903. CriticalBlock procedure(cs);
  1904. _llseek(file,pos,SEEK_SET);
  1905. size32_t ret = _lread(file,data,len);
  1906. if (ret==(size32_t)-1)
  1907. throw MakeErrnoException(errno,"CFileAsyncIO::read");
  1908. return ret;
  1909. }
  1910. size32_t CFileAsyncIO::write(offset_t pos, size32_t len, const void * data)
  1911. {
  1912. CriticalBlock procedure(cs);
  1913. _llseek(file,pos,SEEK_SET);
  1914. size32_t ret = _lwrite(file,data,len);
  1915. if (ret==(size32_t)-1)
  1916. throw MakeErrnoException(errno,"CFileAsyncIO::write");
  1917. return ret;
  1918. }
  1919. void CFileAsyncIO::setSize(offset_t pos)
  1920. {
  1921. CriticalBlock procedure(cs);
  1922. if ((file != NULLFILE)&&(0 != ftruncate(file, pos)))
  1923. throw MakeErrnoException(errno, "CFileIO::setSize");
  1924. }
  1925. IFileAsyncResult *CFileAsyncIO::readAsync(offset_t pos, size32_t len, void * data)
  1926. {
  1927. CFileAsyncResult *res = new CFileAsyncResult(pos,0);
  1928. bzero( &(res->cb), sizeof (struct aiocb));
  1929. res->cb.aio_fildes = file;
  1930. res->cb.aio_offset = pos;
  1931. res->cb.aio_buf = data;
  1932. res->cb.aio_nbytes = len;
  1933. res->cb.aio_sigevent.sigev_notify = SIGEV_NONE;
  1934. int retval = aio_read(&(res->cb));
  1935. if (retval==-1)
  1936. throw MakeErrnoException(errno,"CFileAsyncIO::readAsync");
  1937. return res;
  1938. }
  1939. IFileAsyncResult *CFileAsyncIO::writeAsync(offset_t pos, size32_t len, const void * data)
  1940. {
  1941. CFileAsyncResult *res = new CFileAsyncResult(pos,len);
  1942. bzero( &(res->cb), sizeof (struct aiocb));
  1943. res->cb.aio_fildes = file;
  1944. res->cb.aio_offset = pos;
  1945. res->cb.aio_buf = (void*)data;
  1946. res->cb.aio_nbytes = len;
  1947. res->cb.aio_sigevent.sigev_signo = SIGUSR1;
  1948. res->cb.aio_sigevent.sigev_notify = SIGEV_NONE;
  1949. res->cb.aio_sigevent.sigev_value.sival_ptr = (void*)res;
  1950. int retval = aio_write(&(res->cb));
  1951. if (retval==-1)
  1952. throw MakeErrnoException(errno,"CFileAsyncIO::writeAsync");
  1953. return res;
  1954. }
  1955. #endif
  1956. //---------------------------------------------------------------------------
  1957. CFileIOStream::CFileIOStream(IFileIO * _io)
  1958. {
  1959. io.set(_io);
  1960. curOffset = 0;
  1961. }
  1962. void CFileIOStream::flush()
  1963. {
  1964. }
  1965. size32_t CFileIOStream::read(size32_t len, void * data)
  1966. {
  1967. size32_t numRead = io->read(curOffset, len, data);
  1968. curOffset += numRead;
  1969. return numRead;
  1970. }
  1971. void CFileIOStream::seek(offset_t pos, IFSmode origin)
  1972. {
  1973. switch (origin)
  1974. {
  1975. case IFScurrent:
  1976. curOffset += pos;
  1977. break;
  1978. case IFSend:
  1979. curOffset = io->size() + pos;
  1980. break;
  1981. case IFSbegin:
  1982. curOffset = pos;
  1983. break;
  1984. }
  1985. }
  1986. offset_t CFileIOStream::size()
  1987. {
  1988. return io->size();
  1989. }
  1990. offset_t CFileIOStream::tell()
  1991. {
  1992. return curOffset;
  1993. }
  1994. size32_t CFileIOStream::write(size32_t len, const void * data)
  1995. {
  1996. size32_t numWritten = io->write(curOffset, len, data);
  1997. curOffset += numWritten;
  1998. return numWritten;
  1999. }
  2000. //---------------------------------------------------------------------------
  2001. class CBufferedFileIOStreamBase : public CBufferedIOStreamBase, implements IFileIOStream
  2002. {
  2003. protected:
  2004. virtual offset_t directSize() = 0;
  2005. offset_t curOffset;
  2006. public:
  2007. IMPLEMENT_IINTERFACE;
  2008. CBufferedFileIOStreamBase(unsigned bufSize) : CBufferedIOStreamBase(bufSize), curOffset(0) { }
  2009. virtual void flush() { doflush(); }
  2010. void seek(offset_t pos, IFSmode origin)
  2011. {
  2012. offset_t newOffset;
  2013. switch (origin)
  2014. {
  2015. case IFScurrent:
  2016. newOffset = tell() + pos;
  2017. break;
  2018. case IFSend:
  2019. newOffset = size() + pos;
  2020. break;
  2021. case IFSbegin:
  2022. newOffset = pos;
  2023. break;
  2024. }
  2025. if (reading)
  2026. {
  2027. // slightly weird but curoffset is end of buffer when reading
  2028. if ((newOffset >= curOffset-numInBuffer) && (newOffset <= curOffset))
  2029. {
  2030. curBufferOffset = (size32_t)(newOffset - (curOffset-numInBuffer));
  2031. return;
  2032. }
  2033. }
  2034. else
  2035. {
  2036. if ((newOffset >= curOffset) && (newOffset <= curOffset + numInBuffer))
  2037. {
  2038. curBufferOffset = (size32_t)(newOffset - curOffset);
  2039. return;
  2040. }
  2041. flush();
  2042. }
  2043. curOffset = newOffset;
  2044. numInBuffer = 0;
  2045. curBufferOffset = 0;
  2046. }
  2047. offset_t size()
  2048. {
  2049. offset_t curSize = directSize();
  2050. if (!reading)
  2051. curSize = std::max(curSize, curOffset + numInBuffer);
  2052. return curSize;
  2053. }
  2054. offset_t tell()
  2055. {
  2056. if (reading)
  2057. return curOffset - numInBuffer + curBufferOffset;
  2058. return curOffset + curBufferOffset;
  2059. }
  2060. size32_t read(size32_t len, void * data)
  2061. {
  2062. return CBufferedIOStreamBase::doread(len, data);
  2063. }
  2064. size32_t write(size32_t len, const void * data)
  2065. {
  2066. if (reading)
  2067. curOffset -= bytesRemaining();
  2068. return CBufferedIOStreamBase::dowrite(len, data);
  2069. }
  2070. };
  2071. class CBufferedFileIOStream : public CBufferedFileIOStreamBase
  2072. {
  2073. public:
  2074. CBufferedFileIOStream(IFileIO * _io, unsigned _bufferSize) : CBufferedFileIOStreamBase(_bufferSize), io(_io)
  2075. {
  2076. buffer = new byte[_bufferSize];
  2077. }
  2078. ~CBufferedFileIOStream()
  2079. {
  2080. flush();
  2081. delete [] buffer;
  2082. }
  2083. protected:
  2084. virtual void doflush()
  2085. {
  2086. if (!reading && numInBuffer)
  2087. {
  2088. try {
  2089. io->write(curOffset, numInBuffer, buffer);
  2090. }
  2091. catch (IException *) {
  2092. // if we get exception, clear buffer so doen't reoccur on destructor as well
  2093. numInBuffer = 0;
  2094. curBufferOffset = 0;
  2095. throw;
  2096. }
  2097. curOffset += curBufferOffset;
  2098. numInBuffer = 0;
  2099. curBufferOffset = 0;
  2100. }
  2101. }
  2102. virtual bool fillBuffer()
  2103. {
  2104. reading = true;
  2105. numInBuffer = io->read(curOffset, bufferSize, buffer);
  2106. curOffset += numInBuffer;
  2107. curBufferOffset = 0;
  2108. return numInBuffer!=0;
  2109. }
  2110. virtual size32_t directRead(size32_t len, void * data)
  2111. {
  2112. size32_t sz = io->read(curOffset,len,data);
  2113. curOffset += sz;
  2114. return sz;
  2115. }
  2116. virtual size32_t directWrite(size32_t len, const void * data)
  2117. {
  2118. size32_t sz = io->write(curOffset,len,data);
  2119. curOffset += sz;
  2120. return sz;
  2121. }
  2122. virtual offset_t directSize() { return io->size(); }
  2123. protected:
  2124. IFileIOAttr io;
  2125. };
  2126. //---------------------------------------------------------------------------
  2127. class CBufferedAsyncIOStream: public CBufferedFileIOStreamBase
  2128. {
  2129. Linked<IFileAsyncIO> io;
  2130. byte * blk1;
  2131. byte * blk2;
  2132. IFileAsyncResult *readasyncres;
  2133. IFileAsyncResult *writeasyncres;
  2134. bool readeof;
  2135. public:
  2136. IMPLEMENT_IINTERFACE
  2137. CBufferedAsyncIOStream(IFileAsyncIO * _io, size32_t _bufferSize)
  2138. : CBufferedFileIOStreamBase(_bufferSize/2), io(_io)
  2139. {
  2140. blk1 = new byte[bufferSize];
  2141. blk2 = new byte[bufferSize];
  2142. buffer = blk1;
  2143. readasyncres = NULL;
  2144. writeasyncres = NULL;
  2145. readeof = false;
  2146. minDirectSize = (size32_t)-1; // async always writes using buffer
  2147. }
  2148. ~CBufferedAsyncIOStream()
  2149. {
  2150. flush();
  2151. waitAsyncWrite();
  2152. waitAsyncRead();
  2153. delete [] blk1;
  2154. delete [] blk2;
  2155. }
  2156. void waitAsyncWrite()
  2157. {
  2158. if (writeasyncres) {
  2159. size32_t res;
  2160. writeasyncres->getResult(res,true);
  2161. writeasyncres->Release();
  2162. writeasyncres = NULL;
  2163. }
  2164. }
  2165. size32_t waitAsyncRead()
  2166. {
  2167. size32_t res = 0;
  2168. if (readasyncres) {
  2169. readasyncres->getResult(res,true);
  2170. readasyncres->Release();
  2171. readasyncres = NULL;
  2172. }
  2173. return res;
  2174. }
  2175. void primeAsyncRead(offset_t pos,size32_t size, void *dst)
  2176. {
  2177. assertex(!readasyncres);
  2178. readasyncres = io->readAsync(pos, size, dst);
  2179. }
  2180. void primeAsyncWrite(offset_t pos,size32_t size, const void *src)
  2181. {
  2182. assertex(!writeasyncres);
  2183. writeasyncres = io->writeAsync(pos, size, src);
  2184. }
  2185. // CBufferedFileIOStream overloads
  2186. virtual bool fillBuffer()
  2187. {
  2188. if (!reading) {
  2189. waitAsyncWrite();
  2190. reading = true;
  2191. }
  2192. if (readeof) {
  2193. numInBuffer = 0;
  2194. curBufferOffset = 0;
  2195. return false;
  2196. }
  2197. buffer=(buffer==blk1)?blk2:blk1;
  2198. if (readasyncres==NULL) // first time
  2199. primeAsyncRead(curOffset, bufferSize, buffer);
  2200. numInBuffer = waitAsyncRead();
  2201. curOffset += numInBuffer;
  2202. curBufferOffset = 0;
  2203. if (numInBuffer)
  2204. primeAsyncRead(curOffset, bufferSize, (buffer==blk1)?blk2:blk1);
  2205. else
  2206. readeof = true;
  2207. return !readeof;
  2208. }
  2209. virtual void doflush()
  2210. {
  2211. if (!reading && numInBuffer)
  2212. {
  2213. waitAsyncWrite();
  2214. primeAsyncWrite(curOffset, numInBuffer, buffer);
  2215. buffer=(buffer==blk1)?blk2:blk1;
  2216. curOffset += curBufferOffset;
  2217. numInBuffer = 0;
  2218. curBufferOffset = 0;
  2219. }
  2220. }
  2221. virtual size32_t directRead(size32_t len, void * data) { assertex(false); return 0; } // shouldn't get called
  2222. virtual size32_t directWrite(size32_t len, const void * data) { assertex(false); return 0; } // shouldn't get called
  2223. virtual offset_t directSize() { waitAsyncWrite(); return io->size(); }
  2224. };
  2225. //-- Helper routines --------------------------------------------------------
  2226. static inline size32_t doread(IFileIOStream * stream,void *dst, size32_t size)
  2227. {
  2228. size32_t toread=size;
  2229. while (toread)
  2230. {
  2231. int read = stream->read(toread, dst);
  2232. if (!read)
  2233. return size-toread;
  2234. toread -= read;
  2235. dst = (char *) dst + read;
  2236. }
  2237. return size;
  2238. }
  2239. CIOStreamReadWriteSeq::CIOStreamReadWriteSeq(IFileIOStream * _stream, offset_t _offset, size32_t _size)
  2240. {
  2241. stream.set(_stream);
  2242. // stream->setThrowOnError(true);
  2243. size = _size;
  2244. offset = _offset; // assumption that stream at correct location already
  2245. }
  2246. void CIOStreamReadWriteSeq::put(const void *src)
  2247. {
  2248. stream->write(size, src);
  2249. }
  2250. void CIOStreamReadWriteSeq::putn(const void *src, unsigned n)
  2251. {
  2252. stream->write(size*n, src);
  2253. }
  2254. void CIOStreamReadWriteSeq::flush()
  2255. {
  2256. stream->flush();
  2257. }
  2258. offset_t CIOStreamReadWriteSeq::getPosition()
  2259. {
  2260. return stream->tell();
  2261. }
  2262. bool CIOStreamReadWriteSeq::get(void *dst)
  2263. {
  2264. return doread(stream,dst,size)==size;
  2265. }
  2266. unsigned CIOStreamReadWriteSeq::getn(void *dst, unsigned n)
  2267. {
  2268. return doread(stream,dst,size*n)/size;
  2269. }
  2270. void CIOStreamReadWriteSeq::reset()
  2271. {
  2272. stream->seek(offset, IFSbegin);
  2273. }
  2274. //-- Helper routines --------------------------------------------------------
  2275. size32_t read(IFileIO * in, offset_t pos, size32_t len, MemoryBuffer & buffer)
  2276. {
  2277. const size32_t checkLengthLimit = 0x1000;
  2278. if (len >= checkLengthLimit)
  2279. {
  2280. //Don't allocate a stupid amount of memory....
  2281. offset_t fileLength = in->size();
  2282. if (pos > fileLength)
  2283. pos = fileLength;
  2284. if ((len == (size32_t)-1) || (pos + len > fileLength))
  2285. len = (size32_t)(fileLength - pos);
  2286. }
  2287. void * data = buffer.reserve(len);
  2288. size32_t lenRead = in->read(pos, len, data);
  2289. if (lenRead != len)
  2290. buffer.rewrite(buffer.length() - (len - lenRead));
  2291. return lenRead;
  2292. }
  2293. void copyFile(const char *target, const char *source, size32_t buffersize, ICopyFileProgress *progress)
  2294. {
  2295. OwnedIFile src = createIFile(source);
  2296. if (!src)
  2297. throw MakeStringException(-1, "copyFile: source '%s' not found", source);
  2298. OwnedIFile tgt = createIFile(target);
  2299. if (!tgt)
  2300. throw MakeStringException(-1, "copyFile: target path '%s' could not be created", target);
  2301. copyFile(tgt, src, buffersize,progress);
  2302. }
  2303. void copyFile(IFile * target, IFile * source, size32_t buffersize, ICopyFileProgress *progress)
  2304. {
  2305. source->copyTo(target,buffersize,progress);
  2306. }
  2307. void doCopyFile(IFile * target, IFile * source, size32_t buffersize, ICopyFileProgress *progress, ICopyFileIntercept *copyintercept, bool usetmp)
  2308. {
  2309. if (!buffersize)
  2310. buffersize = 0x100000;
  2311. #ifdef _WIN32
  2312. if (!usetmp) {
  2313. CFile *src = QUERYINTERFACE(source,CFile);
  2314. CFile *dst = QUERYINTERFACE(target,CFile);
  2315. if (src) {
  2316. if (dst) {
  2317. if (src->fastCopyFile(*dst, buffersize, progress))
  2318. return;
  2319. }
  2320. CWindowsRemoteFile *dst2 = QUERYINTERFACE(target,CWindowsRemoteFile);
  2321. if (dst2) {
  2322. if (dst2->fastCopyFileRev(*src, buffersize, progress))
  2323. return;
  2324. }
  2325. }
  2326. CWindowsRemoteFile *src2 = QUERYINTERFACE(source,CWindowsRemoteFile);
  2327. if (src2) {
  2328. if (dst) {
  2329. if (src2->fastCopyFile(*dst,buffersize, progress))
  2330. return;
  2331. }
  2332. CWindowsRemoteFile *dst2 = QUERYINTERFACE(target,CWindowsRemoteFile);
  2333. if (dst2) {
  2334. if (src2->fastCopyFile(*dst2, buffersize, progress))
  2335. return;
  2336. }
  2337. }
  2338. }
  2339. #endif
  2340. OwnedIFileIO sourceIO = source->open(IFOread);
  2341. if (!sourceIO)
  2342. throw MakeStringException(-1, "copyFile: source '%s' not found", source->queryFilename());
  2343. #ifdef __linux__
  2344. // this is not really needed in windows - if it is we will have to
  2345. // test the file extenstion - .exe, .bat
  2346. struct stat info;
  2347. if (stat(source->queryFilename(), &info) == 0) // cannot fail - exception would have been thrown above
  2348. target->setCreateFlags(info.st_mode&(S_IRUSR|S_IRGRP|S_IROTH|S_IWUSR|S_IWGRP|S_IWOTH|S_IXUSR|S_IXGRP|S_IXOTH));
  2349. #endif
  2350. Owned<IFileIO> targetIO;
  2351. Owned<IFile> tmpfile;
  2352. IFile *dest;
  2353. if (usetmp) {
  2354. StringBuffer tmpname;
  2355. makeTempCopyName(tmpname,target->queryFilename());
  2356. tmpfile.setown(createIFile(tmpname.str()));
  2357. dest = tmpfile;
  2358. }
  2359. else
  2360. dest = target;
  2361. targetIO.setown(dest->open(IFOcreate));
  2362. if (!targetIO)
  2363. throw MakeStringException(-1, "copyFile: target path '%s' could not be created", dest->queryFilename());
  2364. MemoryAttr mb;
  2365. void * buffer = copyintercept?NULL:mb.allocate(buffersize);
  2366. offset_t offset = 0;
  2367. offset_t total;
  2368. Owned<IException> exc;
  2369. try
  2370. {
  2371. if (progress)
  2372. total = sourceIO->size(); // only needed for progress
  2373. loop
  2374. {
  2375. size32_t got;
  2376. if (copyintercept) {
  2377. got = (size32_t)copyintercept->copy(sourceIO,targetIO,offset,buffersize);
  2378. if (got == 0)
  2379. break;
  2380. }
  2381. else {
  2382. got = sourceIO->read(offset, buffersize, buffer);
  2383. if (got == 0)
  2384. break;
  2385. targetIO->write(offset, got, buffer);
  2386. }
  2387. offset += got;
  2388. if (progress && progress->onProgress(offset, total) != CFPcontinue)
  2389. break;
  2390. }
  2391. targetIO.clear();
  2392. if (usetmp) {
  2393. StringAttr tail(pathTail(target->queryFilename()));
  2394. target->remove();
  2395. dest->rename(tail);
  2396. }
  2397. }
  2398. catch (IException *e)
  2399. {
  2400. // try to delete partial copy
  2401. StringBuffer s;
  2402. s.append("copyFile target=").append(dest->queryFilename()).append(" source=").append(source->queryFilename()).appendf("; read/write failure (%d): ",e->errorCode());
  2403. exc.setown(MakeStringException(e->errorCode(), "%s", s.str()));
  2404. e->Release();
  2405. EXCLOG(exc, "doCopyFile");
  2406. }
  2407. if (exc.get()) {
  2408. try {
  2409. sourceIO.clear();
  2410. }
  2411. catch (IException *e) {
  2412. EXCLOG(e, "doCopyFile closing source");
  2413. e->Release();
  2414. }
  2415. try {
  2416. targetIO.clear();
  2417. }
  2418. catch (IException *e) {
  2419. EXCLOG(e, "doCopyFile closing dest");
  2420. e->Release();
  2421. }
  2422. try {
  2423. dest->remove();
  2424. }
  2425. catch (IException *e) {
  2426. StringBuffer s;
  2427. EXCLOG(e, s.clear().append("Removing partial copy file: ").append(dest->queryFilename()).str());
  2428. e->Release();
  2429. }
  2430. throw exc.getClear();
  2431. }
  2432. CDateTime createTime, modifiedTime;
  2433. if (source->getTime(&createTime, &modifiedTime, NULL))
  2434. target->setTime(&createTime, &modifiedTime, NULL);
  2435. }
  2436. void makeTempCopyName(StringBuffer &tmpname,const char *destname)
  2437. {
  2438. // simple for the moment (maybe used uid later)
  2439. tmpname.append(destname).append("__");
  2440. #ifdef _WIN32
  2441. genUUID(tmpname,true);
  2442. #else
  2443. genUUID(tmpname);
  2444. #endif
  2445. tmpname.append(".tmp");
  2446. }
  2447. //---------------------------------------------------------------------------
  2448. #ifndef _WIN32
  2449. /// This code is dangerous - pass it arrays smaller than _MAX_xxx and it will write off the end even if you KNOW your strings would fit
  2450. // Should NOT use strncpy here
  2451. void _splitpath(const char *path, char *drive, char *dir, char *fname, char *ext)
  2452. {
  2453. strncpy(dir, path, (_MAX_DIR-1));
  2454. char* last = strrchr(dir, '.');
  2455. if (last != NULL)
  2456. {
  2457. if (strrchr(last, PATHSEPCHAR)==NULL)
  2458. {
  2459. strncpy(ext, last, (_MAX_EXT-1));
  2460. *last = '\0';
  2461. }
  2462. else
  2463. {
  2464. *ext = '\0';
  2465. }
  2466. }
  2467. else
  2468. {
  2469. *ext = '\0';
  2470. }
  2471. last = strrchr(dir, PATHSEPCHAR);
  2472. if (last != NULL)
  2473. {
  2474. strncpy(fname, ++last, (_MAX_FNAME-1));
  2475. *last = '\0';
  2476. }
  2477. else
  2478. {
  2479. strncpy(fname, dir, (_MAX_FNAME-1));
  2480. *dir = '\0';
  2481. }
  2482. *drive = '\0';
  2483. }
  2484. #endif
  2485. void splitFilename(const char * filename, StringBuffer * drive, StringBuffer * path, StringBuffer * tail, StringBuffer * ext)
  2486. {
  2487. char tdrive[_MAX_DRIVE];
  2488. char tdir[_MAX_DIR];
  2489. char ttail[_MAX_FNAME];
  2490. char text[_MAX_EXT];
  2491. ::_splitpath(filename, tdrive, tdir, ttail, text);
  2492. if (drive)
  2493. drive->append(tdrive);
  2494. if (path)
  2495. path->append(tdir);
  2496. if (tail)
  2497. tail->append(ttail);
  2498. if (ext)
  2499. ext->append(text);
  2500. }
  2501. StringBuffer &createUNCFilename(const char * filename, StringBuffer &UNC, bool useHostNames)
  2502. {
  2503. char buf[255];
  2504. #ifdef _WIN32
  2505. char *dummy;
  2506. GetFullPathName(filename, sizeof(buf), buf, &dummy);
  2507. if (buf[1]==':')
  2508. {
  2509. // MORE - assumes it's a local drive not a mapped one
  2510. UNC.append("\\\\");
  2511. if (useHostNames)
  2512. UNC.append(GetCachedHostName());
  2513. else
  2514. queryHostIP().getIpText(UNC);
  2515. UNC.append("\\").append((char)tolower(buf[0])).append(getShareChar()).append(buf+2);
  2516. }
  2517. else
  2518. {
  2519. assertex(buf[0]=='\\' && buf[1]=='\\');
  2520. UNC.append(buf);
  2521. }
  2522. return UNC;
  2523. #else
  2524. if (filename[0]=='/' && filename[1]=='/')
  2525. UNC.append(filename);
  2526. else
  2527. {
  2528. UNC.append("//");
  2529. if (useHostNames)
  2530. UNC.append(GetCachedHostName());
  2531. else
  2532. queryHostIP().getIpText(UNC);
  2533. if (*filename != '/')
  2534. {
  2535. getcwd(buf, sizeof(buf));
  2536. UNC.append(buf).append("/");
  2537. }
  2538. UNC.append(filename);
  2539. }
  2540. return UNC;
  2541. #endif
  2542. }
  2543. bool splitUNCFilename(const char * filename, StringBuffer * machine, StringBuffer * path, StringBuffer * tail, StringBuffer * ext)
  2544. {
  2545. if (!filename || !isPathSepChar(filename[0]) || !isPathSepChar(filename[1]))
  2546. return false;
  2547. const char * cur = filename+2;
  2548. while (*cur && !isPathSepChar(*cur))
  2549. cur++;
  2550. if (!*cur)
  2551. return false;
  2552. const char * startPath = cur;
  2553. const char * lastExt = NULL;
  2554. const char * startTail = NULL;
  2555. char next;
  2556. while ((next = *cur) != 0)
  2557. {
  2558. if (isPathSepChar(next))
  2559. {
  2560. lastExt = NULL;
  2561. startTail = cur+1;
  2562. }
  2563. else if (next == '.')
  2564. lastExt = cur;
  2565. cur++;
  2566. }
  2567. assertex(startTail);
  2568. if (machine)
  2569. machine->append(startPath-filename, filename);
  2570. if (path)
  2571. path->append(startTail - startPath, startPath);
  2572. if (lastExt)
  2573. {
  2574. if (tail)
  2575. tail->append(lastExt - startTail, startTail);
  2576. if (ext)
  2577. ext->append(lastExt);
  2578. }
  2579. else
  2580. {
  2581. if (tail)
  2582. tail->append(startTail);
  2583. }
  2584. return true;
  2585. }
  2586. /**
  2587. * Ensure the filename has desired extension.
  2588. * If it has no extension, add the desired extension, return true.
  2589. * If it has an extension different from the desiredExtension, return false.
  2590. * Otherwise, return true.
  2591. */
  2592. bool ensureFileExtension(StringBuffer& filename, const char* desiredExtension)
  2593. {
  2594. char drive[_MAX_DRIVE];
  2595. char dir[_MAX_DIR];
  2596. char fname[_MAX_FNAME];
  2597. char ext[_MAX_EXT];
  2598. _splitpath(filename.str(), drive, dir, fname, ext);
  2599. if (ext[0]==0)
  2600. {
  2601. filename.append(desiredExtension);
  2602. return true;
  2603. }
  2604. if (strcmp(ext,desiredExtension)!=0)
  2605. return false;
  2606. return true;
  2607. }
  2608. /* Get full file name. If noExtension is true, the extesion (if any) will be trimmed */
  2609. StringBuffer& getFullFileName(StringBuffer& filename, bool noExtension)
  2610. {
  2611. char drive[_MAX_DRIVE];
  2612. char dir[_MAX_DIR];
  2613. char fname[_MAX_FNAME];
  2614. char ext[_MAX_EXT];
  2615. _splitpath(filename.str(), drive, dir, fname, ext);
  2616. filename.clear();
  2617. filename.append(drive).append(dir).append(fname);
  2618. if (!noExtension)
  2619. filename.append(ext);
  2620. return filename;
  2621. }
  2622. /* Get the file name only. If noExtension is true, the extension (if any) will be trimmed */
  2623. StringBuffer& getFileNameOnly(StringBuffer& filename, bool noExtension)
  2624. {
  2625. char drive[_MAX_DRIVE];
  2626. char dir[_MAX_DIR];
  2627. char fname[_MAX_FNAME];
  2628. char ext[_MAX_EXT];
  2629. _splitpath(filename.str(), drive, dir, fname, ext);
  2630. filename.clear();
  2631. filename.append(fname);
  2632. if (!noExtension)
  2633. filename.append(ext);
  2634. return filename;
  2635. }
  2636. //---------------------------------------------------------------------------
  2637. class CNullDirectoryIterator : public CInterface, implements IDirectoryIterator
  2638. {
  2639. public:
  2640. IMPLEMENT_IINTERFACE;
  2641. bool first()
  2642. {
  2643. return false;
  2644. }
  2645. bool next()
  2646. {
  2647. return false;
  2648. }
  2649. StringBuffer &getName(StringBuffer &buf)
  2650. {
  2651. return buf;
  2652. }
  2653. __int64 getFileSize()
  2654. {
  2655. return -1;
  2656. }
  2657. bool getModifiedTime(CDateTime &ret)
  2658. {
  2659. return false;
  2660. }
  2661. virtual bool isValid() { return false; }
  2662. virtual IFile & query() { return *(IFile *)NULL; }
  2663. virtual bool isDir() { return false; }
  2664. };
  2665. class CDirectoryIterator : public CInterface, implements IDirectoryIterator
  2666. {
  2667. public:
  2668. CDirectoryIterator(const char * _path, const char * _mask, bool _sub, bool _includedir)
  2669. {
  2670. StringBuffer tmp;
  2671. if (!_path || !*_path)
  2672. _path = "." PATHSEPSTR;
  2673. else if (_path[strlen(_path)-1] != PATHSEPCHAR)
  2674. _path = tmp.append(_path).append(PATHSEPCHAR);
  2675. path.set(_path);
  2676. mask.set(_mask);
  2677. sub = _sub;
  2678. includedir = _includedir;
  2679. subidx = 0;
  2680. curisdir = false;
  2681. }
  2682. IMPLEMENT_IINTERFACE
  2683. virtual bool first()=0;
  2684. virtual bool next()=0;
  2685. virtual bool isValid() { return cur != NULL; }
  2686. virtual IFile & query() { return *cur; }
  2687. virtual StringBuffer &getName(StringBuffer &buf)=0;
  2688. virtual bool isDir() { return curisdir; }
  2689. protected:
  2690. Owned<IFile> cur;
  2691. bool curisdir;
  2692. StringAttr path;
  2693. StringAttr mask;
  2694. bool includedir;
  2695. bool sub; // TBD
  2696. StringAttrArray subpaths;
  2697. unsigned subidx; // -1 to index subpaths
  2698. };
  2699. #ifdef _WIN32
  2700. class CWindowsDirectoryIterator : public CDirectoryIterator
  2701. {
  2702. WIN32_FIND_DATA info;
  2703. HANDLE handle;
  2704. bool setCurrent()
  2705. {
  2706. if (strcmp(info.cFileName, ".") == 0 || strcmp(info.cFileName, "..") == 0)
  2707. return false;
  2708. bool match = (!mask.length() || WildMatch(info.cFileName, mask, true));
  2709. curisdir = (info.dwFileAttributes&FILE_ATTRIBUTE_DIRECTORY)!=0;
  2710. if (!match&&!curisdir)
  2711. return false;
  2712. StringBuffer f(path);
  2713. if (subidx)
  2714. f.append(subpaths.item(subidx-1).text).append('\\');
  2715. f.append(info.cFileName);
  2716. if (curisdir) {
  2717. if (sub) {
  2718. const char *s = f.str()+path.length();
  2719. unsigned i;
  2720. for (i=0;i<subpaths.ordinality();i++)
  2721. if (stricmp(subpaths.item(i).text,s)==0)
  2722. break;
  2723. if (i==subpaths.ordinality())
  2724. subpaths.append(*new StringAttrItem(s));
  2725. }
  2726. if (!includedir)
  2727. return false;
  2728. }
  2729. if (!match)
  2730. return false;
  2731. cur.setown(createIFile(f.str()));
  2732. return true;
  2733. }
  2734. bool open()
  2735. {
  2736. close();
  2737. while (subidx<=subpaths.ordinality()) {
  2738. StringBuffer location(path);
  2739. if (subidx)
  2740. location.append(subpaths.item(subidx-1).text).append('\\');
  2741. location.append("*");
  2742. handle = FindFirstFile(location.str(), &info);
  2743. if (handle != INVALID_HANDLE_VALUE)
  2744. return true;
  2745. subidx++;
  2746. }
  2747. return false;
  2748. }
  2749. void close()
  2750. {
  2751. cur.clear();
  2752. if (handle != INVALID_HANDLE_VALUE) {
  2753. FindClose(handle);
  2754. handle = INVALID_HANDLE_VALUE;
  2755. }
  2756. };
  2757. public:
  2758. CWindowsDirectoryIterator(const char * _path, const char * _mask, bool _sub, bool _includedir)
  2759. : CDirectoryIterator(_path,_mask,_sub,_includedir)
  2760. {
  2761. handle = INVALID_HANDLE_VALUE;
  2762. }
  2763. ~CWindowsDirectoryIterator()
  2764. {
  2765. close();
  2766. }
  2767. bool first()
  2768. {
  2769. subpaths.kill();
  2770. subidx = 0;
  2771. if (!open())
  2772. return false;
  2773. if (setCurrent())
  2774. return true;
  2775. return next();
  2776. }
  2777. bool next()
  2778. {
  2779. loop {
  2780. loop {
  2781. if (!FindNextFile(handle, &info))
  2782. break;
  2783. if (setCurrent())
  2784. return true;
  2785. }
  2786. subidx++;
  2787. if (!open())
  2788. break;
  2789. if (setCurrent())
  2790. return true;
  2791. }
  2792. return false;
  2793. }
  2794. StringBuffer &getName(StringBuffer &buf)
  2795. {
  2796. if (subidx)
  2797. buf.append(subpaths.item(subidx-1).text).append('\\');
  2798. return buf.append(info.cFileName);
  2799. }
  2800. __int64 getFileSize()
  2801. {
  2802. if (curisdir)
  2803. return -1;
  2804. LARGE_INTEGER x;
  2805. x.LowPart = info.nFileSizeLow;
  2806. x.HighPart = info.nFileSizeHigh;
  2807. return x.QuadPart;
  2808. }
  2809. bool getModifiedTime(CDateTime &ret)
  2810. {
  2811. FILETIMEtoIDateTime(&ret, info.ftLastWriteTime);
  2812. return true;
  2813. }
  2814. };
  2815. IDirectoryIterator * createDirectoryIterator(const char * path, const char * mask)
  2816. {
  2817. if (mask&&!*mask) // only NULL is wild
  2818. return new CNullDirectoryIterator;
  2819. if (!path || !*path) // cur directory so no point in checking for remote etc.
  2820. return new CWindowsDirectoryIterator(path, mask,false,true);
  2821. OwnedIFile iFile = createIFile(path);
  2822. if (!iFile||(iFile->isDirectory()!=foundYes))
  2823. return new CNullDirectoryIterator;
  2824. return iFile->directoryFiles(mask, false, true);
  2825. }
  2826. IDirectoryIterator *CFile::directoryFiles(const char *mask,bool sub,bool includedirs)
  2827. {
  2828. if ((mask&&!*mask)|| // only NULL is wild
  2829. (isDirectory()!=foundYes))
  2830. return new CNullDirectoryIterator;
  2831. return new CWindowsDirectoryIterator(filename, mask,sub,includedirs);
  2832. }
  2833. bool CFile::getInfo(bool &isdir,offset_t &size,CDateTime &modtime)
  2834. {
  2835. WIN32_FILE_ATTRIBUTE_DATA info;
  2836. if (GetFileAttributesEx(filename, GetFileExInfoStandard, &info) != 0) {
  2837. LARGE_INTEGER x;
  2838. x.LowPart = info.nFileSizeLow;
  2839. x.HighPart = info.nFileSizeHigh;
  2840. size = (offset_t)x.QuadPart;
  2841. isdir = (info.dwFileAttributes != (DWORD)-1)&&(info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY);
  2842. FILETIMEtoIDateTime(&modtime, info.ftLastWriteTime);
  2843. return true;
  2844. }
  2845. size = 0;
  2846. modtime.clear();
  2847. isdir = false;
  2848. return false;
  2849. }
  2850. #else
  2851. class CLinuxDirectoryIterator : public CDirectoryIterator
  2852. {
  2853. StringAttr tail;
  2854. DIR * handle;
  2855. struct stat st;
  2856. bool gotst;
  2857. bool loadst()
  2858. {
  2859. if (!gotst&&cur)
  2860. gotst = (stat(cur->queryFilename(), &st) == 0);
  2861. return gotst;
  2862. }
  2863. public:
  2864. CLinuxDirectoryIterator(const char * _path, const char * _mask, bool _sub,bool _includedir)
  2865. : CDirectoryIterator(_path,_mask,_sub,_includedir)
  2866. {
  2867. handle = NULL;
  2868. gotst = false;
  2869. }
  2870. ~CLinuxDirectoryIterator()
  2871. {
  2872. close();
  2873. }
  2874. bool open()
  2875. {
  2876. close();
  2877. while (subidx<=subpaths.ordinality()) {
  2878. StringBuffer location(path);
  2879. if (subidx)
  2880. location.append(subpaths.item(subidx-1).text);
  2881. // not sure if should remove trailing '/'
  2882. handle = ::opendir(location.str());
  2883. // better error handling here?
  2884. if (handle)
  2885. return true;
  2886. subidx++;
  2887. }
  2888. return false;
  2889. }
  2890. void close()
  2891. {
  2892. cur.clear();
  2893. if (handle) {
  2894. closedir(handle);
  2895. handle = NULL;
  2896. }
  2897. }
  2898. bool first()
  2899. {
  2900. subpaths.kill();
  2901. subidx = 0;
  2902. if (open())
  2903. return next();
  2904. return false;
  2905. }
  2906. bool next()
  2907. {
  2908. loop {
  2909. struct dirent *entry;
  2910. loop {
  2911. gotst = false;
  2912. entry = readdir(handle); // don't need _r here
  2913. // need better checking here?
  2914. if (!entry)
  2915. break;
  2916. if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
  2917. continue;
  2918. bool match = (!mask.length() || WildMatch(entry->d_name, mask, false));
  2919. curisdir = (entry->d_type==DT_DIR);
  2920. bool islnk = (entry->d_type==DT_LNK);
  2921. bool isunknown = (entry->d_type==DT_UNKNOWN); // to work around xfs bug
  2922. if (match||curisdir||islnk||isunknown) {
  2923. StringBuffer f(path);
  2924. if (subidx)
  2925. f.append(subpaths.item(subidx-1).text).append('/');
  2926. f.append(entry->d_name);
  2927. if (islnk||isunknown) {
  2928. struct stat info;
  2929. if (stat(f.str(), &info) == 0) // will follow link
  2930. curisdir = S_ISDIR(info.st_mode);
  2931. else
  2932. curisdir = false;
  2933. }
  2934. if (curisdir) {
  2935. if (sub) {
  2936. const char *s = f.str()+path.length();
  2937. unsigned i;
  2938. for (i=0;i<subpaths.ordinality();i++)
  2939. if (strcmp(subpaths.item(i).text,s)==0)
  2940. break;
  2941. if (i==subpaths.ordinality())
  2942. subpaths.append(*new StringAttrItem(s));
  2943. }
  2944. if (!includedir)
  2945. match = false;
  2946. }
  2947. if (match) {
  2948. tail.set(entry->d_name);
  2949. cur.setown(createIFile(f.str()));
  2950. return true;
  2951. }
  2952. }
  2953. }
  2954. subidx++;
  2955. if (!open())
  2956. break;
  2957. }
  2958. return false;
  2959. }
  2960. StringBuffer &getName(StringBuffer &buf)
  2961. {
  2962. if (subidx)
  2963. buf.append(subpaths.item(subidx-1).text).append('/');
  2964. return buf.append(tail);
  2965. }
  2966. __int64 getFileSize()
  2967. {
  2968. if (curisdir)
  2969. return -1;
  2970. if (!loadst())
  2971. return -1;
  2972. return st.st_size ;
  2973. }
  2974. bool getModifiedTime(CDateTime &ret)
  2975. {
  2976. if (!loadst())
  2977. return false;
  2978. timetToIDateTime(&ret,st.st_mtime);
  2979. return true;
  2980. }
  2981. };
  2982. IDirectoryIterator * createDirectoryIterator(const char * path, const char * mask)
  2983. {
  2984. if (mask&&!*mask) // only NULL is wild
  2985. return new CNullDirectoryIterator;
  2986. if (!path || !*path) // no point in checking for remote etc.
  2987. return new CLinuxDirectoryIterator(path, mask,false,true);
  2988. OwnedIFile iFile = createIFile(path);
  2989. if (!iFile||(iFile->isDirectory()!=foundYes))
  2990. return new CNullDirectoryIterator;
  2991. return iFile->directoryFiles(mask, false, true);
  2992. }
  2993. IDirectoryIterator *CFile::directoryFiles(const char *mask,bool sub,bool includedirs)
  2994. {
  2995. if ((mask&&!*mask)|| // only NULL is wild
  2996. (isDirectory()!=foundYes))
  2997. return new CNullDirectoryIterator;
  2998. return new CLinuxDirectoryIterator(filename, mask,sub,includedirs);
  2999. }
  3000. bool CFile::getInfo(bool &isdir,offset_t &size,CDateTime &modtime)
  3001. {
  3002. struct stat info;
  3003. if (stat(filename, &info) == 0) {
  3004. size = (offset_t)info.st_size;
  3005. isdir = S_ISDIR(info.st_mode);
  3006. timetToIDateTime(&modtime, info.st_mtime);
  3007. return true;
  3008. }
  3009. size = 0;
  3010. modtime.clear();
  3011. isdir = false;
  3012. return false;
  3013. }
  3014. #endif
  3015. class CDirEntry: extends CInterface
  3016. {
  3017. public:
  3018. StringAttr name;
  3019. Owned<IFile> file;
  3020. __int64 size;
  3021. CDateTime modified;
  3022. byte flags; // IDDI*
  3023. bool isdir;
  3024. CDirEntry(IDirectoryIterator *iter)
  3025. {
  3026. StringBuffer n;
  3027. name.set(iter->getName(n).str());
  3028. size = iter->getFileSize();
  3029. iter->getModifiedTime(modified);
  3030. file.set(&iter->query());
  3031. flags = 0;
  3032. isdir = iter->isDir();
  3033. }
  3034. bool match(byte f)
  3035. {
  3036. return (flags==0)||((f&flags)!=0);
  3037. }
  3038. int compare(const CDirEntry *e)
  3039. {
  3040. return strcmp(name.get(),e->name.get());
  3041. }
  3042. int compareProp(const CDirEntry *e)
  3043. {
  3044. int c = compare(e);
  3045. if (c)
  3046. return c;
  3047. if (isdir!=e->isdir)
  3048. return isdir?-1:1;
  3049. if (size!=e->size)
  3050. return (size<e->size)?-1:1;
  3051. return modified.compare(e->modified,false);
  3052. }
  3053. };
  3054. class CDirectoryDifferenceIterator : public CIArrayOf<CDirEntry>, extends CInterface, implements IDirectoryDifferenceIterator
  3055. {
  3056. static int compare(CInterface **_a, CInterface **_b)
  3057. {
  3058. CDirEntry *a = *(CDirEntry **)_a;
  3059. CDirEntry *b = *(CDirEntry **)_b;
  3060. return a->compare(b);
  3061. }
  3062. unsigned idx;
  3063. byte mask;
  3064. public:
  3065. IMPLEMENT_IINTERFACE;
  3066. CDirectoryDifferenceIterator(IDirectoryIterator *iter, CDirectoryDifferenceIterator *cmp)
  3067. {
  3068. mask = IDDIstandard;
  3069. idx = 0;
  3070. ForEach(*iter)
  3071. append(*new CDirEntry(iter));
  3072. CIArrayOf<CDirEntry>::sort(compare);
  3073. if (cmp) { // assumes cmp sorted
  3074. unsigned i = 0;
  3075. unsigned ni = ordinality();
  3076. unsigned j = 0;
  3077. unsigned nj = cmp->ordinality();
  3078. loop {
  3079. CDirEntry *a = NULL;
  3080. CDirEntry *b = NULL;
  3081. if (i>=ni) {
  3082. if (j>=nj)
  3083. break;
  3084. b = &cmp->item(j++);
  3085. }
  3086. else if (j>=nj) {
  3087. a = &item(i++);
  3088. }
  3089. else {
  3090. a = &item(i);
  3091. b = &cmp->item(j);
  3092. int c = a->compare(b);
  3093. if (c==0) {
  3094. if (a->compareProp(b)==0) {
  3095. a->flags = IDDIunchanged;
  3096. a = NULL;
  3097. b = NULL;
  3098. }
  3099. i++;
  3100. j++;
  3101. }
  3102. else if (c<0) {
  3103. b = NULL;
  3104. i++;
  3105. }
  3106. else {
  3107. a = NULL;
  3108. j++;
  3109. }
  3110. }
  3111. if (a) {
  3112. if (b)
  3113. a->flags|=IDDImodified;
  3114. else
  3115. a->flags|=IDDIadded;
  3116. }
  3117. else if (b) {
  3118. b->Link();
  3119. b->flags = IDDIdeleted;
  3120. add(*b,i);
  3121. i++;
  3122. ni++;
  3123. }
  3124. }
  3125. }
  3126. }
  3127. virtual bool first()
  3128. {
  3129. idx = 0;
  3130. while (idx<ordinality()) {
  3131. if (item(idx).match(mask))
  3132. return true;
  3133. idx++;
  3134. }
  3135. return false;
  3136. }
  3137. virtual bool next()
  3138. {
  3139. idx++;
  3140. while (idx<ordinality()) {
  3141. if (item(idx).match(mask))
  3142. return true;
  3143. idx++;
  3144. }
  3145. return false;
  3146. }
  3147. virtual bool isValid()
  3148. {
  3149. return (idx<ordinality());
  3150. }
  3151. virtual IFile & query()
  3152. {
  3153. if (isValid())
  3154. return *item(idx).file;
  3155. return *(IFile *) NULL;
  3156. }
  3157. virtual StringBuffer &getName(StringBuffer &buf)
  3158. {
  3159. if (isValid())
  3160. buf.append(item(idx).name);
  3161. return buf;
  3162. }
  3163. virtual bool isDir()
  3164. {
  3165. if (isValid())
  3166. return item(idx).isdir;
  3167. return false;
  3168. }
  3169. __int64 getFileSize()
  3170. {
  3171. if (isValid())
  3172. return item(idx).isdir?0:item(idx).size;
  3173. return 0;
  3174. }
  3175. virtual bool getModifiedTime(CDateTime &ret)
  3176. {
  3177. if (isValid()) {
  3178. ret.set(item(idx).modified);
  3179. }
  3180. return false;
  3181. }
  3182. virtual void setMask(unsigned _mask)
  3183. {
  3184. mask = (byte)_mask;
  3185. }
  3186. virtual unsigned getFlags()
  3187. {
  3188. if (isValid())
  3189. return item(idx).flags;
  3190. return 0;
  3191. }
  3192. unsigned numChanges()
  3193. {
  3194. unsigned ret=0;
  3195. ForEachItemIn(i,*this) {
  3196. byte f = item(i).flags;
  3197. if ((f!=0)&&(f!=IDDIunchanged))
  3198. ret++;
  3199. }
  3200. return ret;
  3201. }
  3202. };
  3203. IDirectoryDifferenceIterator *CFile::monitorDirectory(IDirectoryIterator *_prev, // in
  3204. const char *mask,
  3205. bool sub,
  3206. bool includedirs,
  3207. unsigned checkinterval,
  3208. unsigned timeout,
  3209. Semaphore *abortsem)
  3210. {
  3211. Linked<IDirectoryIterator> prev;
  3212. if (_prev)
  3213. prev.set(_prev);
  3214. else
  3215. prev.setown(directoryFiles(mask,sub,includedirs));
  3216. if (!prev)
  3217. return NULL;
  3218. Owned<CDirectoryDifferenceIterator> base = new CDirectoryDifferenceIterator(prev,NULL);
  3219. prev.clear(); // not needed now
  3220. unsigned start=msTick();
  3221. loop {
  3222. if (abortsem) {
  3223. if (abortsem->wait(checkinterval))
  3224. break;
  3225. }
  3226. else
  3227. Sleep(checkinterval);
  3228. Owned<IDirectoryIterator> current = directoryFiles(mask,sub,includedirs);
  3229. if (!current)
  3230. break;
  3231. Owned<CDirectoryDifferenceIterator> cmp = new CDirectoryDifferenceIterator(current,base);
  3232. current.clear();
  3233. if (cmp->numChanges())
  3234. return cmp.getClear();
  3235. if (msTick()-start>timeout)
  3236. break;
  3237. }
  3238. return NULL; // timed out
  3239. }
  3240. //---------------------------------------------------------------------------
  3241. class FixedPasswordProvider : public CInterface, implements IPasswordProvider
  3242. {
  3243. public:
  3244. FixedPasswordProvider(const char * _username, const char * _password) { username.set(_username); password.set(_password); }
  3245. IMPLEMENT_IINTERFACE;
  3246. virtual bool getPassword(const IpAddress & ip, StringBuffer & _username, StringBuffer & _password)
  3247. {
  3248. _username.append(username.get());
  3249. _password.append(password.get());
  3250. return true;
  3251. }
  3252. protected:
  3253. StringAttr username;
  3254. StringAttr password;
  3255. };
  3256. IPasswordProvider * queryPasswordProvider()
  3257. {
  3258. return passwordProvider;
  3259. }
  3260. void setPasswordProvider(IPasswordProvider * provider)
  3261. {
  3262. passwordProvider.set(provider);
  3263. }
  3264. void setDefaultUser(const char * username,const char *password)
  3265. {
  3266. Owned<IPasswordProvider> provider = new FixedPasswordProvider(username, password);
  3267. setPasswordProvider(provider);
  3268. }
  3269. //---------------------------------------------------------------------------
  3270. bool recursiveCreateDirectory(const char * path)
  3271. {
  3272. #ifndef _WIN32
  3273. #ifdef USE_SAMBA
  3274. return localCreateDirectory(path);
  3275. #endif
  3276. #endif
  3277. Owned<IFile> file = createIFile(path);
  3278. return file->createDirectory();
  3279. }
  3280. bool recursiveCreateDirectoryForFile(const char *fullFileName)
  3281. {
  3282. StringBuffer path;
  3283. splitFilename(fullFileName, &path, &path, NULL, NULL);
  3284. return recursiveCreateDirectory(path.str());
  3285. }
  3286. //---------------------------------------------------------------------------
  3287. size32_t DirectBufferI::read(offset_t pos, size32_t len, void * data)
  3288. {
  3289. if (pos + len > buffLen)
  3290. {
  3291. if (pos > buffLen)
  3292. pos = buffLen;
  3293. len = (size32_t)(buffLen - pos);
  3294. }
  3295. memcpy(data, buffer+pos, len);
  3296. return len;
  3297. }
  3298. size32_t DirectBufferI::write(offset_t pos, size32_t len, const void * data)
  3299. {
  3300. UNIMPLEMENTED;
  3301. }
  3302. size32_t DirectBufferIO::write(offset_t pos, size32_t len, const void * data)
  3303. {
  3304. if (pos + len > buffLen)
  3305. {
  3306. if (pos > buffLen)
  3307. pos = buffLen;
  3308. len = (size32_t)(buffLen - pos);
  3309. }
  3310. memcpy(buffer+pos, data, len);
  3311. return len;
  3312. }
  3313. //---------------------------------------------------------------------------
  3314. size32_t StringBufferIO::read(offset_t pos, size32_t len, void * data)
  3315. {
  3316. unsigned strLen = buffer.length();
  3317. if (pos + len > strLen)
  3318. {
  3319. if (pos > strLen)
  3320. pos = strLen;
  3321. len = (size32_t)(strLen - pos);
  3322. }
  3323. buffer.getChars((size32_t)pos, (size32_t)(pos+len), (char *)data);
  3324. return len;
  3325. }
  3326. offset_t StringBufferIO::size()
  3327. {
  3328. return buffer.length();
  3329. }
  3330. size32_t StringBufferIO::write(offset_t pos, size32_t len, const void * data)
  3331. {
  3332. unsigned strLen = buffer.length();
  3333. buffer.ensureCapacity((size32_t)(pos+len));
  3334. memcpy((byte *)buffer.toCharArray() + pos, data, len);
  3335. return len;
  3336. }
  3337. void StringBufferIO::setSize(offset_t size)
  3338. {
  3339. if (size > buffer.length())
  3340. buffer.ensureCapacity((size32_t)size);
  3341. else
  3342. buffer.setLength((size32_t)size);
  3343. }
  3344. //---------------------------------------------------------------------------
  3345. size32_t MemoryBufferIO::read(offset_t pos, size32_t len, void * data)
  3346. {
  3347. unsigned strLen = buffer.length();
  3348. if (pos + len > strLen)
  3349. {
  3350. if (pos > strLen)
  3351. pos = strLen;
  3352. len = (size32_t)(strLen - pos);
  3353. }
  3354. memcpy(data, buffer.toByteArray() + pos, len);
  3355. return len;
  3356. }
  3357. offset_t MemoryBufferIO::size()
  3358. {
  3359. return buffer.length();
  3360. }
  3361. size32_t MemoryBufferIO::write(offset_t pos, size32_t len, const void * data)
  3362. {
  3363. unsigned strLen = buffer.length();
  3364. buffer.ensureCapacity((size32_t)(pos+len));
  3365. memcpy((byte *)buffer.toByteArray() + pos, data, len);
  3366. return len;
  3367. }
  3368. void MemoryBufferIO::setSize(offset_t size)
  3369. {
  3370. if (size > buffer.length())
  3371. buffer.ensureCapacity((size32_t)size);
  3372. else
  3373. buffer.setLength((size32_t)size);
  3374. }
  3375. //---------------------------------------------------------------------------
  3376. IFile * createIFile(const char * filename)
  3377. {
  3378. if (!filename)
  3379. return NULL;
  3380. bool linremote=(memcmp(filename,"//",2)==0);
  3381. if (!linremote&&(memcmp(filename,"\\\\",2)!=0)) // see if remote looking
  3382. return new CFile(filename);
  3383. RemoteFilename rfn;
  3384. rfn.setRemotePath(filename);
  3385. if (rfn.isNull())
  3386. throw MakeStringException(-1, "CreateIFile cannot resolve %s", filename);
  3387. if (rfn.isLocal()) { // ignore dafilesrv request if local and standard port
  3388. StringBuffer tmplocal;
  3389. return new CFile(rfn.getLocalPath(tmplocal).str());
  3390. }
  3391. #ifdef _WIN32
  3392. StringBuffer tmplocal;
  3393. if (linremote||(rfn.queryEndpoint().port!=0)) {
  3394. IFile * ret = createIFileByHook(rfn); // use daliservix in preference
  3395. if (ret)
  3396. return ret;
  3397. while (*filename) { // no daliservix so swap '/' for '\' and hope for best
  3398. if (*filename=='/')
  3399. tmplocal.append('\\');
  3400. else
  3401. tmplocal.append(*filename);
  3402. filename++;
  3403. }
  3404. filename =tmplocal.str();
  3405. }
  3406. return new CWindowsRemoteFile(filename);
  3407. #else
  3408. #ifdef USE_SAMBA
  3409. if(strncmp(filename, "smb://", 6) == 0)
  3410. return new CSambaRemoteFile(filename);
  3411. if(memcmp(filename, "//", 2) == 0) {
  3412. StringBuffer smbfile("smb:");
  3413. smbfile.append(filename);
  3414. return new CSambaRemoteFile(smbfile.str());
  3415. }
  3416. if(memcmp(filename, "\\\\", 2) == 0) {
  3417. StringBuffer smbfile("smb:");
  3418. int i = 0;
  3419. while(filename[i]) {
  3420. if(filename[i] == '\\')
  3421. smbfile.append('/');
  3422. else
  3423. smbfile.append(filename[i]);
  3424. i++;
  3425. }
  3426. return new CSambaRemoteFile(smbfile.str());
  3427. }
  3428. #else
  3429. if (memcmp(filename,"smb://",6)==0) // don't support samba - try remote
  3430. return createIFile(filename+4);
  3431. IFile * ret = createIFileByHook(rfn);
  3432. if (!ret)
  3433. throw MakeStringException(-1, "CreateIFile::cannot attach to %s. (remote.so not linked?)", filename);
  3434. return ret;
  3435. #endif
  3436. return new CFile(filename);
  3437. #endif
  3438. }
  3439. IFileIOStream * createIOStream(IFileIO * file)
  3440. {
  3441. return new CFileIOStream(file);
  3442. }
  3443. IFileIO * createIORange(IFileIO * io, offset_t header, offset_t length)
  3444. {
  3445. return new CFileRangeIO(io, header, length);
  3446. }
  3447. IFileIOStream * createBufferedIOStream(IFileIO * io, unsigned bufsize)
  3448. {
  3449. if (bufsize == (unsigned)-1)
  3450. bufsize = DEFAULT_BUFFER_SIZE;
  3451. return new CBufferedFileIOStream(io, bufsize);
  3452. }
  3453. IFileIOStream * createBufferedAsyncIOStream(IFileAsyncIO * io, unsigned bufsize)
  3454. {
  3455. if (bufsize == (unsigned)-1)
  3456. bufsize = DEFAULT_BUFFER_SIZE*2;
  3457. return new CBufferedAsyncIOStream(io, bufsize);
  3458. }
  3459. IReadSeq *createReadSeq(IFileIOStream * stream, offset_t offset, size32_t size)
  3460. {
  3461. return new CIOStreamReadWriteSeq(stream, offset, size);
  3462. }
  3463. IWriteSeq *createWriteSeq(IFileIOStream * stream, size32_t size)
  3464. {
  3465. return new CIOStreamReadWriteSeq(stream, 0, size);
  3466. }
  3467. extern jlib_decl offset_t filesize(const char *name)
  3468. {
  3469. CFile f(name);
  3470. return f.size();
  3471. }
  3472. extern jlib_decl offset_t getFreeSpace(const char* name)
  3473. {
  3474. offset_t freeBytesToCaller;
  3475. #ifdef _WIN32
  3476. offset_t totalBytes;
  3477. offset_t freeBytes;
  3478. int fResult = GetDiskFreeSpaceEx (name,
  3479. (PULARGE_INTEGER)&freeBytesToCaller,
  3480. (PULARGE_INTEGER)&totalBytes,
  3481. (PULARGE_INTEGER)&freeBytes);
  3482. if (fResult == 0) // error
  3483. {
  3484. return 0;
  3485. }
  3486. #elif defined (__linux__)
  3487. struct statfs buf;
  3488. int fResult = statfs(name, &buf);
  3489. if (fResult == -1) // error
  3490. {
  3491. return 0;
  3492. }
  3493. // from "man statfs the def of f_bavail and f_bsize...
  3494. // mult "free blocks avail to non-superuser" by "optimal transfer block size" to get the available size
  3495. freeBytesToCaller = (offset_t)buf.f_bavail * buf.f_bsize;
  3496. #else
  3497. UNIMPLEMENTED;
  3498. #endif
  3499. return freeBytesToCaller;
  3500. }
  3501. extern jlib_decl void createHardLink(const char* fileName, const char* existingFileName)
  3502. {
  3503. #ifdef _WIN32
  3504. // requirements...need to make sure that the directory already exists
  3505. // and the 2 directories need to be on the same drive
  3506. if (!CreateHardLink(fileName, existingFileName, NULL))
  3507. {
  3508. LPVOID lpMsgBuf;
  3509. FormatMessage(
  3510. FORMAT_MESSAGE_ALLOCATE_BUFFER |
  3511. FORMAT_MESSAGE_FROM_SYSTEM |
  3512. FORMAT_MESSAGE_IGNORE_INSERTS,
  3513. NULL,
  3514. GetLastError(),
  3515. MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), // Default language
  3516. (LPTSTR) &lpMsgBuf,
  3517. 0,
  3518. NULL
  3519. );
  3520. StringBuffer err;
  3521. err.appendf("Failed to create log alias %s for %s: %s", fileName, existingFileName, lpMsgBuf);
  3522. LocalFree( lpMsgBuf );
  3523. throw MakeStringException(-1, "createHardLink:: %s.",err.str());
  3524. }
  3525. #else
  3526. if (link(existingFileName, fileName) != 0) // error
  3527. throw MakeStringException(-1, "Failed to create log alias %s for %s: error code = %d", fileName, existingFileName, errno);
  3528. #endif
  3529. }
  3530. #if 0
  3531. void testDirectory()
  3532. {
  3533. //setDefaultUser("administrator", "password");
  3534. Owned<IFile> dir = createIFile("\\test1\\gavin\\home\\base");
  3535. dir->createDirectory();
  3536. Owned<IFile> dir2 = createIFile("\\test1\\gavin\\home\\base2\\");
  3537. dir2->createDirectory();
  3538. dir2->createDirectory();
  3539. Owned<IFile> dir3 = createIFile("c:\\test1\\gavin\\home2\\base");
  3540. dir3->createDirectory();
  3541. Owned<IFile> dir4 = createIFile("\\\\ghalliday\\c$\\test1\\gavin\\home3\\base");
  3542. dir4->createDirectory();
  3543. Owned<IFile> dir5 = createIFile("\\\\192.168.1.3\\d$\\test2\\gavin\\home3\\base");
  3544. dir5->createDirectory();
  3545. StringBuffer x;
  3546. x.clear();
  3547. splitUNCFilename("\\\\127.0.0.1\\gavin\\abc.ext", &x, &x, &x, &x);
  3548. x.clear();
  3549. splitUNCFilename("\\\\127.0.0.1\\gavin\\abc.ext\\etc", &x, &x, &x, &x);
  3550. x.clear();
  3551. splitUNCFilename("\\\\127.0.0.1\\", &x, &x, &x, &x);
  3552. }
  3553. #endif
  3554. // -- RemoteFilename class (file location encapsulation)
  3555. //#define _TRACERFN
  3556. void RemoteFilename::badFilename(const char * filename)
  3557. {
  3558. throw MakeStringException(-1, "Badly formatted file entry %s", filename);
  3559. }
  3560. bool RemoteFilename::equals(const RemoteFilename & other) const
  3561. {
  3562. if (isNull())
  3563. return other.isNull();
  3564. if (other.isNull())
  3565. return false;
  3566. if (!ep.ipequals(other.ep)) // only use ip for compare
  3567. return false;
  3568. if (isUnixPath()) { // case sensitive
  3569. if (localhead.length()&&other.localhead.length()) {
  3570. if (strcmp(localhead.get(),other.localhead.get())!=0)
  3571. return false;
  3572. }
  3573. else if (sharehead.length()&&other.sharehead.length()) {
  3574. if (strcmp(sharehead.get(),other.sharehead.get())!=0)
  3575. return false;
  3576. }
  3577. else if (sharehead.length()!=other.sharehead.length())
  3578. return false;
  3579. if (strcmp(tailpath.get(),other.tailpath.get())!=0)
  3580. return false;
  3581. }
  3582. else {
  3583. if (localhead.length()&&other.localhead.length()) {
  3584. if (stricmp(localhead.get(),other.localhead.get())!=0)
  3585. return false;
  3586. }
  3587. else if (sharehead.length()&&other.sharehead.length()) {
  3588. if (stricmp(sharehead.get(),other.sharehead.get())!=0)
  3589. return false;
  3590. }
  3591. else if (sharehead.length()!=other.sharehead.length())
  3592. return false;
  3593. if (stricmp(tailpath.get(),other.tailpath.get())!=0)
  3594. return false;
  3595. }
  3596. return true;
  3597. }
  3598. bool RemoteFilename::isNull() const
  3599. {
  3600. return (ep.isNull()||(tailpath.length()==0));
  3601. }
  3602. void RemoteFilename::clear()
  3603. {
  3604. ep.set(NULL, 0);
  3605. localhead.clear();
  3606. sharehead.clear();
  3607. tailpath.clear();
  3608. }
  3609. void RemoteFilename::serialize(MemoryBuffer & out)
  3610. {
  3611. ep.serialize(out);
  3612. out.append(localhead);
  3613. out.append(sharehead);
  3614. out.append(tailpath);
  3615. }
  3616. void RemoteFilename::deserialize(MemoryBuffer & in)
  3617. {
  3618. ep.deserialize(in);
  3619. in.read(localhead);
  3620. in.read(sharehead);
  3621. in.read(tailpath);
  3622. }
  3623. StringBuffer & RemoteFilename::getTail(StringBuffer &name) const
  3624. {
  3625. split(NULL, NULL, &name, &name);
  3626. return name;
  3627. }
  3628. StringBuffer & RemoteFilename::getPath(StringBuffer & name) const
  3629. {
  3630. if (isLocal())
  3631. return getLocalPath(name);
  3632. else
  3633. return getRemotePath(name);
  3634. }
  3635. bool RemoteFilename::isUnixPath() const // bit arbitrary
  3636. {
  3637. if (tailpath.length()!=0) {
  3638. char c = tailpath[0];
  3639. if (c=='/')
  3640. return true;
  3641. if (c=='\\')
  3642. return false;
  3643. }
  3644. if (sharehead.length()!=0) {
  3645. char c = sharehead[0];
  3646. if (c=='/')
  3647. return true;
  3648. if (c=='\\')
  3649. return false;
  3650. }
  3651. if (localhead.length()!=0) {
  3652. const char *s=localhead;
  3653. if (*s=='/')
  3654. return true;
  3655. if ((s[1]==':')&&(s[2]=='/'))
  3656. return false;
  3657. // those are the expected cases, otherwise look for first separator (legacy)
  3658. while (*s) {
  3659. if (*s=='\\')
  3660. return false;
  3661. s++;
  3662. if (*s=='/')
  3663. return true;
  3664. }
  3665. }
  3666. #ifdef _WIN32 // assume local on starting machine
  3667. return false;
  3668. #else
  3669. return true;
  3670. #endif
  3671. }
  3672. char RemoteFilename::getPathSeparator() const
  3673. {
  3674. return isUnixPath()?'/':'\\';
  3675. }
  3676. StringBuffer & RemoteFilename::getLocalPath(StringBuffer & out) const
  3677. {
  3678. if (tailpath.length()!=0) {
  3679. out.append(localhead).append(tailpath);
  3680. }
  3681. #ifdef _TRACERFN
  3682. StringBuffer eps;
  3683. PROGLOG("getLocalPath '%s' '%s' '%s' -> '%s'",sharehead.get()?sharehead.get():"NULL",localhead.get()?localhead.get():"NULL",tailpath.get()?tailpath.get():"NULL",out.str());
  3684. #endif
  3685. return out;
  3686. }
  3687. StringBuffer & RemoteFilename::getRemotePath(StringBuffer & out) const
  3688. { // this creates a name that can be used by windows or linux
  3689. // note - no longer loses port
  3690. char c=getPathSeparator();
  3691. out.append(c).append(c);
  3692. ep.getUrlStr(out);
  3693. const char *fn;
  3694. StringBuffer loc;
  3695. if (sharehead.length())
  3696. fn = loc.append(sharehead).append(tailpath).str();
  3697. else // try and guess from just tail (may likely fail other than for windows)
  3698. fn=getLocalPath(loc).str();
  3699. if ((c=='\\')&&(fn[1]==':')) { // windows \\d$
  3700. out.append((char)tolower(c)).append(*fn).append(getShareChar());
  3701. fn+=2;
  3702. }
  3703. out.append(fn);
  3704. #ifdef _TRACERFN
  3705. StringBuffer eps;
  3706. PROGLOG("getRemotePath '%s' '%s' '%s' -> '%s'",sharehead.get()?sharehead.get():"NULL",localhead.get()?localhead.get():"NULL",tailpath.get()?tailpath.get():"NULL",out.str());
  3707. #endif
  3708. return out;
  3709. }
  3710. bool RemoteFilename::isLocal() const
  3711. {
  3712. if (ep.port&&(ep.port!=DAFILESRV_PORT))
  3713. return false; // treat non-dafilesrv port as remote
  3714. return ep.isLocal() || ep.isNull();
  3715. }
  3716. void RemoteFilename::set(const RemoteFilename & other)
  3717. {
  3718. ep.set(other.ep);
  3719. localhead.set(other.localhead);
  3720. sharehead.set(other.sharehead);
  3721. tailpath.set(other.tailpath);
  3722. }
  3723. void RemoteFilename::setExtension(const char * newext)
  3724. {
  3725. const char * dot = NULL;
  3726. const char * start = tailpath;
  3727. const char * cur = start;
  3728. while (*cur)
  3729. {
  3730. if (*cur == '.')
  3731. dot = cur;
  3732. cur++;
  3733. }
  3734. StringBuffer newtail;
  3735. if (dot)
  3736. newtail.append(dot-start,start);
  3737. else
  3738. newtail.append(start);
  3739. if (newext)
  3740. {
  3741. if (newext[0] != '.')
  3742. newtail.append('.');
  3743. newtail.append(newext);
  3744. }
  3745. tailpath.set(newtail);
  3746. }
  3747. void RemoteFilename::setPath(const SocketEndpoint & _ep, const char * _filename)
  3748. {
  3749. const char * filename=_filename;
  3750. StringBuffer full;
  3751. ep.set(_ep);
  3752. localhead.clear();
  3753. sharehead.clear();
  3754. if (filename&&*filename) {
  3755. if (isSpecialPath(filename)) {
  3756. tailpath.set(filename);
  3757. return;
  3758. }
  3759. if (isLocal()&&!isAbsolutePath(filename)) {
  3760. char dir[_MAX_PATH];
  3761. GetCurrentDirectory(sizeof(dir), dir);
  3762. if (*filename==PATHSEPCHAR) {
  3763. #ifdef _WIN32
  3764. if (*dir && (dir[1]==':'))
  3765. dir[2] = 0;
  3766. #endif
  3767. filename++;
  3768. }
  3769. full.append(dir);
  3770. if (full.charAt(full.length()-1) != PATHSEPCHAR)
  3771. full.append(PATHSEPCHAR);
  3772. full.append(filename);
  3773. filename = full.str();
  3774. }
  3775. bool isunix;
  3776. if (filename[0]=='/')
  3777. isunix=true;
  3778. else if (filename[0]=='\\')
  3779. isunix=false;
  3780. else if ((filename[1]==':')&&(filename[2]=='\\'))
  3781. isunix = false;
  3782. else // invalid call really as supposed to be full local path!
  3783. #ifdef _WIN32
  3784. isunix = false;
  3785. #else
  3786. isunix = true;
  3787. #endif
  3788. StringBuffer tmpxlat;
  3789. char sepchar = isunix?'/':'\\';
  3790. char altchar = isunix?'\\':'/';
  3791. // fix mixed separator path
  3792. if (strchr(filename,altchar)) {
  3793. while (*filename) {
  3794. if (*filename==altchar)
  3795. tmpxlat.append(sepchar);
  3796. else
  3797. tmpxlat.append(*filename);
  3798. filename++;
  3799. }
  3800. filename = tmpxlat.str();
  3801. }
  3802. if (isunix) {
  3803. if (filename[0]=='/') {
  3804. const char *tail;
  3805. tail=strchr(filename+1,'/');
  3806. if (tail) {
  3807. sharehead.set(filename,tail-filename); // we don't know share so guess same as leading dir
  3808. localhead.set(filename,tail-filename);
  3809. filename = tail;
  3810. }
  3811. }
  3812. }
  3813. else { // windows can guess anonymous share
  3814. StringBuffer sharestr;
  3815. if ((filename[1]==':')&&(filename[2]=='\\')) { // this should be always true if true full windows path
  3816. localhead.set(filename,2);
  3817. sharestr.append('\\').append((char)tolower(filename[0])).append(getShareChar());
  3818. filename += 2;
  3819. }
  3820. else if (filename[0]=='\\') {
  3821. const char *tail = strchr(filename+1,'\\');
  3822. if (tail) {
  3823. sharestr.append(tail-filename,filename);
  3824. localhead.set(filename,tail-filename);
  3825. filename = tail;
  3826. }
  3827. else {
  3828. localhead.set("c:");
  3829. sharestr.append("\\c").append(getShareChar());
  3830. }
  3831. }
  3832. sharehead.set(sharestr);
  3833. }
  3834. tailpath.set(filename);
  3835. }
  3836. else
  3837. tailpath.clear();
  3838. #ifdef _TRACERFN
  3839. StringBuffer eps;
  3840. PROGLOG("setPath (%s,%s) -> '%s' '%s' '%s'",ep.getUrlStr(eps).str(),_filename?_filename:"NULL",sharehead.get()?sharehead.get():"NULL",localhead.get()?localhead.get():"NULL",tailpath.get()?tailpath.get():"NULL");
  3841. #endif
  3842. }
  3843. void RemoteFilename::setRemotePath(const char * _url,const char *localpath)
  3844. {
  3845. // url should be full (share) path including ep
  3846. const char *url=_url;
  3847. // if ep not included then assume local (bit weird though)
  3848. char sep = url?url[0]:0;
  3849. if (isPathSepChar(sep)&&(sep==url[1])) {
  3850. url+=2;
  3851. const char *end = findPathSepChar(url);
  3852. if (end) {
  3853. StringBuffer eps(end-url,url);
  3854. ep.set(eps.str());
  3855. url = end;
  3856. }
  3857. else {
  3858. ep.set(NULL,0);
  3859. url--; // don't know what is going on!
  3860. }
  3861. if (isPathSepChar(*url)||!*url)
  3862. sep = *url;
  3863. }
  3864. else
  3865. ep.setLocalHost(0);
  3866. if (localpath&&*localpath)
  3867. setPath(ep,localpath);
  3868. else {
  3869. localhead.clear();
  3870. sharehead.clear();
  3871. tailpath.clear();
  3872. }
  3873. if (sep&&*url) {
  3874. // url should point to the share now
  3875. const char *tail=findPathSepChar(url+1);
  3876. if (tail) { // hopefully must be!
  3877. sharehead.set(url,tail-url);
  3878. url = tail;
  3879. }
  3880. if (localhead.length()==0) { // we don't know so guess
  3881. if (sharehead.length()!=0) {
  3882. const char *share=sharehead.get();
  3883. if (sep=='\\') {
  3884. StringBuffer locstr;
  3885. if (sharehead.length()>2) {
  3886. if (isShareChar(share[2])) {
  3887. locstr.append(share[1]).append(':');
  3888. share+=3;
  3889. }
  3890. else // we haven't a clue!
  3891. locstr.append("c:");
  3892. }
  3893. else if (sharehead[1]!='$') // data format
  3894. locstr.append("c:");
  3895. locstr.append(share);
  3896. localhead.set(locstr);
  3897. }
  3898. else { // we haven't a clue so assume same as share
  3899. localhead.set(sharehead);
  3900. }
  3901. }
  3902. }
  3903. }
  3904. if (tailpath.length()==0)
  3905. tailpath.set(url);
  3906. #ifdef _TRACERFN
  3907. StringBuffer eps;
  3908. PROGLOG("setRemotePath ('%s','%s') -> '%s' '%s' '%s'",_url,localpath?localpath:"NULL",sharehead.get()?sharehead.get():"NULL",localhead.get()?localhead.get():"NULL",tailpath.get()?tailpath.get():"NULL");
  3909. #endif
  3910. }
  3911. void RemoteFilename::setLocalPath(const char *filename)
  3912. {
  3913. SocketEndpoint localEP;
  3914. localEP.setLocalHost(0);
  3915. setPath(localEP, filename);
  3916. }
  3917. void RemoteFilename::split(StringBuffer * drive, StringBuffer * path, StringBuffer * tail, StringBuffer * ext) const
  3918. {
  3919. StringBuffer localname;
  3920. const char *s = getLocalPath(localname).str();
  3921. char sep;
  3922. if (isUnixPath())
  3923. sep = '/';
  3924. else {
  3925. sep = '\\';
  3926. if (s[1]==':') {
  3927. if (drive)
  3928. drive->append(2,s);
  3929. s+=2;
  3930. }
  3931. }
  3932. const char *t=NULL;
  3933. const char *e=s;
  3934. while (*e) {
  3935. if (*e==sep)
  3936. t = e+1;
  3937. e++;
  3938. }
  3939. if (t) {
  3940. if (path)
  3941. path->append(t-s, s);
  3942. s=t;
  3943. }
  3944. if (!tail&&!ext)
  3945. return;
  3946. t = NULL;
  3947. e=s;
  3948. while (*e) {
  3949. if (*e=='.') // NB final extension
  3950. t = e;
  3951. e++;
  3952. }
  3953. if (t) {
  3954. if (tail)
  3955. tail->append(t-s,s);
  3956. if (ext)
  3957. ext->append(e-t,t);
  3958. }
  3959. else if (tail)
  3960. tail->append(e-s,s);
  3961. }
  3962. void RemoteMultiFilename::append(const char *mpath,const char *defaultdir)
  3963. {
  3964. StringArray paths;
  3965. expand(mpath, paths);
  3966. StringBuffer fullpath;
  3967. StringBuffer deftmp;
  3968. if (paths.ordinality()&&(defaultdir==NULL)&&isAbsolutePath(paths.item(0))) {
  3969. splitDirTail(paths.item(0),deftmp);
  3970. defaultdir = deftmp.str();
  3971. }
  3972. ForEachItemIn(i,paths) {
  3973. const char *filename = paths.item(i);
  3974. RemoteFilename rfn;
  3975. if (isPathSepChar(*filename)&&isPathSepChar(filename[1])) // full URL
  3976. rfn.setRemotePath(filename);
  3977. else {
  3978. if (defaultdir&&!isAbsolutePath(filename))
  3979. filename = addPathSepChar(fullpath.clear().append(defaultdir)).append(filename).str();
  3980. rfn.setPath(ep,filename);
  3981. }
  3982. append(rfn);
  3983. }
  3984. }
  3985. void RemoteMultiFilename::append(const RemoteFilename &inrfn)
  3986. {
  3987. RemoteFilename rfn;
  3988. rfn.set(inrfn);
  3989. if (ep.isNull())
  3990. ep = rfn.queryEndpoint();
  3991. else if (rfn.queryIP().isNull()) {
  3992. rfn.setEp(ep);
  3993. }
  3994. else if (!rfn.queryIP().ipequals(ep)) {
  3995. StringBuffer path;
  3996. rfn.getRemotePath(path);
  3997. throw MakeStringException(-1, "Component file IP does not match: %s", path.str());
  3998. }
  3999. RemoteFilenameArray::append(rfn);
  4000. }
  4001. void RemoteMultiFilename::deserialize(MemoryBuffer & in)
  4002. {
  4003. clear();
  4004. ep.deserialize(in);
  4005. unsigned n;
  4006. in.read(n);
  4007. StringBuffer last;
  4008. StringBuffer filename;
  4009. RemoteFilename rfn;
  4010. for (unsigned i=0;i<n;i++) {
  4011. byte l;
  4012. in.read(l);
  4013. filename.clear();
  4014. if (l)
  4015. filename.append((size32_t)l,last.str());
  4016. StringAttr s;
  4017. in.read(s);
  4018. filename.append(s);
  4019. rfn.setPath(ep,filename.str());
  4020. RemoteFilenameArray::append(rfn);
  4021. last.swapWith(filename);
  4022. }
  4023. }
  4024. void RemoteMultiFilename::serialize(MemoryBuffer & out)
  4025. {
  4026. ep.serialize(out);
  4027. unsigned n=ordinality();
  4028. out.append(n);
  4029. // do simple compression
  4030. StringBuffer last;
  4031. StringBuffer filename;
  4032. for (unsigned i=0;i<n;i++) {
  4033. item(i).getLocalPath(filename.clear());
  4034. const char *s1 = last.str();
  4035. const char *s2 = filename.str();
  4036. byte l=0;
  4037. while ((l<255)&&*s1&&*s2&&(*s1==*s2)) {
  4038. s1++;
  4039. s2++;
  4040. l++;
  4041. }
  4042. out.append(l);
  4043. out.append(s2);
  4044. last.swapWith(filename);
  4045. }
  4046. }
  4047. bool RemoteMultiFilename::isWild(unsigned idx) const
  4048. {
  4049. if (idx==(unsigned)-1) {
  4050. ForEachItem(i)
  4051. if (isWild(i))
  4052. return true;
  4053. }
  4054. else {
  4055. StringBuffer s;
  4056. item(idx).getTail(s);
  4057. if (containsFileWildcard(s.str()))
  4058. return true;
  4059. }
  4060. return false;
  4061. }
  4062. void RemoteMultiFilename::expandWild()
  4063. {
  4064. bool anywild = false;
  4065. BoolArray iswild;
  4066. ForEachItem(i1) {
  4067. if (isWild(i1)) {
  4068. anywild = true;
  4069. iswild.append(true);
  4070. }
  4071. else
  4072. iswild.append(false);
  4073. }
  4074. if (!anywild)
  4075. return; // nothing to do
  4076. // first cache old values (bit long winded but want to preserve order)
  4077. RemoteFilenameArray tmpa;
  4078. Int64Array tmpsz;
  4079. ForEachItem(i2) {
  4080. RemoteFilename rfn(item(i2));
  4081. tmpa.append(rfn);
  4082. if (i2<sizescache.ordinality())
  4083. tmpsz.append(sizescache.item(i2));
  4084. else
  4085. tmpsz.append(-1);
  4086. }
  4087. RemoteFilenameArray::kill();
  4088. sizescache.kill();
  4089. ForEachItemIn(i3,tmpa) {
  4090. RemoteFilename rfn(tmpa.item(i3));
  4091. if (iswild.item(i3)) {
  4092. StringBuffer name;
  4093. rfn.getLocalPath(name);
  4094. const char *s=name.str();
  4095. const char *t=s;
  4096. loop {
  4097. const char *sep=findPathSepChar(t);
  4098. if (!sep)
  4099. break;
  4100. t = sep+1;
  4101. }
  4102. StringAttr tail(t);
  4103. name.setLength(t-s);
  4104. rfn.setPath(rfn.queryEndpoint(),name);
  4105. Owned<IFile> dir = createIFile(rfn);
  4106. Owned<IDirectoryIterator> iter = dir->directoryFiles(tail.get());
  4107. ForEach(*iter) {
  4108. append(iter->query().queryFilename());
  4109. sizescache.append(iter->getFileSize());
  4110. }
  4111. }
  4112. else {
  4113. append(rfn);
  4114. sizescache.append(tmpsz.item(i3));
  4115. }
  4116. }
  4117. }
  4118. offset_t RemoteMultiFilename::getSize(unsigned i)
  4119. {
  4120. __int64 ret = (i<sizescache.ordinality())?sizescache.item(i):-1;
  4121. if (ret==-1) {
  4122. RemoteFilename rfn(item(i));
  4123. Owned<IFile> file = createIFile(rfn);
  4124. ret = file->size();
  4125. }
  4126. return (offset_t)ret;
  4127. }
  4128. void RemoteMultiFilename::setIp(const IpAddress & ip)
  4129. {
  4130. ep.ipset(ip);
  4131. ForEachItem(i)
  4132. item(i).setIp(ip);
  4133. }
  4134. void RemoteMultiFilename::setEp(const SocketEndpoint & _ep)
  4135. {
  4136. ep.set(_ep);
  4137. ForEachItem(i)
  4138. item(i).setEp(_ep);
  4139. }
  4140. void RemoteMultiFilename::setPort(unsigned short port)
  4141. {
  4142. ep.port = port;
  4143. ForEachItem(i)
  4144. item(i).setPort(port);
  4145. }
  4146. void RemoteMultiFilename::set(const RemoteMultiFilename & other)
  4147. {
  4148. clear();
  4149. ep.set(other.ep);
  4150. ForEachItemIn(i,other) {
  4151. append(other.item(i));
  4152. if (i<other.sizescache.ordinality())
  4153. sizescache.append(other.sizescache.item(i));
  4154. }
  4155. }
  4156. bool RemoteMultiFilename::equals(const RemoteMultiFilename & other)
  4157. {
  4158. if (!ep.equals(other.ep))
  4159. return false;
  4160. if (ordinality()!=other.ordinality())
  4161. return false;
  4162. ForEachItem(i)
  4163. if (!item(i).equals(other.item(i)))
  4164. return false;
  4165. return true;
  4166. }
  4167. void RemoteMultiFilename::expand(const char *mpath, StringArray &array)
  4168. {
  4169. StringBuffer path;
  4170. StringBuffer fullpath;
  4171. loop {
  4172. while (isspace(*mpath))
  4173. mpath++;
  4174. if (!*mpath)
  4175. break;
  4176. bool inquote=false;
  4177. while (*mpath) {
  4178. if (*mpath=='"') {
  4179. mpath++;
  4180. if (inquote) {
  4181. if (*mpath!='"') {
  4182. inquote = false;
  4183. continue;
  4184. }
  4185. }
  4186. else {
  4187. inquote = true;
  4188. continue;
  4189. }
  4190. }
  4191. if ((*mpath==',')&&!inquote) {
  4192. mpath++;
  4193. break;
  4194. }
  4195. path.append(*mpath);
  4196. mpath++;
  4197. }
  4198. path.clip();
  4199. if (path.length()) {
  4200. array.append(path.str());
  4201. path.clear();
  4202. }
  4203. }
  4204. }
  4205. void RemoteMultiFilename::tostr(StringArray &array,StringBuffer &out)
  4206. {
  4207. ForEachItemIn(i,array) {
  4208. const char *s = array.item(i);
  4209. if (!s||!*s)
  4210. continue;
  4211. if (i!=0)
  4212. out.append(',');
  4213. bool needquote=false;
  4214. for (const char *e=s;*e;e++)
  4215. if (isspace(*e)||(*e==',')) {
  4216. needquote = true;
  4217. break;
  4218. }
  4219. if (needquote)
  4220. out.append('"');
  4221. out.append(s);
  4222. if (needquote)
  4223. out.append('"');
  4224. }
  4225. }
  4226. //===================================================================================================
  4227. static IArrayOf<IFileCreateHook> hooks;
  4228. static CriticalSection hooksect;
  4229. void addIFileCreateHook(IFileCreateHook *hook)
  4230. {
  4231. if (hook) {
  4232. hook->Link();
  4233. CriticalBlock block(hooksect);
  4234. hooks.append(*hook);
  4235. }
  4236. }
  4237. void removeIFileCreateHook(IFileCreateHook *hook)
  4238. {
  4239. CriticalBlock block(hooksect);
  4240. hooks.zap(*hook);
  4241. }
  4242. static IFile *createIFileByHook(const RemoteFilename & filename)
  4243. {
  4244. CriticalBlock block(hooksect); // this may be a problem (serializing opens)
  4245. ForEachItemIn(i,hooks) {
  4246. IFile * ret = hooks.item(i).createIFile(filename);
  4247. if (ret)
  4248. return ret;
  4249. }
  4250. return NULL;
  4251. }
  4252. IFile * createIFile(const RemoteFilename & filename)
  4253. {
  4254. IFile * ret = createIFileByHook(filename);
  4255. if (ret)
  4256. return ret;
  4257. StringBuffer name;
  4258. return createIFile(getLocalOrRemoteName(name,filename).str());
  4259. }
  4260. StringBuffer &makeAbsolutePath(const char *relpath,StringBuffer &out)
  4261. {
  4262. if (isPathSepChar(relpath[0])&&(relpath[0]==relpath[1]))
  4263. return out.append(relpath); // if remote then already should be absolute
  4264. #ifdef _WIN32
  4265. char path[256];
  4266. path[0] = 0;
  4267. char *filepart;
  4268. GetFullPathName(relpath, sizeof(path), path, &filepart);
  4269. #else
  4270. char path[PATH_MAX+1];
  4271. path[0] = 0;
  4272. realpath(relpath,path);
  4273. #endif
  4274. return out.append(path);
  4275. }
  4276. const char *splitRelativePath(const char *full,const char *basedir,StringBuffer &reldir)
  4277. {
  4278. if (basedir&&*basedir) {
  4279. size32_t bl = strlen(basedir);
  4280. if (isPathSepChar(basedir[bl-1]))
  4281. bl--;
  4282. if ((memicmp(full,basedir,bl)==0)&&isPathSepChar(full[bl]))
  4283. full += bl+1;
  4284. }
  4285. const char *t = full;
  4286. loop {
  4287. const char *n = findPathSepChar(t);
  4288. if (!n)
  4289. break;
  4290. t = n+1;
  4291. }
  4292. if (t!=full)
  4293. reldir.append(t-full,full);
  4294. return t;
  4295. }
  4296. const char *splitDirMultiTail(const char *multipath,StringBuffer &dir,StringBuffer &tail)
  4297. {
  4298. // the first directory is the significant one
  4299. // others only removed if same
  4300. dir.clear();
  4301. StringArray files;
  4302. RemoteMultiFilename::expand(multipath,files);
  4303. StringBuffer reldir;
  4304. ForEachItemIn(i,files) {
  4305. const char *s = files.item(i);
  4306. if (i==0) {
  4307. if (isAbsolutePath(s)) {
  4308. StringAttr tail(splitDirTail(s,dir));
  4309. if (dir.length())
  4310. files.replace(tail,i);
  4311. }
  4312. }
  4313. else if (dir.length()) {
  4314. s= splitRelativePath(s,dir.str(),reldir.clear());
  4315. if (reldir.length()) {
  4316. reldir.append(s);
  4317. files.replace(reldir.str(),i);
  4318. }
  4319. }
  4320. }
  4321. RemoteMultiFilename::tostr(files,tail);
  4322. return tail.str();
  4323. }
  4324. StringBuffer &mergeDirMultiTail(const char *dir,const char *tail, StringBuffer &multipath)
  4325. {
  4326. StringArray files;
  4327. RemoteMultiFilename::expand(tail,files);
  4328. StringBuffer reldir;
  4329. if (dir && *dir) {
  4330. ForEachItemIn(i,files) {
  4331. const char *s = files.item(i);
  4332. if (!isAbsolutePath(s)) {
  4333. reldir.clear().append(dir);
  4334. addPathSepChar(reldir).append(s);
  4335. files.replace(reldir.str(),i);
  4336. }
  4337. }
  4338. }
  4339. RemoteMultiFilename::tostr(files,multipath);
  4340. return multipath;
  4341. }
  4342. StringBuffer &removeRelativeMultiPath(const char *full,const char *reldir,StringBuffer &res)
  4343. {
  4344. StringArray files;
  4345. RemoteMultiFilename::expand(full,files);
  4346. StringBuffer tmp1;
  4347. StringBuffer tmp2;
  4348. StringBuffer dir;
  4349. ForEachItemIn(i,files) {
  4350. const char *s = files.item(i);
  4351. if (isAbsolutePath(s)) {
  4352. if (!dir.length())
  4353. splitDirTail(s,dir);
  4354. }
  4355. else if (dir.length()) {
  4356. tmp1.clear().append(dir);
  4357. addPathSepChar(tmp1).append(s);
  4358. s = tmp1.str();
  4359. }
  4360. s = splitRelativePath(s,reldir,tmp2.clear());
  4361. tmp2.append(s);
  4362. files.replace(tmp2.str(),i);
  4363. }
  4364. RemoteMultiFilename::tostr(files,res);
  4365. return res;
  4366. }
  4367. // removes basedir if matches, returns relative multipath
  4368. //===================================================================================================
  4369. ExtractedBlobInfo::ExtractedBlobInfo(const char * _filename, offset_t _length, offset_t _offset)
  4370. {
  4371. filename.set(_filename);
  4372. length = _length;
  4373. offset = _offset;
  4374. }
  4375. void ExtractedBlobInfo::serialize(MemoryBuffer & buffer)
  4376. {
  4377. ::serialize(buffer, filename.get());
  4378. buffer.append(length).append(offset);
  4379. }
  4380. void ExtractedBlobInfo::deserialize(MemoryBuffer & buffer)
  4381. {
  4382. ::deserialize(buffer, filename);
  4383. buffer.read(length).read(offset);
  4384. }
  4385. //----------------------------------------------------------------------------
  4386. #define DFTERR_InvalidSplitPrefixFormat 8091
  4387. #define DFTERR_InvalidSplitPrefixFormat_Text "Cannot process file %s using the splitprefix supplied"
  4388. static void * readLength(MemoryBuffer & buffer, IFileIOStream * in, size_t len, const char * filenameText)
  4389. {
  4390. void * ptr = buffer.clear().reserve(len);
  4391. if (in->read(len, ptr) != len)
  4392. throwError1(DFTERR_InvalidSplitPrefixFormat, filenameText);
  4393. return ptr;
  4394. }
  4395. void extractBlobElements(const char * prefix, const RemoteFilename &filename, ExtractedBlobArray & extracted)
  4396. {
  4397. StringBuffer filenameText;
  4398. filename.getPath(filenameText);
  4399. Owned<IFile> file = createIFile(filename);
  4400. Owned<IFileIO> inIO = file->open(IFOread);
  4401. if (!inIO)
  4402. throw MakeStringException(-1, "extractBlobElements: file '%s' not found", filenameText.str());
  4403. Owned<IFileIOStream> in = createIOStream(inIO);
  4404. MemoryBuffer buffer;
  4405. offset_t endOffset = in->size();
  4406. while (in->tell() != endOffset)
  4407. {
  4408. StringAttr blobFilename;
  4409. offset_t blobLength = (offset_t)-1;
  4410. const char * finger = prefix;
  4411. while (finger)
  4412. {
  4413. StringAttr command;
  4414. const char * comma = strchr(finger, ',');
  4415. if (comma)
  4416. {
  4417. command.set(finger, comma-finger);
  4418. finger = comma+1;
  4419. }
  4420. else
  4421. {
  4422. command.set(finger);
  4423. finger = NULL;
  4424. }
  4425. command.toUpperCase();
  4426. if (memcmp(command, "FILENAME", 8) == 0)
  4427. {
  4428. if (command[8] == ':')
  4429. {
  4430. unsigned maxLen = atoi(command+9);
  4431. const char * nameptr = (const char *)readLength(buffer, in, maxLen, filenameText.str());
  4432. blobFilename.set(nameptr, maxLen);
  4433. }
  4434. else
  4435. {
  4436. unsigned * lenptr = (unsigned *)readLength(buffer, in, sizeof(unsigned), filenameText.str());
  4437. #if __BYTE_ORDER != __LITTLE_ENDIAN
  4438. _rev(sizeof(*lenptr), lenptr);
  4439. #endif
  4440. unsigned filenamelen = *lenptr;
  4441. const char * nameptr = (const char *)readLength(buffer, in, filenamelen, filenameText.str());
  4442. blobFilename.set(nameptr, filenamelen);
  4443. }
  4444. }
  4445. else if ((memcmp(command, "FILESIZE", 8) == 0) || (command.length() == 2))
  4446. {
  4447. const char * format = command;
  4448. if (memcmp(format, "FILESIZE", 8) == 0)
  4449. {
  4450. if (format[8] == ':')
  4451. format = format+9;
  4452. else
  4453. format = "L4";
  4454. }
  4455. bool bigEndian;
  4456. char c = format[0];
  4457. if (c == 'B')
  4458. bigEndian = true;
  4459. else if (c == 'L')
  4460. bigEndian = false;
  4461. else
  4462. throwError1(DFTERR_InvalidSplitPrefixFormat, format);
  4463. c = format[1];
  4464. if ((c <= '0') || (c > '8'))
  4465. throwError1(DFTERR_InvalidSplitPrefixFormat, format);
  4466. unsigned length = (c - '0');
  4467. byte * lenptr = (byte *)readLength(buffer, in, length, filenameText.str());
  4468. if (!bigEndian)
  4469. _rev(length, lenptr);
  4470. blobLength = 0;
  4471. for (unsigned i=0; i<length; i++)
  4472. {
  4473. blobLength = (blobLength << 8) | lenptr[i];
  4474. }
  4475. }
  4476. else if (memcmp(command, "SKIP:", 5) == 0)
  4477. {
  4478. unsigned skipLen = atoi(command+5);
  4479. in->seek(in->tell()+skipLen, IFSbegin);
  4480. }
  4481. else if (memcmp(command, "SEQ:", 4) == 0)
  4482. {
  4483. unsigned skipLen = atoi(command+4);
  4484. in->seek(in->tell()+skipLen, IFSbegin);
  4485. }
  4486. else
  4487. throwError1(DFTERR_InvalidSplitPrefixFormat, command.get());
  4488. }
  4489. if ((blobLength == (offset_t)-1) || !blobFilename.get())
  4490. throwError1(DFTERR_InvalidSplitPrefixFormat, filenameText.str());
  4491. offset_t blobOffset = in->tell();
  4492. extracted.append(* new ExtractedBlobInfo(blobFilename, blobLength, blobOffset));
  4493. in->seek(blobOffset + blobLength, IFSbegin);
  4494. }
  4495. }
  4496. bool mountDrive(const char *drv,const RemoteFilename &rfn)
  4497. {
  4498. #ifdef _WIN32
  4499. return false;
  4500. #else
  4501. unmountDrive(drv);
  4502. localCreateDirectory(drv);
  4503. int ret;
  4504. for (unsigned vtry=0;vtry<2;vtry++) {
  4505. StringBuffer cmd;
  4506. cmd.append("mount ");
  4507. rfn.queryIP().getIpText(cmd);
  4508. cmd.append(':');
  4509. rfn.getLocalPath(cmd);
  4510. cmd.append(' ').append(drv).append(" -t nfs ");
  4511. if (vtry==0)
  4512. cmd.append("-o nfsvers=v3 "); // prefer v3
  4513. cmd.append("2> /dev/null");
  4514. ret = system(cmd.str());
  4515. if (ret==0)
  4516. break;
  4517. }
  4518. return (ret==0);
  4519. #endif
  4520. }
  4521. bool unmountDrive(const char *drv)
  4522. {
  4523. #ifdef _WIN32
  4524. return false;
  4525. #else
  4526. StringBuffer cmd;
  4527. cmd.append("umount ").append(drv).append(" 2> /dev/null");
  4528. int ret = system(cmd.str());
  4529. return (ret==0);
  4530. #endif
  4531. }
  4532. IFileIO *createUniqueFile(const char *dir, const char *prefix, StringBuffer &filename)
  4533. {
  4534. CDateTime dt;
  4535. dt.setNow();
  4536. unsigned t = (unsigned)dt.getSimple();
  4537. if (dir)
  4538. {
  4539. filename.append(dir);
  4540. addPathSepChar(filename);
  4541. }
  4542. if (prefix && *prefix)
  4543. filename.append(prefix);
  4544. else
  4545. filename.append("uniq");
  4546. filename.appendf("_%"I64F"x.%x.%x.tmp", (__int64)GetCurrentThreadId(), (unsigned)GetCurrentProcessId(), t);
  4547. OwnedIFile iFile = createIFile(filename.str());
  4548. IFileIO *iFileIO = NULL;
  4549. unsigned attempts = 5; // max attempts
  4550. loop
  4551. {
  4552. if (!iFile->exists())
  4553. {
  4554. try { return iFile->openShared(IFOcreate, IFSHnone); } // NB: could be null if path not found
  4555. catch (IException *e)
  4556. {
  4557. EXCLOG(e, "createUniqueFile");
  4558. e->Release();
  4559. }
  4560. }
  4561. if (0 == --attempts)
  4562. break;
  4563. t += getRandom();
  4564. filename.clear().appendf("uniq_%"I64F"x.%x.%x.tmp", (__int64)GetCurrentThreadId(), (unsigned)GetCurrentProcessId(), t);
  4565. iFile.setown(createIFile(filename.str()));
  4566. }
  4567. return NULL;
  4568. }
  4569. unsigned sortDirectory( CIArrayOf<CDirectoryEntry> &sortedfiles,
  4570. IDirectoryIterator &iter,
  4571. SortDirectoryMode mode,
  4572. bool rev,
  4573. bool includedirs
  4574. )
  4575. {
  4576. sortedfiles.kill();
  4577. StringBuffer name;
  4578. ForEach(iter) {
  4579. if (!iter.isDir()||includedirs)
  4580. sortedfiles.append(*new CDirectoryEntry(iter));
  4581. }
  4582. if (mode!=SD_nosort) {
  4583. struct icmp: implements ICompare
  4584. {
  4585. SortDirectoryMode mode;
  4586. bool rev;
  4587. int docompare(const void *l,const void *r) const
  4588. {
  4589. int ret=0;
  4590. const CDirectoryEntry *dl = (const CDirectoryEntry *)l;
  4591. const CDirectoryEntry *dr = (const CDirectoryEntry *)r;
  4592. switch (mode) {
  4593. case SD_byname:
  4594. ret = strcmp(dl->name,dr->name);
  4595. break;
  4596. case SD_bynameNC:
  4597. ret = stricmp(dl->name,dr->name);
  4598. break;
  4599. case SD_bydate:
  4600. ret = dl->modifiedTime.compare(dr->modifiedTime);
  4601. break;
  4602. case SD_bysize:
  4603. ret = (dl->size>dr->size)?1:((dl->size<dr->size)?-1:0);
  4604. break;
  4605. }
  4606. if (rev)
  4607. ret = -ret;
  4608. return ret;
  4609. }
  4610. } cmp;
  4611. cmp.mode = mode;
  4612. cmp.rev = rev;
  4613. qsortvec((void **)sortedfiles.getArray(), sortedfiles.ordinality(), cmp);
  4614. }
  4615. return sortedfiles.ordinality();
  4616. }
  4617. class CReplicatedFile : public CInterface, implements IReplicatedFile
  4618. {
  4619. RemoteFilenameArray copies;
  4620. public:
  4621. IMPLEMENT_IINTERFACE;
  4622. RemoteFilenameArray &queryCopies()
  4623. {
  4624. return copies;
  4625. }
  4626. IFile *open()
  4627. {
  4628. StringBuffer locations;
  4629. Owned<IException> exc;
  4630. ForEachItemIn(copy,copies) {
  4631. RemoteFilename &rfn = copies.item(copy);
  4632. try {
  4633. OwnedIFile iFile = createIFile(rfn);
  4634. if (iFile->exists())
  4635. return iFile.getClear();
  4636. if (locations.length())
  4637. locations.append(", ");
  4638. rfn.getRemotePath(locations);
  4639. }
  4640. catch(IException *e) {
  4641. EXCLOG(e,"CReplicatedFile::open");
  4642. if (exc)
  4643. e->Release();
  4644. else
  4645. exc.setown(e);
  4646. }
  4647. }
  4648. if (exc.get())
  4649. throw exc.getClear();
  4650. throw MakeStringException(0, "%s: Failed to open part file at any of the following locations: ", locations.str());
  4651. }
  4652. };
  4653. IReplicatedFile *createReplicatedFile()
  4654. {
  4655. return new CReplicatedFile;
  4656. }
  4657. // ---------------------------------------------------------------------------------
  4658. class CSerialStreamBase : public CInterface, implements ISerialStream
  4659. {
  4660. private:
  4661. size32_t bufsize;
  4662. size32_t bufpos;
  4663. size32_t bufmax;
  4664. offset_t bufbase;
  4665. offset_t endpos; // -1 if not known
  4666. MemoryAttr ma;
  4667. byte *buf;
  4668. bool eoinput;
  4669. IFileSerialStreamCallback *tally;
  4670. inline size32_t doread(offset_t pos, size32_t max_size, void *ptr)
  4671. {
  4672. if (endpos!=(offset_t)-1) {
  4673. if (pos>=endpos)
  4674. return 0;
  4675. if (endpos-pos<max_size)
  4676. max_size = (size32_t)(endpos-pos);
  4677. }
  4678. size32_t size_read = rawread(pos, max_size, ptr);
  4679. if (tally)
  4680. tally->process(pos,size_read,ptr);
  4681. return size_read;
  4682. }
  4683. const void * dopeek(size32_t sz, size32_t &got)
  4684. {
  4685. loop
  4686. {
  4687. size32_t left = bufmax-bufpos;
  4688. got = left;
  4689. if (left>=sz)
  4690. return buf+bufpos;
  4691. if (eoinput)
  4692. return left?(buf+bufpos):NULL;
  4693. size32_t reqsz = sz+bufsize; // NB not sz-left as want some slack
  4694. if (ma.length()<reqsz) {
  4695. MemoryAttr ma2;
  4696. void *nb = ma2.allocate(reqsz);
  4697. memcpy(nb,buf+bufpos,left);
  4698. ma.setOwn(reqsz,ma2.detach());
  4699. buf = (byte *)nb;
  4700. }
  4701. else
  4702. memmove(buf,buf+bufpos,left);
  4703. bufbase += bufpos;
  4704. size32_t rd = doread(bufbase+left,bufsize,buf+left);
  4705. if (!rd)
  4706. eoinput = true;
  4707. bufmax = rd+left;
  4708. bufpos = 0;
  4709. }
  4710. }
  4711. protected:
  4712. virtual size32_t rawread(offset_t pos, size32_t max_size, void *ptr) = 0;
  4713. public:
  4714. IMPLEMENT_IINTERFACE;
  4715. CSerialStreamBase(offset_t _offset, offset_t _len, size32_t _bufsize, IFileSerialStreamCallback *_tally)
  4716. {
  4717. tally = _tally;
  4718. bufsize = _bufsize;
  4719. if (bufsize==(size32_t)-1)
  4720. bufsize = DEFAULT_STREAM_BUFFER_SIZE;
  4721. if (bufsize<4096)
  4722. bufsize = 4096;
  4723. else
  4724. bufsize = ((bufsize+4095)/4096)*4096;
  4725. buf = (byte *)ma.allocate(bufsize+4096); // 4K initial slack
  4726. bufpos = 0;
  4727. bufmax = 0;
  4728. bufbase = _offset;
  4729. if (_len==(offset_t)-1)
  4730. endpos = (offset_t)-1;
  4731. else
  4732. endpos = _offset+_len;
  4733. eoinput = (_len==0);
  4734. }
  4735. void reset(offset_t _offset, offset_t _len)
  4736. {
  4737. bufpos = 0;
  4738. bufmax = 0;
  4739. bufbase = _offset;
  4740. if (_len==(offset_t)-1)
  4741. endpos = (offset_t)-1;
  4742. else
  4743. endpos = _offset+_len;
  4744. eoinput = (_len==0);
  4745. }
  4746. const void *peek(size32_t sz,size32_t &got)
  4747. {
  4748. return dopeek(sz, got);
  4749. }
  4750. void get(size32_t len, void * ptr)
  4751. {
  4752. size32_t cpy = bufmax-bufpos;
  4753. if (cpy>len)
  4754. cpy = len;
  4755. memcpy(ptr,(const byte *)buf+bufpos,cpy);
  4756. len -= cpy;
  4757. if (len==0) {
  4758. bufpos += cpy;
  4759. return;
  4760. }
  4761. bufbase += bufmax;
  4762. bufpos = 0;
  4763. bufmax = 0;
  4764. size32_t rd = 0;
  4765. if (!eoinput) {
  4766. ptr = (byte *)ptr+cpy;
  4767. if (len>=bufsize) {
  4768. size32_t tord = (len/bufsize)*bufsize;
  4769. rd = doread(bufbase,tord,ptr); // copy directly if large
  4770. bufbase += rd;
  4771. if (rd!=tord) {
  4772. eoinput = true;
  4773. PrintStackReport();
  4774. ERRLOG("CFileSerialStream::get read past end of stream.1 (%u,%u) %s",rd,tord,eoinput?"eoinput":"");
  4775. throw MakeStringException(-1,"CFileSerialStream::get read past end of stream");
  4776. }
  4777. cpy += rd;
  4778. len -= rd;
  4779. if (!len)
  4780. return;
  4781. ptr = (byte *)ptr+rd;
  4782. }
  4783. const void *p = dopeek(len,rd);
  4784. if (len<=rd) {
  4785. memcpy(ptr,p,len);
  4786. bufpos += len;
  4787. return;
  4788. }
  4789. }
  4790. PrintStackReport();
  4791. ERRLOG("CFileSerialStream::get read past end of stream.2 (%u,%u) %s",len,rd,eoinput?"eoinput":"");
  4792. throw MakeStringException(-1,"CFileSerialStream::get read past end of stream");
  4793. }
  4794. bool eos()
  4795. {
  4796. if (bufmax-bufpos)
  4797. return false;
  4798. size32_t rd;
  4799. return dopeek(1,rd)==NULL;
  4800. }
  4801. void skip(size32_t len)
  4802. {
  4803. size32_t left = bufmax-bufpos;
  4804. if (left>=len) {
  4805. bufpos += len;
  4806. return;
  4807. }
  4808. len -= left;
  4809. bufbase += bufmax;
  4810. bufpos = 0;
  4811. bufmax = 0;
  4812. if (!eoinput) {
  4813. while (len>=bufsize) {
  4814. size32_t rd;
  4815. if (tally) {
  4816. rd = doread(bufbase,bufsize,buf);
  4817. if (rd!=bufsize) {
  4818. eoinput = true;
  4819. throw MakeStringException(-1,"CFileSerialStream::skip read past end of stream");
  4820. }
  4821. }
  4822. else {
  4823. rd = (len/bufsize)*bufsize;
  4824. //rd=doskip(bufbase,bufsize,buf); to cope with reading from sockets etc?
  4825. }
  4826. bufbase += rd;
  4827. len -= bufsize;
  4828. }
  4829. if (len==0)
  4830. return;
  4831. size32_t got;
  4832. dopeek(len,got);
  4833. if (len<=got) {
  4834. bufpos += got;
  4835. return;
  4836. }
  4837. }
  4838. throw MakeStringException(-1,"CFileSerialStream::skip read past end of stream");
  4839. }
  4840. offset_t tell()
  4841. {
  4842. return bufbase+bufpos;
  4843. }
  4844. };
  4845. class CFileSerialStream: public CSerialStreamBase
  4846. {
  4847. Linked<IFileIO> fileio;
  4848. virtual size32_t rawread(offset_t pos, size32_t max_size, void *ptr)
  4849. {
  4850. return fileio->read(pos,max_size,ptr);
  4851. }
  4852. public:
  4853. CFileSerialStream(IFileIO *_fileio,offset_t _offset, offset_t _len, size32_t _bufsize, IFileSerialStreamCallback *_tally)
  4854. : CSerialStreamBase(_offset, _len, _bufsize, _tally), fileio(_fileio)
  4855. {
  4856. }
  4857. };
  4858. ISerialStream *createFileSerialStream(IFileIO *fileio,offset_t ofs, offset_t flen, size32_t bufsize,IFileSerialStreamCallback *callback)
  4859. {
  4860. if (!fileio)
  4861. return NULL;
  4862. return new CFileSerialStream(fileio,ofs,flen,bufsize,callback);
  4863. }
  4864. class CIoSerialStream: public CSerialStreamBase
  4865. {
  4866. Linked<IFileIOStream> io;
  4867. offset_t lastpos;
  4868. virtual size32_t rawread(offset_t pos, size32_t max_size, void *ptr)
  4869. {
  4870. if (lastpos!=pos)
  4871. throw MakeStringException(-1,"CIoSerialStream: non-sequential read (%"I64F"d,%"I64F"d)",lastpos,pos);
  4872. size32_t rd = io->read(max_size,ptr);
  4873. lastpos = pos+rd;
  4874. return rd;
  4875. }
  4876. public:
  4877. CIoSerialStream(IFileIOStream * _io,offset_t _offset, size32_t _bufsize, IFileSerialStreamCallback *_tally)
  4878. : CSerialStreamBase(_offset, (offset_t)-1, _bufsize, _tally), io(_io)
  4879. {
  4880. lastpos = _offset;
  4881. }
  4882. };
  4883. ISerialStream *createFileSerialStream(IFileIOStream *io,size32_t bufsize,IFileSerialStreamCallback *callback)
  4884. {
  4885. if (!io)
  4886. return NULL;
  4887. return new CIoSerialStream(io,0,bufsize,callback);
  4888. }
  4889. class CSocketSerialStream: public CSerialStreamBase
  4890. {
  4891. Linked<ISocket> socket;
  4892. unsigned timeout;
  4893. offset_t lastpos;
  4894. virtual size32_t rawread(offset_t pos, size32_t max_size, void *ptr)
  4895. {
  4896. if (lastpos!=pos)
  4897. throw MakeStringException(-1,"CSocketSerialStream: non-sequential read (%"I64F"d,%"I64F"d)",lastpos,pos);
  4898. size32_t size_read;
  4899. socket->readtms(ptr, 0, max_size, size_read, timeout);
  4900. lastpos = pos+size_read;
  4901. return size_read;
  4902. }
  4903. public:
  4904. CSocketSerialStream(ISocket * _socket, unsigned _timeout, offset_t _offset, size32_t _bufsize, IFileSerialStreamCallback *_tally)
  4905. : CSerialStreamBase(_offset, (offset_t)-1, _bufsize, _tally), socket(_socket), timeout(_timeout)
  4906. {
  4907. lastpos = _offset;
  4908. }
  4909. };
  4910. ISerialStream *createSocketSerialStream(ISocket * socket, unsigned timeoutms, size32_t bufsize, IFileSerialStreamCallback *callback)
  4911. {
  4912. if (!socket)
  4913. return NULL;
  4914. return new CSocketSerialStream(socket,timeoutms,0,bufsize,callback);
  4915. }
  4916. class CSimpleReadSerialStream: public CSerialStreamBase
  4917. {
  4918. Linked<ISimpleReadStream> input;
  4919. offset_t lastpos;
  4920. virtual size32_t rawread(offset_t pos, size32_t max_size, void *ptr)
  4921. {
  4922. if (lastpos!=pos)
  4923. throw MakeStringException(-1,"CSimpleReadSerialStream: non-sequential read (%"I64F"d,%"I64F"d)",lastpos,pos);
  4924. size32_t rd = input->read(max_size, ptr);
  4925. lastpos += rd;
  4926. return rd;
  4927. }
  4928. public:
  4929. CSimpleReadSerialStream(ISimpleReadStream * _input, offset_t _offset, size32_t _bufsize, IFileSerialStreamCallback *_tally)
  4930. : CSerialStreamBase(_offset, (offset_t)-1, _bufsize, _tally), input(_input)
  4931. {
  4932. lastpos = _offset;
  4933. }
  4934. void reset(offset_t _ofs, offset_t _len)
  4935. {
  4936. // assume knows what doing
  4937. lastpos = _ofs;
  4938. CSerialStreamBase::reset(_ofs,_len);
  4939. }
  4940. };
  4941. ISerialStream *createSimpleSerialStream(ISimpleReadStream * input, size32_t bufsize, IFileSerialStreamCallback *callback)
  4942. {
  4943. if (!input)
  4944. return NULL;
  4945. return new CSimpleReadSerialStream(input,0,bufsize,callback);
  4946. }
  4947. class CMemoryMappedSerialStream: public CInterface, implements ISerialStream
  4948. {
  4949. Linked<IMemoryMappedFile> mmfile;
  4950. const byte *mmbase;
  4951. memsize_t mmsize;
  4952. memsize_t mmofs;
  4953. bool eoinput;
  4954. IFileSerialStreamCallback *tally;
  4955. public:
  4956. IMPLEMENT_IINTERFACE;
  4957. CMemoryMappedSerialStream(IMemoryMappedFile *_mmfile, offset_t ofs, offset_t flen, IFileSerialStreamCallback *_tally)
  4958. : mmfile(_mmfile)
  4959. {
  4960. tally = _tally;
  4961. offset_t fs = mmfile->fileSize();
  4962. if ((memsize_t)fs!=fs)
  4963. throw MakeStringException(-1,"CMemoryMappedSerialStream file too big to be mapped");
  4964. if ((flen!=(offset_t)-1)&&(fs>flen))
  4965. fs = flen;
  4966. mmsize = (memsize_t)fs;
  4967. mmofs = (memsize_t)((ofs<fs)?ofs:fs);
  4968. mmbase = (const byte *)mmfile->base();
  4969. eoinput = false;
  4970. }
  4971. void reset(offset_t _ofs, offset_t _len)
  4972. {
  4973. offset_t fs = mmfile->fileSize();
  4974. if ((_len!=(offset_t)-1)&&(fs>_len))
  4975. fs = _len;
  4976. mmsize = (memsize_t)fs;
  4977. mmofs = (memsize_t)((_ofs<fs)?_ofs:fs);
  4978. mmsize = (memsize_t)fs;
  4979. }
  4980. CMemoryMappedSerialStream(const void *buf, memsize_t len, IFileSerialStreamCallback *_tally)
  4981. {
  4982. tally = _tally;
  4983. mmsize = len;
  4984. mmofs = 0;
  4985. mmbase = (const byte *)buf;
  4986. eoinput = false;
  4987. }
  4988. const void *peek(size32_t sz,size32_t &got)
  4989. {
  4990. memsize_t left = mmsize-mmofs;
  4991. if (sz>left)
  4992. sz = (size32_t)left;
  4993. else if (left>=UINT_MAX)
  4994. got = UINT_MAX-1;
  4995. else
  4996. got = (size32_t)left;
  4997. return mmbase+mmofs;
  4998. }
  4999. void get(size32_t len, void * ptr)
  5000. {
  5001. memsize_t left = mmsize-mmofs;
  5002. if (len>left) {
  5003. PrintStackReport();
  5004. ERRLOG("CFileSerialStream::get read past end of stream.3 (%u,%u)",(unsigned)len,(unsigned)left);
  5005. throw MakeStringException(-1,"CMemoryMappedSerialStream::get read past end of stream (%u,%u)",(unsigned)len,(unsigned)left);
  5006. }
  5007. if (tally)
  5008. tally->process(mmofs,len,mmbase+mmofs);
  5009. memcpy(ptr,mmbase+mmofs,len);
  5010. mmofs += len;
  5011. }
  5012. bool eos()
  5013. {
  5014. return (mmsize<=mmofs);
  5015. }
  5016. void skip(size32_t len)
  5017. {
  5018. memsize_t left = mmsize-mmofs;
  5019. if (len>left)
  5020. throw MakeStringException(-1,"CMemoryMappedSerialStream::skip read past end of stream (%u,%u)",(unsigned)len,(unsigned)left);
  5021. if (tally)
  5022. tally->process(mmofs,len,mmbase+mmofs);
  5023. mmofs += len;
  5024. }
  5025. offset_t tell()
  5026. {
  5027. return mmofs;
  5028. }
  5029. virtual void reset(offset_t _offset)
  5030. {
  5031. mmofs = (memsize_t)((_offset<mmsize)?_offset:mmsize);
  5032. eoinput = false;
  5033. }
  5034. };
  5035. ISerialStream *createFileSerialStream(IMemoryMappedFile *mmfile, offset_t ofs, offset_t flen, IFileSerialStreamCallback *callback)
  5036. {
  5037. return new CMemoryMappedSerialStream(mmfile,ofs,flen,callback);
  5038. }
  5039. ISerialStream *createMemorySerialStream(const void *buffer, memsize_t len, IFileSerialStreamCallback *callback)
  5040. {
  5041. return new CMemoryMappedSerialStream(buffer,len,callback);
  5042. }
  5043. class CMemoryBufferSerialStream: public CInterface, implements ISerialStream
  5044. {
  5045. MemoryBuffer & buffer;
  5046. IFileSerialStreamCallback *tally;
  5047. public:
  5048. IMPLEMENT_IINTERFACE;
  5049. CMemoryBufferSerialStream(MemoryBuffer & _buffer, IFileSerialStreamCallback * _tally)
  5050. : buffer(_buffer), tally(_tally)
  5051. {
  5052. }
  5053. virtual const void *peek(size32_t sz,size32_t &got)
  5054. {
  5055. got = buffer.remaining();
  5056. return buffer.readDirect(0);
  5057. }
  5058. virtual void get(size32_t len, void * ptr)
  5059. {
  5060. if (len>buffer.remaining()) {
  5061. ERRLOG("CFileSerialStream::get read past end of stream.4(%u,%u)",(unsigned)len,(unsigned)buffer.remaining());
  5062. throw MakeStringException(-1,"CMemoryBufferSerialStream::get read past end of stream (%u,%u)",(unsigned)len,(unsigned)buffer.remaining());
  5063. }
  5064. const void * data = buffer.readDirect(len);
  5065. if (tally)
  5066. tally->process(buffer.getPos()-len,len,data);
  5067. memcpy(ptr,data,len);
  5068. }
  5069. virtual bool eos()
  5070. {
  5071. return buffer.remaining() == 0;
  5072. }
  5073. virtual void skip(size32_t len)
  5074. {
  5075. if (len>buffer.remaining())
  5076. throw MakeStringException(-1,"CMemoryBufferSerialStream::skip read past end of stream (%u,%u)",(unsigned)len,(unsigned)buffer.remaining());
  5077. const void * data = buffer.readDirect(len);
  5078. if (tally)
  5079. tally->process(buffer.getPos()-len,len,data);
  5080. }
  5081. virtual offset_t tell()
  5082. {
  5083. return buffer.getPos();
  5084. }
  5085. virtual void reset(offset_t _offset,offset_t _len)
  5086. {
  5087. size32_t ofs = (size32_t)_offset;
  5088. assertex(ofs==_offset);
  5089. assertex((_len==(offset_t)-1)||(_len>=buffer.length())); // don't support len on memory buffer
  5090. buffer.reset(ofs);
  5091. }
  5092. };
  5093. ISerialStream *createMemoryBufferSerialStream(MemoryBuffer & buffer, IFileSerialStreamCallback *callback)
  5094. {
  5095. return new CMemoryBufferSerialStream(buffer,callback);
  5096. }
  5097. // Memory Mapped Files
  5098. #define MEMORYMAP_PAGESIZE (0x1000) // could be different but won't ever be!
  5099. class CMemoryMappedFile: public CInterface, implements IMemoryMappedFile
  5100. {
  5101. byte *ptr; // base
  5102. offset_t ofs;
  5103. offset_t realofs; // rounded down to page size
  5104. offset_t filesize;
  5105. memsize_t size;
  5106. bool writeaccess;
  5107. memsize_t windowsize;
  5108. HANDLE hfile;
  5109. #ifdef _WIN32
  5110. static size32_t pagesize;
  5111. HANDLE hmap;
  5112. #endif
  5113. Linked<IFile> file;
  5114. inline offset_t pageround(offset_t o)
  5115. {
  5116. #ifdef _WIN32
  5117. return o-(o%pagesize);
  5118. #else
  5119. return o-(o%MEMORYMAP_PAGESIZE);
  5120. #endif
  5121. }
  5122. inline void * realptr()
  5123. {
  5124. return ptr?(ptr-(ofs-realofs)):NULL;
  5125. }
  5126. inline memsize_t realsize()
  5127. {
  5128. return (memsize_t)(size+(ofs-realofs));
  5129. }
  5130. public:
  5131. IMPLEMENT_IINTERFACE;
  5132. CMemoryMappedFile(HANDLE _hfile, offset_t _filesize, offset_t _ofs, memsize_t _size, bool _writeaccess)
  5133. {
  5134. hfile = _hfile;
  5135. #ifdef _WIN32
  5136. if (pagesize==0) {
  5137. SYSTEM_INFO sysinfo;
  5138. GetSystemInfo(&sysinfo);
  5139. pagesize = sysinfo.dwAllocationGranularity;
  5140. }
  5141. hmap = NULLFILE;
  5142. #endif
  5143. ptr = NULL;
  5144. filesize = _filesize;
  5145. windowsize = _size; // first open size assumed to be window size
  5146. reinit(_ofs, _size, _writeaccess);
  5147. }
  5148. ~CMemoryMappedFile()
  5149. {
  5150. #ifdef _WIN32
  5151. if (hmap!=NULLFILE)
  5152. CloseHandle(hmap);
  5153. CloseHandle(hfile);
  5154. #else
  5155. close(hfile);
  5156. #endif
  5157. }
  5158. byte *base() { return ptr; }
  5159. offset_t offset() { return ofs; }
  5160. virtual memsize_t length() { return size; }
  5161. virtual offset_t fileSize() { return filesize; }
  5162. virtual int compareWithin(const void *p)
  5163. {
  5164. if (p<ptr)
  5165. return -1;
  5166. if (p>=size+ptr)
  5167. return 1;
  5168. return 0;
  5169. }
  5170. bool writeAccess() { return writeaccess; }
  5171. void flush()
  5172. {
  5173. if (ptr) {
  5174. #ifdef _WIN32
  5175. FlushViewOfFile(realptr(),0);
  5176. #elif defined(__linux__)
  5177. msync(realptr(),realsize(),MS_SYNC);
  5178. #else
  5179. UNIMPLEMENTED;
  5180. #endif
  5181. }
  5182. }
  5183. byte *nextPtr(const void *p,offset_t skip, memsize_t req, memsize_t &got)
  5184. {
  5185. // for scanning in sequence
  5186. if (p==NULL)
  5187. p = ptr;
  5188. else if ((p<ptr)||(p>ptr+size))
  5189. throw MakeStringException(-1,"CMemoryMappedFile::nextPtr - outside map");
  5190. memsize_t d = (byte *)p-ptr;
  5191. offset_t o = ofs+d+skip;
  5192. if (o>=filesize) {
  5193. got = 0;
  5194. return NULL;
  5195. }
  5196. offset_t left = filesize-o;
  5197. if (left<req)
  5198. req = (memsize_t)left;
  5199. if (o+req-ofs>size) {
  5200. reinit(o,windowsize,writeaccess);
  5201. assertex(o==ofs);
  5202. got = windowsize;
  5203. if (left<got)
  5204. got = (memsize_t)left;
  5205. return ptr;
  5206. }
  5207. got = (memsize_t)(size+o-ofs);
  5208. if (left<got)
  5209. got = (memsize_t)left;
  5210. return (byte *)p+skip;
  5211. }
  5212. virtual void reinit(offset_t _ofs, memsize_t _size, bool _writeaccess)
  5213. {
  5214. writeaccess = _writeaccess;
  5215. if (ptr) {
  5216. #ifdef _WIN32
  5217. // no need to unmap view
  5218. if (hmap != INVALID_HANDLE_VALUE) {
  5219. CloseHandle(hmap);
  5220. hmap = INVALID_HANDLE_VALUE;
  5221. }
  5222. #elif defined(__linux__)
  5223. munmap(realptr(),realsize());
  5224. // error checking TBD
  5225. #else
  5226. UNIMPLEMENTED;
  5227. #endif
  5228. ptr = NULL;
  5229. }
  5230. if (_ofs>filesize)
  5231. ofs = filesize;
  5232. else
  5233. ofs = _ofs;
  5234. realofs = pageround(ofs);
  5235. if (_size==(memsize_t)-1) {
  5236. size = (memsize_t)(filesize-_ofs);
  5237. if (size!=(filesize-_ofs))
  5238. throw MakeStringException(-1,"CMemoryMappedFile::reinit file too big");
  5239. }
  5240. else
  5241. size = _size;
  5242. memsize_t mapsz = realsize();
  5243. if (filesize-realofs<mapsz)
  5244. mapsz = (memsize_t)(filesize-realofs);
  5245. #ifdef _WIN32
  5246. LARGE_INTEGER li;
  5247. if (hmap == INVALID_HANDLE_VALUE) {
  5248. hmap = CreateFileMapping(hfile,NULL,writeaccess?PAGE_READWRITE:PAGE_READONLY,0, 0, NULL);
  5249. if (!hmap) {
  5250. DWORD err = GetLastError();
  5251. throw MakeOsException(err,"CMemoryMappedFile::reinit");
  5252. }
  5253. }
  5254. li.QuadPart = realofs;
  5255. ptr = (byte *) MapViewOfFile(hmap, writeaccess?(FILE_MAP_READ|FILE_MAP_WRITE):FILE_MAP_READ, li.HighPart, li.LowPart, mapsz);
  5256. if (!ptr) {
  5257. DWORD err = GetLastError();
  5258. throw MakeOsException(err,"CMemoryMappedFile::reinit");
  5259. }
  5260. #elif defined (_linux__)
  5261. ptr = (byte *) mmap(NULL, mapsz, writeaccess?(PROT_READ|PROT_WRITE):PROT_READ, MAP_SHARED|MAP_NORESERVE, hfile, realofs);
  5262. // error checking TBD
  5263. #else
  5264. UNIMPLEMENTED;
  5265. #endif
  5266. if (ptr)
  5267. ptr += (ofs-realofs);
  5268. }
  5269. };
  5270. #ifdef _WIN32
  5271. size32_t CMemoryMappedFile::pagesize=0;
  5272. #endif
  5273. // TBD MADV_SEQUENTIAL & MADV_WILLNEED?
  5274. IMemoryMappedFile *CFile::openMemoryMapped(offset_t ofs, memsize_t len, bool write)
  5275. {
  5276. HANDLE hfile = openHandle(write?IFOcreaterw:IFOread, IFSHread, false);
  5277. if (hfile == NULLFILE)
  5278. return NULL;
  5279. return new CMemoryMappedFile(hfile,size(),ofs,len,write);
  5280. }
  5281. bool isSpecialPath(const char *path)
  5282. {
  5283. // used for remote queries
  5284. if (!path)
  5285. return false;
  5286. if (isPathSepChar(path[0])&&(path[0]==path[1])) {
  5287. path += 2;
  5288. loop {
  5289. if (*path=='/')
  5290. return (path[1]=='>');
  5291. if (!*path||(*path=='\\'))
  5292. return false;
  5293. path++;
  5294. }
  5295. }
  5296. return (path&&(*path=='/')&&(path[1]=='>'));
  5297. }
  5298. size32_t SendFile(ISocket *target, IFileIO *fileio,offset_t start,size32_t len)
  5299. {
  5300. assertex(target);
  5301. assertex(fileio);
  5302. offset_t fsz = fileio->size();
  5303. if (start>=fsz)
  5304. return 0;
  5305. if (start+len>fsz)
  5306. len = (size32_t)(fsz-start);
  5307. if (!len)
  5308. return 0;
  5309. CFileIO *cfile = QUERYINTERFACE(fileio,CFileIO);
  5310. if (cfile) {
  5311. HANDLE fh = cfile->queryHandle();
  5312. if ((fh!=(HANDLE)0)&&(fh!=(HANDLE)-1)) {
  5313. unsigned sh = target->OShandle();
  5314. if ((sh!=(unsigned)0)&&(sh!=(unsigned)-1)) {
  5315. #ifdef _WIN32
  5316. // MORE - should there be something here?
  5317. #elif defined(__linux__)
  5318. // TransmitFile not in std DLLs so don't bother with
  5319. off_t ofs = start;
  5320. ssize_t sent = ::sendfile(sh,fh,&ofs,len);
  5321. if (sent!=(ssize_t)-1)
  5322. return (size32_t)sent;
  5323. int err = errno;
  5324. if ((err!=EINVAL)&&(err!=ENOSYS))
  5325. throw MakeOsException(err,"sendfile");
  5326. #else
  5327. UNIMPLEMENTED;
  5328. #endif
  5329. }
  5330. }
  5331. }
  5332. #ifdef _DEBUG
  5333. #ifndef _WIN32
  5334. WARNLOG("SendFile falling back");
  5335. #endif
  5336. #endif
  5337. // fallback
  5338. MemoryAttr ma;
  5339. void *buf = ma.allocate(len);
  5340. size32_t rd = fileio->read(start,len,buf);
  5341. target->write(buf,rd);
  5342. return rd;
  5343. }
  5344. void asyncClose(IFileIO *io)
  5345. {
  5346. if (!io)
  5347. return;
  5348. static CriticalSection ADsect;
  5349. CriticalBlock block(ADsect);
  5350. static Owned<IWorkQueueThread> adwp = createWorkQueueThread();
  5351. class cWQI: public CInterface,implements IWorkQueueItem
  5352. {
  5353. Owned<IFileIO> io;
  5354. public:
  5355. IMPLEMENT_IINTERFACE;
  5356. cWQI(IFileIO *_io)
  5357. : io(_io)
  5358. {
  5359. }
  5360. void execute()
  5361. {
  5362. io.clear();
  5363. }
  5364. };
  5365. adwp->post(new cWQI(io));
  5366. };
  5367. int stdIoHandle(const char *path)
  5368. {
  5369. if (strcmp(path,"stdin:")==0)
  5370. return 0;
  5371. if (strcmp(path,"stdout:")==0)
  5372. return 1;
  5373. if (strcmp(path,"stderr:")==0)
  5374. return 2;
  5375. return -1;
  5376. }
  5377. extern jlib_decl bool containsFileWildcard(const char * path)
  5378. {
  5379. if (!path)
  5380. return false;
  5381. return strchr(path, '*') || strchr(path, '?');
  5382. }
  5383. extern jlib_decl bool isDirectory(const char * path)
  5384. {
  5385. Owned<IFile> file = createIFile(path);
  5386. return file->isDirectory() == foundYes;
  5387. }
  5388. // IFileIOCache
  5389. class CLazyFileIOCache;
  5390. class CCachedFileIO: public CInterface, implements IFileIO
  5391. {
  5392. CLazyFileIOCache &owner;
  5393. RemoteFilename filename;
  5394. CriticalSection &sect;
  5395. IFOmode mode;
  5396. void writeNotSupported(const char *s)
  5397. {
  5398. StringBuffer tmp;
  5399. filename.getRemotePath(tmp);
  5400. throw MakeStringException(-1, "CCachedFileIO(%s) %s not supported", tmp.str(), s);
  5401. }
  5402. public:
  5403. unsigned accesst;
  5404. Owned<IFileIO> cachedio;
  5405. CCachedFileIO(CLazyFileIOCache &_owner, CriticalSection &_sect, RemoteFilename &_filename, IFOmode _mode)
  5406. : owner(_owner), sect(_sect)
  5407. {
  5408. filename.set(_filename);
  5409. mode = _mode;
  5410. }
  5411. virtual void Link(void) const { CInterface::Link(); } \
  5412. virtual bool Release(void) const;
  5413. IFileIO *open();
  5414. size32_t read(offset_t pos, size32_t len, void * data)
  5415. {
  5416. CriticalBlock block(sect);
  5417. Owned<IFileIO> io = open();
  5418. return io->read(pos,len,data);
  5419. }
  5420. offset_t size()
  5421. {
  5422. CriticalBlock block(sect);
  5423. Owned<IFileIO> io = open();
  5424. return io->size();
  5425. }
  5426. size32_t write(offset_t pos, size32_t len, const void * data)
  5427. {
  5428. CriticalBlock block(sect);
  5429. Owned<IFileIO> io = open();
  5430. return io->write(pos,len,data);
  5431. }
  5432. offset_t appendFile(IFile *file,offset_t pos,offset_t len)
  5433. {
  5434. CriticalBlock block(sect);
  5435. Owned<IFileIO> io = open();
  5436. return io->appendFile(file,pos,len);
  5437. }
  5438. void setSize(offset_t size)
  5439. {
  5440. CriticalBlock block(sect);
  5441. Owned<IFileIO> io = open();
  5442. io->setSize(size);
  5443. }
  5444. };
  5445. class CLazyFileIOCache: public CInterface, implements IFileIOCache
  5446. {
  5447. CriticalSection sect;
  5448. unsigned max;
  5449. PointerIArrayOf<CCachedFileIO> cache;
  5450. public:
  5451. IMPLEMENT_IINTERFACE;
  5452. CLazyFileIOCache(unsigned _max)
  5453. {
  5454. max = _max;
  5455. }
  5456. CCachedFileIO *addFile( RemoteFilename &filename, IFOmode mode )
  5457. {
  5458. // check exists
  5459. unsigned i = cache.ordinality();
  5460. ForEachItemIn(i2,cache) {
  5461. if (!cache.item(i2)) {
  5462. i = i2;
  5463. break;
  5464. }
  5465. }
  5466. Linked<CCachedFileIO> ret = new CCachedFileIO(*this,sect,filename,mode);
  5467. if (i<cache.ordinality())
  5468. cache.replace(ret,i);
  5469. else
  5470. cache.append(ret);
  5471. return ret.getClear();
  5472. }
  5473. void checkCache(unsigned wanted=1)
  5474. {
  5475. // called in sect
  5476. loop {
  5477. CCachedFileIO *oldest = NULL;
  5478. unsigned oldestt = 0;
  5479. unsigned t = msTick();
  5480. unsigned n = wanted;
  5481. ForEachItemIn(i1,cache) {
  5482. CCachedFileIO *cio = cache.item(i1);
  5483. if (cio&&cio->cachedio.get()) {
  5484. if (t-cio->accesst>=oldestt) {
  5485. oldest = cio;
  5486. oldestt = t-cio->accesst;
  5487. }
  5488. n++;
  5489. }
  5490. }
  5491. if (n<max)
  5492. break;
  5493. if (!oldest)
  5494. break;
  5495. oldest->cachedio.clear();
  5496. }
  5497. }
  5498. const CCachedFileIO *removeFile(const CCachedFileIO *io)
  5499. {
  5500. CriticalBlock block(sect);
  5501. ForEachItemIn(oi,cache) {
  5502. if (io = cache.item(oi)) {
  5503. cache.replace(NULL,oi,true);
  5504. break;
  5505. }
  5506. }
  5507. return io;
  5508. }
  5509. };
  5510. bool CCachedFileIO::Release(void) const
  5511. {
  5512. if (CInterface::Release())
  5513. return true;
  5514. if (!CInterface::IsShared())
  5515. return owner.removeFile(this)->Release();
  5516. return false;
  5517. }
  5518. IFileIO *CCachedFileIO::open()
  5519. {
  5520. // called in sect
  5521. if (!cachedio) {
  5522. owner.checkCache();
  5523. Owned<IFile> inFile = createIFile(filename);
  5524. cachedio.setown(inFile->open(mode));
  5525. if (!cachedio.get()) {
  5526. StringBuffer tmp;
  5527. filename.getRemotePath(tmp);
  5528. throw MakeStringException(-1, "CCachedFileIO::open(%d) '%s' failed", (int)mode, tmp.str());
  5529. }
  5530. }
  5531. accesst = msTick();
  5532. return cachedio.getLink();
  5533. }
  5534. IFileIOCache* createFileIOCache(unsigned max)
  5535. {
  5536. return new CLazyFileIOCache(max);
  5537. }
  5538. extern jlib_decl IFile * createSentinelTarget()
  5539. {
  5540. const char * sentinelFilename = getenv("SENTINEL");
  5541. if (sentinelFilename)
  5542. return createIFile(sentinelFilename);
  5543. else
  5544. return NULL;
  5545. }
  5546. extern jlib_decl void removeSentinelFile(IFile * sentinelFile)
  5547. {
  5548. if (sentinelFile)
  5549. {
  5550. if(sentinelFile->exists() && !sentinelFile->isDirectory())
  5551. {
  5552. DBGLOG("Removing sentinel file %s", sentinelFile->queryFilename());
  5553. try
  5554. {
  5555. sentinelFile->remove();
  5556. }
  5557. catch(IException *E)
  5558. {
  5559. StringBuffer s;
  5560. EXCLOG(E, s.appendf("Failed to remove sentinel file %s", sentinelFile->queryFilename()).str());
  5561. E->Release();
  5562. throw MakeOsException(errno, "removeSentinelFile - file cannot be removed.");
  5563. }
  5564. }
  5565. }
  5566. }
  5567. extern jlib_decl void writeSentinelFile(IFile * sentinelFile)
  5568. {
  5569. if ( sentinelFile )
  5570. {
  5571. DBGLOG("Creating sentinel file %s for rerun from script", sentinelFile->queryFilename());
  5572. try
  5573. {
  5574. Owned<IFileIO> sentinel = sentinelFile->open(IFOcreate);
  5575. sentinel->write(0, 5, "rerun");
  5576. }
  5577. catch(IException *E)
  5578. {
  5579. StringBuffer s;
  5580. EXCLOG(E, s.appendf("Failed to create sentinel file %s for rerun from script", sentinelFile->queryFilename()).str());
  5581. E->Release();
  5582. throw MakeOsException(errno, "writeSentinelFile - file not created.");
  5583. }
  5584. }
  5585. }
  5586. jlib_decl StringBuffer & appendCurrentDirectory(StringBuffer & target, bool blankIfFails)
  5587. {
  5588. char temp[_MAX_PATH+1];
  5589. if (!getcwd(temp,sizeof(temp)))
  5590. {
  5591. if (blankIfFails)
  5592. return target;
  5593. throw MakeStringException(JLIBERR_InternalError, "getcwd failed (%d)", errno);
  5594. }
  5595. return target.append(temp);
  5596. }