1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853685468556856685768586859686068616862686368646865686668676868686968706871687268736874687568766877687868796880688168826883688468856886688768886889689068916892689368946895689668976898689969006901690269036904690569066907690869096910691169126913691469156916691769186919692069216922692369246925692669276928692969306931693269336934693569366937693869396940694169426943694469456946694769486949695069516952695369546955695669576958695969606961696269636964696569666967696869696970697169726973697469756976697769786979698069816982698369846985698669876988698969906991699269936994699569966997699869997000700170027003700470057006700770087009701070117012701370147015701670177018701970207021702270237024702570267027702870297030703170327033703470357036703770387039704070417042704370447045704670477048704970507051705270537054705570567057705870597060706170627063706470657066706770687069707070717072707370747075707670777078707970807081708270837084708570867087708870897090709170927093709470957096709770987099710071017102710371047105710671077108710971107111711271137114711571167117711871197120712171227123712471257126712771287129713071317132713371347135713671377138713971407141714271437144714571467147714871497150715171527153715471557156715771587159716071617162716371647165716671677168716971707171717271737174717571767177717871797180718171827183718471857186718771887189719071917192719371947195719671977198719972007201720272037204720572067207720872097210721172127213721472157216721772187219722072217222722372247225722672277228722972307231723272337234723572367237723872397240724172427243724472457246724772487249725072517252725372547255725672577258725972607261726272637264726572667267726872697270727172727273727472757276727772787279728072817282728372847285728672877288728972907291729272937294729572967297729872997300730173027303730473057306730773087309731073117312731373147315731673177318731973207321732273237324732573267327732873297330733173327333733473357336733773387339734073417342734373447345734673477348734973507351735273537354735573567357735873597360736173627363736473657366736773687369737073717372737373747375737673777378737973807381738273837384738573867387738873897390739173927393739473957396739773987399740074017402740374047405740674077408740974107411741274137414741574167417741874197420742174227423742474257426742774287429743074317432743374347435743674377438743974407441744274437444744574467447744874497450745174527453745474557456745774587459746074617462746374647465746674677468746974707471747274737474747574767477747874797480748174827483748474857486748774887489749074917492749374947495749674977498749975007501750275037504750575067507750875097510751175127513751475157516751775187519752075217522752375247525752675277528752975307531753275337534753575367537753875397540754175427543754475457546754775487549755075517552755375547555755675577558755975607561756275637564756575667567756875697570757175727573757475757576757775787579758075817582758375847585758675877588758975907591759275937594759575967597759875997600760176027603760476057606760776087609761076117612761376147615761676177618761976207621762276237624762576267627762876297630763176327633763476357636763776387639764076417642764376447645764676477648764976507651765276537654765576567657765876597660766176627663766476657666766776687669767076717672767376747675767676777678767976807681768276837684768576867687768876897690769176927693769476957696769776987699770077017702770377047705770677077708770977107711771277137714771577167717771877197720772177227723772477257726772777287729773077317732773377347735773677377738773977407741774277437744774577467747774877497750775177527753775477557756775777587759776077617762776377647765776677677768776977707771777277737774777577767777777877797780778177827783778477857786778777887789779077917792779377947795779677977798779978007801780278037804780578067807780878097810781178127813781478157816781778187819782078217822782378247825782678277828782978307831783278337834783578367837783878397840784178427843784478457846784778487849785078517852785378547855785678577858785978607861786278637864786578667867786878697870787178727873787478757876787778787879788078817882788378847885788678877888788978907891789278937894789578967897789878997900790179027903790479057906790779087909791079117912791379147915791679177918791979207921792279237924792579267927792879297930793179327933793479357936793779387939794079417942794379447945794679477948794979507951795279537954795579567957795879597960796179627963796479657966796779687969797079717972797379747975797679777978797979807981798279837984798579867987798879897990799179927993799479957996799779987999800080018002800380048005800680078008800980108011801280138014801580168017801880198020802180228023802480258026802780288029803080318032803380348035803680378038803980408041804280438044804580468047804880498050805180528053805480558056805780588059806080618062806380648065806680678068806980708071807280738074807580768077807880798080808180828083808480858086808780888089809080918092809380948095809680978098809981008101810281038104810581068107810881098110811181128113811481158116811781188119812081218122812381248125812681278128812981308131813281338134813581368137813881398140814181428143814481458146814781488149815081518152815381548155815681578158815981608161816281638164816581668167816881698170817181728173817481758176817781788179818081818182818381848185818681878188818981908191819281938194819581968197819881998200820182028203820482058206820782088209821082118212821382148215821682178218821982208221822282238224822582268227822882298230823182328233823482358236823782388239824082418242824382448245824682478248824982508251825282538254825582568257825882598260826182628263826482658266826782688269827082718272827382748275827682778278827982808281828282838284828582868287828882898290829182928293829482958296829782988299830083018302830383048305830683078308830983108311831283138314831583168317831883198320832183228323832483258326832783288329833083318332833383348335833683378338833983408341834283438344834583468347834883498350835183528353835483558356835783588359836083618362836383648365836683678368836983708371837283738374837583768377837883798380838183828383838483858386838783888389839083918392839383948395839683978398839984008401840284038404840584068407840884098410841184128413841484158416841784188419842084218422842384248425842684278428842984308431843284338434843584368437843884398440844184428443844484458446844784488449845084518452845384548455845684578458845984608461846284638464846584668467846884698470847184728473847484758476847784788479848084818482848384848485848684878488848984908491849284938494849584968497849884998500850185028503850485058506850785088509851085118512851385148515851685178518851985208521852285238524852585268527852885298530853185328533853485358536853785388539854085418542854385448545854685478548854985508551855285538554855585568557855885598560856185628563856485658566856785688569857085718572857385748575857685778578857985808581858285838584858585868587858885898590859185928593859485958596859785988599860086018602860386048605860686078608860986108611861286138614861586168617861886198620862186228623862486258626862786288629863086318632863386348635863686378638863986408641864286438644864586468647864886498650865186528653865486558656865786588659866086618662866386648665866686678668866986708671867286738674867586768677867886798680868186828683868486858686868786888689869086918692869386948695869686978698869987008701870287038704870587068707870887098710871187128713871487158716871787188719872087218722872387248725872687278728872987308731873287338734873587368737873887398740874187428743874487458746874787488749875087518752875387548755875687578758875987608761876287638764876587668767876887698770877187728773877487758776877787788779878087818782878387848785878687878788878987908791879287938794879587968797879887998800880188028803880488058806880788088809881088118812881388148815881688178818881988208821882288238824882588268827882888298830883188328833883488358836883788388839884088418842884388448845884688478848884988508851885288538854885588568857885888598860886188628863886488658866886788688869887088718872887388748875887688778878887988808881888288838884888588868887888888898890889188928893889488958896889788988899890089018902890389048905890689078908890989108911891289138914891589168917891889198920892189228923892489258926892789288929893089318932893389348935893689378938893989408941894289438944894589468947894889498950895189528953895489558956895789588959896089618962896389648965896689678968896989708971897289738974897589768977897889798980898189828983898489858986898789888989899089918992899389948995899689978998899990009001900290039004900590069007900890099010901190129013901490159016901790189019902090219022902390249025902690279028902990309031903290339034903590369037903890399040904190429043904490459046904790489049905090519052905390549055905690579058905990609061906290639064906590669067906890699070907190729073907490759076907790789079908090819082908390849085908690879088908990909091909290939094909590969097909890999100910191029103910491059106910791089109911091119112911391149115911691179118911991209121912291239124912591269127912891299130913191329133913491359136913791389139914091419142914391449145914691479148914991509151915291539154915591569157915891599160916191629163916491659166916791689169917091719172917391749175917691779178917991809181918291839184918591869187918891899190919191929193919491959196919791989199920092019202920392049205920692079208920992109211921292139214921592169217921892199220922192229223922492259226922792289229923092319232923392349235923692379238923992409241924292439244924592469247924892499250925192529253925492559256925792589259926092619262 |
- /*##############################################################################
- HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- ############################################################################## */
- #include <unordered_map>
- #include <unordered_set>
- #include <string>
- #include "platform.h"
- #include "jarray.hpp"
- #include "jdebug.hpp"
- #include "jhash.hpp"
- #include "jmutex.hpp"
- #include "jexcept.hpp"
- #include "jlzw.hpp"
- #include "jregexp.hpp"
- #include "jstring.hpp"
- #include "jutil.hpp"
- #include "jmisc.hpp"
- #include "yaml.h"
- #include <initializer_list>
- #define MAKE_LSTRING(name,src,length) \
- const char *name = (const char *) alloca((length)+1); \
- memcpy((char *) name, (src), (length)); \
- *(char *) (name+(length)) = '\0';
- #include "jfile.hpp"
- #include "jlog.hpp"
- #include "jptree.ipp"
- #define WARNLEGACYCOMPARE
- #define XMLTAG_CONTENT "<>"
- #undef UNIMPLEMENTED
- #define UNIMPLEMENTED throw MakeIPTException(-1, "UNIMPLEMENTED")
- #define CHECK_ATTRIBUTE(X) if (X && isAttribute(X)) throw MakeIPTException(PTreeExcpt_XPath_Unsupported, "Attribute usage invalid here");
- #define AMBIGUOUS_PATH(X,P) { StringBuffer buf; buf.append(X": ambiguous xpath \"").append(P).append("\""); throw MakeIPTException(PTreeExcpt_XPath_Ambiguity,"%s",buf.str()); }
- #define PTREE_COMPRESS_THRESHOLD (4*1024) // i.e. only use compress if > threshold
- #define PTREE_COMPRESS_BOTHER_PECENTAGE (80) // i.e. if it doesn't compress to <80 % of original size don't bother
- class NullPTreeIterator final : implements IPropertyTreeIterator
- {
- public:
- virtual ~NullPTreeIterator() {}
- virtual void Link() const override {}
- virtual bool Release() const override { return true; }
- // IPropertyTreeIterator
- virtual bool first() override { return false; }
- virtual bool next() override { return false; }
- virtual bool isValid() override { return false; }
- virtual IPropertyTree & query() override { throwUnexpected(); }
- } *nullPTreeIterator;
- IPropertyTreeIterator *createNullPTreeIterator() { return LINK(nullPTreeIterator); } // initialized in init mod below.
- //===================================================================
- #ifdef USE_READONLY_ATOMTABLE
- RONameTable *AttrStrUnionWithTable::roNameTable = nullptr;
- RONameTable *AttrStrUnionWithValueTable::roValueTable = nullptr;
- #endif
- static AtomRefTable *keyTable = nullptr;
- static AtomRefTable *keyTableNC = nullptr;
- static CriticalSection hashcrit;
- static CAttrValHashTable *attrHT = nullptr;
- static AttrValue **freelist = nullptr;
- static unsigned freelistmax = 0;
- static CLargeMemoryAllocator freeallocator((memsize_t)-1, 0x1000*sizeof(AttrValue), true);
- #ifdef USE_READONLY_ATOMTABLE
- static const char * roAttributes[] =
- {
- #include "jptree-attrs.hpp" // potentially auto-generated
- nullptr
- };
- static const char * roAttributeValues[] =
- {
- #include "jptree-attrvalues.hpp" // potentially auto-generated
- nullptr
- };
- void initializeRoTable()
- {
- for (const char **attr = roAttributes; *attr; attr++)
- {
- AttrStrUnionWithTable::roNameTable->find(*attr, true);
- }
- for (const char **value = roAttributeValues; *value; value++)
- {
- AttrStrUnionWithValueTable::roValueTable->find(*value, true);
- }
- // also populate read-only value table by generating some common constants
- StringBuffer constStr;
- for (unsigned c=0; c<1000; c++) // common unsigned values in attributes
- {
- constStr.clear().append(c);
- AttrStrUnionWithValueTable::roValueTable->find(constStr.str(), true);
- }
- for (unsigned c=1; c<=400; c++) // outer graphs
- {
- constStr.clear().append("graph").append(c);
- AttrStrUnionWithValueTable::roValueTable->find(constStr.str(), true);
- constStr.clear().append("Graph graph ").append(c);
- AttrStrUnionWithValueTable::roValueTable->find(constStr.str(), true);
- }
- for (unsigned c=1; c<=200; c++) // subgraphs
- {
- constStr.clear().append("sg").append(c);
- AttrStrUnionWithValueTable::roValueTable->find(constStr.str(), true);
- }
- for (unsigned c=1; c<=200; c++) // Edge 0
- {
- constStr.clear().append(c).append("_0");
- AttrStrUnionWithValueTable::roValueTable->find(constStr.str(), true);
- }
- for (unsigned c=0; c<35; c++)
- {
- char ch = c<9 ? ('1' + c) : ('A' + (c-9));
- constStr.clear().append("~spill::").append(ch); // spills
- AttrStrUnionWithValueTable::roValueTable->find(constStr.str(), true);
- constStr.clear().append("gl").append(ch); // graph results
- AttrStrUnionWithValueTable::roValueTable->find(constStr.str(), true);
- constStr.clear().append("mf").append(ch); // meta factories
- AttrStrUnionWithValueTable::roValueTable->find(constStr.str(), true);
- }
- for (unsigned c=1; c<=10; c++) // global auto attributes
- {
- constStr.clear().append("auto").append(c);
- AttrStrUnionWithValueTable::roValueTable->find(constStr.str(), true);
- }
- #ifdef TRACE_ATOM_SIZE
- // If you are wanting an idea of the savings from use of the RO hash table, it may be useful to reset
- // the counts here. But it's more correct to actually leave them in place.
- //AttrStrAtom::totsize = 0;
- //AttrStrAtom::maxsize = 0;
- #endif
- #ifdef _DEBUG
- for (const char **a = roAttributes; *a; a++)
- {
- // sanity check
- unsigned idx = AttrStrUnionWithTable::roNameTable->findIndex(*a, AttrStrC::getHash(*a));
- AttrStrC *val = AttrStrUnionWithTable::roNameTable->getIndex(idx);
- assert(val && val->eq(*a));
- }
- for (const char **v = roAttributeValues; *v; v++)
- {
- // sanity check
- unsigned idx = AttrStrUnionWithValueTable::roValueTable->findIndex(*v, AttrStrC::getHash(*v));
- AttrStrC *val = AttrStrUnionWithValueTable::roValueTable->getIndex(idx);
- assert(val && val->eq(*v));
- }
- #endif
- }
- #endif
- MODULE_INIT(INIT_PRIORITY_JPTREE)
- {
- nullPTreeIterator = new NullPTreeIterator;
- #ifdef USE_READONLY_ATOMTABLE
- AttrStrUnionWithTable::roNameTable = new RONameTable(255);
- AttrStrUnionWithValueTable::roValueTable = new RONameTable(4095);
- initializeRoTable();
- #endif
- keyTable = new AtomRefTable;
- keyTableNC = new AtomRefTable(true);
- attrHT = new CAttrValHashTable;
- return true;
- }
- MODULE_EXIT()
- {
- delete nullPTreeIterator;
- delete attrHT;
- keyTable->Release();
- keyTableNC->Release();
- #ifdef USE_READONLY_ATOMTABLE
- delete AttrStrUnionWithTable::roNameTable;
- delete AttrStrUnionWithValueTable::roValueTable;
- #endif
- free(freelist);
- freelist = NULL;
- }
- static int comparePropTrees(IInterface * const *ll, IInterface * const *rr)
- {
- IPropertyTree *l = (IPropertyTree *) *ll;
- IPropertyTree *r = (IPropertyTree *) *rr;
- return stricmp(l->queryName(), r->queryName());
- };
- class CPTArrayIterator : public ArrayIIteratorOf<IArrayOf<IPropertyTree>, IPropertyTree, IPropertyTreeIterator>
- {
- IArrayOf<IPropertyTree> elems;
- public:
- CPTArrayIterator(IPropertyTreeIterator &iter, TreeCompareFunc compare) : ArrayIIteratorOf<IArrayOf<IPropertyTree>, IPropertyTree, IPropertyTreeIterator>(elems)
- {
- ForEach(iter)
- elems.append(iter.get());
- elems.sort(compare);
- }
- CPTArrayIterator(IArrayOf<IPropertyTree> & ownedElems, TreeCompareFunc compare) : ArrayIIteratorOf<IArrayOf<IPropertyTree>, IPropertyTree, IPropertyTreeIterator>(elems)
- {
- elems.swapWith(ownedElems);
- elems.sort(compare);
- }
- };
- IPropertyTreeIterator * createSortedIterator(IPropertyTreeIterator & iter)
- {
- return new CPTArrayIterator(iter, comparePropTrees);
- }
- IPropertyTreeIterator * createSortedIterator(IPropertyTreeIterator & iter, TreeCompareFunc compare)
- {
- return new CPTArrayIterator(iter, compare);
- }
- IPropertyTreeIterator * createSortedIterator(IArrayOf<IPropertyTree> & ownedElems, TreeCompareFunc compare)
- {
- return new CPTArrayIterator(ownedElems, compare);
- }
- //////////////////
- unsigned ChildMap::getHashFromElement(const void *e) const
- {
- PTree &elem = (PTree &) (*(IPropertyTree *)e);
- return elem.queryHash();
- }
- unsigned ChildMap::numChildren() const
- {
- SuperHashIteratorOf<IPropertyTree> iter(*this);
- if (!iter.first()) return 0;
- unsigned count = 0;
- do
- {
- PTree *element = (PTree *) &iter.query();
- if (element->value && element->value->isArray())
- count += element->value->elements();
- else
- ++count;
- }
- while (iter.next());
- return count;
- }
- IPropertyTreeIterator *ChildMap::getIterator(bool sort)
- {
- class CPTHashIterator : implements IPropertyTreeIterator, public CInterface
- {
- SuperHashIteratorOf<IPropertyTree> *hiter;
- public:
- IMPLEMENT_IINTERFACE;
- CPTHashIterator(SuperHashTable &table) { hiter = new SuperHashIteratorOf<IPropertyTree>(table); }
- ~CPTHashIterator() { hiter->Release(); }
- // IPropertyTreeIterator
- virtual bool first() override { return hiter->first(); }
- virtual bool next() override { return hiter->next(); }
- virtual bool isValid() override { return hiter->isValid(); }
- virtual IPropertyTree & query() override { return hiter->query(); }
- };
- Owned<IPropertyTreeIterator> baseIter = new CPTHashIterator(*this);
- if (!sort)
- return baseIter.getClear();
- return createSortedIterator(*baseIter);
- }
- void ChildMap::onRemove(void *e)
- {
- PTree &elem = static_cast<PTree &>(*(IPropertyTree *)e);
- IPTArrayValue *value = elem.queryValue();
- if (value && value->isArray())
- {
- IPropertyTree **elems = value->getRawArray();
- IPropertyTree **elemsEnd = elems+value->elements();
- while (elems != elemsEnd)
- {
- PTree *pElem = static_cast<PTree *>(*elems++);
- pElem->setOwner(nullptr);
- }
- }
- elem.Release();
- }
- ///////////
- bool validateXMLTag(const char *name)
- {
- if (!isValidXPathStartChr(*name)) return false;
- ++name;
- while (*name != '\0')
- {
- if (!isValidXPathChr(*name)) return false;
- ++name;
- }
- return true;
- }
- class jlib_thrown_decl CPTreeException : implements IPTreeException, public CInterface
- {
- int errCode;
- StringBuffer errMsg;
- public:
- IMPLEMENT_IINTERFACE;
- CPTreeException(int _errCode, const char *_errMsg, va_list &args) __attribute__((format(printf,3,0))) : errCode(_errCode)
- {
- if (_errMsg)
- errMsg.valist_appendf(_errMsg, args);
- }
- StringBuffer &translateCode(StringBuffer &out) const
- {
- out.append("IPropertyTree: ");
- switch (errCode)
- {
- case PTreeExcpt_XPath_Ambiguity:
- return out.append("Ambiguous xpath used");
- case PTreeExcpt_XPath_ParseError:
- return out.append("xpath parse error");
- case PTreeExcpt_XPath_Unsupported:
- return out.append("unsupported xpath syntax used");
- case PTreeExcpt_InvalidTagName:
- return out.append("Invalid tag name");
- default:
- return out.append("UNKNOWN ERROR CODE: ").append(errCode);
- }
- }
- // IException
- int errorCode() const { return errCode; }
- StringBuffer &errorMessage(StringBuffer &out) const
- {
- return translateCode(out).append("\n").append(errMsg.str());
- }
- MessageAudience errorAudience() const { return MSGAUD_user; }
- };
- static IPTreeException *MakeIPTException(int code, const char *format, ...) __attribute__((format(printf,2,3)));
- static IPTreeException *MakeXPathException(const char *xpath, int code, size_t pos, const char *format, ...) __attribute__((format(printf,4,5)));
- IPTreeException *MakeIPTException(int code, const char *format, ...)
- {
- va_list args;
- va_start(args, format);
- IPTreeException *e = new CPTreeException(code, format, args);
- va_end(args);
- return e;
- }
- IPTreeException *MakeXPathException(const char *xpath, int code, size_t pos, const char *format, ...)
- {
- va_list args;
- va_start(args, format);
- StringBuffer s("XPath Exception: ");
- s.valist_appendf(format, args);
- va_end(args);
- #ifdef _DEBUG
- PrintStackReport();
- #endif
- const char *msg = "in xpath = ";
- s.append("\n").append(msg).append(xpath);
- s.append("\n").appendN((size32_t)(strlen(msg)+pos), ' ').append("^");
- return MakeIPTException(code, "%s", s.str());
- }
- inline static void readID(const char *&xxpath, bool started)
- {
- const char *xpath = xxpath;
- if (isValidXPathStartChr(*xpath) || (started && isValidXPathChr(*xpath)))
- {
- do
- {
- xpath++;
- } while (isValidXPathChr(*xpath));
- xxpath = xpath;
- }
- }
- inline static void readWildId(const char *&xpath, bool &wild)
- {
- wild = false;
- for (;;)
- {
- readID(xpath, wild);
- if ('*' != *xpath)
- break;
- wild = true;
- ++xpath;
- }
- }
- inline const char * readIndex(const char *xpath, StringAttr &index)
- {
- const char *start = xpath;
- do { xpath++; } while (isdigit(*xpath));
- index.set(start, (xpath - start));
- return xpath;
- }
- inline static void readWildIdIndex(const char *&xpath, bool &wild, bool &numeric)
- {
- const char *_xpath = xpath;
- readWildId(xpath, wild);
- if ('[' == *xpath) // check for local index not iterative qualifier.
- {
- const char *end = xpath+1;
- if (isdigit(*end))
- {
- StringAttr index;
- end = readIndex(end, index);
- if (']' != *end)
- throw MakeXPathException(_xpath, PTreeExcpt_XPath_ParseError, xpath-_xpath, "Qualifier brace unclosed");
- xpath = end+1;
- numeric = true;
- }
- else
- numeric = false;
- }
- else
- numeric = false;
- }
- inline static unsigned getTailIdLength(const char *xxpath, unsigned xxpathlength)
- {
- const char *xpath = xxpath+xxpathlength;
- const char *end = xpath;
- while (xpath != xxpath)
- {
- --xpath;
- if (!isValidXPathChr(*xpath)) break;
- }
- if (!isAttribute(xpath) && xpath != xxpath) ++xpath;
- return end-xpath;
- }
- const char *splitXPathUQ(const char *xpath, StringBuffer &path)
- {
- size32_t xpathSize = (size32_t) strlen(xpath);
- size32_t idSize = getTailIdLength(xpath, xpathSize);
- path.append(xpathSize-idSize, xpath);
- return xpath + (xpathSize-idSize);
- }
- const char *splitXPathX(const char *xpath)
- {
- size32_t xpathSize = (size32_t) strlen(xpath);
- size32_t idSize = getTailIdLength(xpath, xpathSize);
- return xpath + (xpathSize-idSize);
- }
- // similar to above, splitXPathUQ doesn't split if qualified
- const char *splitXPath(const char *xpath, StringBuffer &headPath)
- {
- StringBuffer path;
- const char *end = xpath+strlen(xpath);
- const char *prop = end;
- bool quote = false;
- bool braced = false;
- while (xpath != prop)
- {
- --prop;
- if (*prop == '"')
- {
- if (quote) quote = false;
- else quote = true;
- }
- else if (*prop == ']' && !quote)
- {
- assertex(!braced);
- braced = true;
- }
- else if (*prop == '[' && !quote)
- {
- assertex(braced);
- braced = false;
- }
- else if (*prop == '/' && !quote && !braced)
- {
- ++prop;
- break;
- }
- }
- if (prop == end)
- return NULL;
- else if (xpath != prop)
- {
- size32_t ps = prop-xpath-1;
- headPath.append(ps, xpath);
- }
- return prop;
- }
- const char *queryNextUnquoted(const char *str, char c)
- {
- bool quote = false;
- for (;;)
- {
- char next = *str;
- if (next == '\0')
- return NULL;
- if ('"' == next)
- quote = !quote;
- else if (c == next && !quote)
- return str;
- ++str;
- }
- }
- const char *queryHead(const char *xpath, StringBuffer &head)
- {
- if (!xpath) return NULL;
- const char *start = xpath;
- bool quote = false;
- bool braced = false;
- for (;;)
- {
- if (*xpath == '\0')
- return NULL;
- ++xpath;
- char next = *xpath;
- if ('"' == next)
- quote = !quote;
- else if (next == ']' && !quote)
- {
- assertex(braced);
- braced = false;
- }
- else if (next == '[' && !quote)
- {
- assertex(!braced);
- braced = true;
- }
- else if (next == '/' && !quote && !braced)
- {
- if ('/' == *start) // so leading '//'
- return start;
- else if ('/' == *(xpath+1)) // in middle of path
- {
- head.append(xpath-start, start);
- return xpath;
- }
- break;
- }
- }
- head.append(xpath-start, start);
- return xpath+1;
- }
- ///////////////////
- static constexpr unsigned defaultSiblingMapThreshold = 100;
- static unsigned siblingMapThreshold = (unsigned)-1; // off until configuration default it on.
- void setPTreeMappingThreshold(unsigned threshold)
- {
- /*
- * NB: setPTreeMappingThreshold() will automatically be called via loadConfiguration
- * Redefining this limit, will not effect existing maps, and should generally only be called once during startup.
- */
- if (0 == threshold)
- threshold = (unsigned)-1;
- siblingMapThreshold = threshold;
- }
- class CValueMap : public std::unordered_multimap<std::string, const IPropertyTree *>
- {
- public:
- CValueMap(const char *_lhs, IPTArrayValue &array)
- {
- IPropertyTree **elements = array.getRawArray();
- IPropertyTree **last = elements+array.elements();
- dbgassertex(elements != last);
- while (true)
- {
- const char *v = (*elements)->queryProp(_lhs);
- if (v)
- emplace(std::make_pair(std::string(v), *elements));
- elements++;
- if (last == elements)
- break;
- }
- }
- std::pair<CValueMap::iterator, CValueMap::iterator> find(const char *rhs)
- {
- return equal_range(std::string(rhs));
- }
- void insertEntry(const char *v, const IPropertyTree *tree)
- {
- emplace(std::make_pair(std::string(v), tree));
- }
- bool removeEntry(const char *v, const IPropertyTree *tree)
- {
- auto range = equal_range(std::string(v));
- if (range.first == range.second)
- return false;
- auto it = range.first;
- while (true)
- {
- if (it->second == tree)
- {
- it = erase(it);
- return true;
- }
- ++it;
- if (it == range.second)
- break;
- }
- throwUnexpected();
- }
- void replaceEntry(const char *oldV, const char *newV, const IPropertyTree *tree)
- {
- verifyex(removeEntry(oldV, tree));
- if (newV)
- insertEntry(newV, tree);
- }
- };
- class CQualifierMap
- {
- std::unordered_map<std::string, CValueMap *> attrValueMaps;
- CriticalSection crit;
- public:
- CQualifierMap()
- {
- }
- ~CQualifierMap()
- {
- for (auto &e: attrValueMaps)
- delete e.second;
- }
- CValueMap *addMapping(const char *lhs, IPTArrayValue &array)
- {
- CValueMap *valueMap = new CValueMap(lhs, array);
- attrValueMaps.emplace(std::make_pair(std::string(lhs), valueMap));
- return valueMap;
- }
- CValueMap *addMappingIfNew(const char *lhs, IPTArrayValue &array)
- {
- CriticalBlock b(crit);
- auto it = attrValueMaps.find(lhs);
- if (it == attrValueMaps.end())
- return addMapping(lhs, array);
- else
- return it->second;
- }
- void addMatchingValues(const IPropertyTree *tree)
- {
- for (auto &e: attrValueMaps)
- {
- const char *v = tree->queryProp(e.first.c_str());
- if (v)
- e.second->insertEntry(v, tree);
- }
- }
- void removeMatchingValues(const IPropertyTree *tree)
- {
- for (auto &e: attrValueMaps)
- {
- const char *lhsp = e.first.c_str();
- const char *oldV = tree->queryProp(lhsp);
- if (oldV)
- verifyex(e.second->removeEntry(oldV, tree));
- }
- }
- void replaceMatchingValues(const IPropertyTree *oldTree, const IPropertyTree *newTree)
- {
- for (auto &e: attrValueMaps)
- {
- const char *lhsp = e.first.c_str();
- const char *oldV = oldTree->queryProp(lhsp);
- if (oldV)
- {
- verifyex(e.second->removeEntry(oldV, oldTree));
- const char *newV = newTree->queryProp(lhsp);
- if (newV)
- e.second->insertEntry(newV, newTree);
- }
- }
- }
- CValueMap *find(const char *lhs)
- {
- auto it = attrValueMaps.find(lhs);
- if (it == attrValueMaps.end())
- return nullptr;
- return it->second;
- }
- void removeEntryIfMapped(const char *lhs, const char *v, const IPropertyTree *tree)
- {
- auto it = attrValueMaps.find(lhs);
- if (it != attrValueMaps.end())
- it->second->removeEntry(v, tree);
- }
- void insertEntryIfMapped(const char *lhs, const char *v, const IPropertyTree *tree)
- {
- auto it = attrValueMaps.find(lhs);
- if (it != attrValueMaps.end())
- it->second->insertEntry(v, tree);
- }
- void replaceEntryIfMapped(const char *lhs, const char *oldv, const char *newv, const IPropertyTree *tree)
- {
- auto it = attrValueMaps.find(lhs);
- if (it != attrValueMaps.end())
- it->second->replaceEntry(oldv, newv, tree);
- }
- };
- // parse qualifier, returns true if simple equality expression found
- static bool parseEqualityQualifier(const char *&xxpath, unsigned &lhsLen, const char *&rhsBegin, unsigned &rhsLen)
- {
- const char *xpath = xxpath;
- while (*xpath == ' ' || *xpath == '\t') xpath++;
- if ('@' != *xpath) // only attributes supported
- return false;
- const char *start = xpath;
- char quote = 0;
- const char *lhsEnd, *quoteBegin, *quoteEnd, *rhsEnd;
- lhsEnd = quoteBegin = quoteEnd = rhsBegin = rhsEnd = NULL;
- bool equalSignFound = false;
- for (;;)
- {
- switch (*xpath)
- {
- case '"':
- case '\'':
- if (quote)
- {
- if (*xpath == quote)
- {
- quote = 0;
- quoteEnd = xpath;
- }
- }
- else
- {
- if (quoteBegin)
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Quoted left hand side already seen");
- quote = *xpath;
- quoteBegin = xpath+1;
- }
- break;
- case '[':
- if (!quote)
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Unclosed qualifier detected");
- break;
- case ']':
- if (!quote)
- {
- if (!lhsEnd)
- lhsEnd = xpath;
- rhsEnd = xpath;
- }
- break;
- case ' ':
- case '\t':
- if (!lhsEnd)
- lhsEnd = xpath;
- break;
- case '!':
- case '>':
- case '<':
- case '~':
- case '/':
- if (!quote)
- return false;
- break;
- case '=':
- if (!quote)
- {
- if (equalSignFound)
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Unexpected expression operator xpath");
- equalSignFound = true;
- if (!lhsEnd)
- lhsEnd = xpath;
- }
- break;
- case '?':
- case '*':
- return false;
- case '\0':
- rhsEnd = xpath;
- break;
- }
- if (rhsEnd)
- break;
- xpath++;
- if (!rhsBegin && equalSignFound && !isspace(*xpath))
- rhsBegin = xpath;
- }
- if (quote)
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Parse error, unclosed quoted content");
- if (!equalSignFound)
- return false;
- lhsLen = lhsEnd-start;
- if (quoteBegin && !quoteEnd)
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Parse error, RHS missing closing quote");
- if (rhsBegin && !rhsEnd)
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Parse error, RHS missing closing quote");
- if (!quoteBegin && rhsEnd) // only if numeric
- return false;
- else // quoted
- {
- rhsBegin = quoteBegin;
- rhsLen = quoteEnd - rhsBegin;
- }
- if (rhsEnd && *xpath == ']')
- xpath++;
- xxpath = xpath;
- return true;
- }
- class CMapQualifierIterator : public CInterfaceOf<IPropertyTreeIterator>
- {
- CValueMap::iterator startRange, endRange;
- CValueMap::iterator currentIter;
- public:
- CMapQualifierIterator(CQualifierMap &_map, CValueMap::iterator _startRange, CValueMap::iterator _endRange)
- : startRange(_startRange), endRange(_endRange)
- {
- }
- // IPropertyTreeIterator
- virtual bool first() override
- {
- currentIter = startRange;
- return currentIter != endRange;
- }
- virtual bool next() override
- {
- currentIter++;
- return currentIter != endRange;
- }
- virtual bool isValid() override
- {
- return currentIter != endRange;
- }
- virtual IPropertyTree & query() override { return const_cast<IPropertyTree &>(*currentIter->second); }
- };
- IPropertyTreeIterator *checkMapIterator(const char *&xxpath, IPropertyTree &child)
- {
- /*
- * NB: IPT's are not thread safe. It is up to the caller to ensure multiple writers do not contend.
- * ( Dali for example ensures writer threads are exclusive )
- *
- * That means multiple reader threads could be here concurrently.
- * >1 could be constructing the qualifier map for the 1st time.
- * For new attr updates (where map already exists), it will block on map::crit,
- * so that there is at most 1 thread updating the map. The underlying unordered_multiset
- * is thread safe if 1 writer, and multiple readers.
- *
- * On initial map creation, allow concurrency, but only 1 will succeed to swap in the new active map.
- * That could mean a new attr/prop. mapping is lost, until next used.
- * NB: once the map is live, updates are write ops. and so, just as with the IPT
- * itself, it is expected that something will keep it thread safe (as Dali does)
- *
- */
- // NB: only support simple @<attrname>=<value> qualifiers
- if (((unsigned)-1) == siblingMapThreshold) // disabled
- return nullptr;
- PTree &_child = (PTree &)child;
- if (child.isCaseInsensitive()) // NB: could support but not worth it.
- return nullptr;
- IPTArrayValue *value = _child.queryValue();
- if (!value)
- return nullptr;
- CQualifierMap *map = value->queryMap();
- if (!map)
- {
- if (!value->isArray() || (value->elements() < siblingMapThreshold))
- return nullptr;
- }
- unsigned lhsLen, rhsLen;
- const char *rhsStart;
- const char *xpath = xxpath;
- if (!parseEqualityQualifier(xpath, lhsLen, rhsStart, rhsLen))
- return nullptr;
- MAKE_LSTRING(lhs, xxpath, lhsLen);
- MAKE_LSTRING(rhs, rhsStart, rhsLen);
- // NB: there can be a race here where >1 reader is constructing new map
- CValueMap *valueMap = nullptr;
- if (map)
- valueMap = map->addMappingIfNew(lhs, *value);
- else
- {
- OwnedPtr<CQualifierMap> newMap = new CQualifierMap();
- valueMap = newMap->addMapping(lhs, *value);
- /*
- * NB: it's possible another read thread got here 1st, and swapped in a map.
- * setMap returns the existing map, and the code below checks to see if it already
- * handles the 'lhs' we're adding, if it doesn't it re-adds the qualifier mappings.
- */
- map = value->setMap(newMap);
- if (!map) // successfully swapped newMap in.
- map = newMap.getClear(); // NB: setMap owns
- else // another thread has swapped in a map whilst I was creating new one
- valueMap = map->addMappingIfNew(lhs, *value);
- }
- xxpath = xpath; // update parsed position
- auto range = valueMap->find(rhs);
- if (range.first != range.second)
- return new CMapQualifierIterator(*map, range.first, range.second);
- else
- return LINK(nullPTreeIterator);
- }
- ///////////////////
- class SeriesPTIterator : implements IPropertyTreeIterator, public CInterface
- {
- public:
- IMPLEMENT_IINTERFACE;
- SeriesPTIterator() : current(NULL), cp(0)
- {
- }
- void addIterator(IPropertyTreeIterator *iter) { iters.append(*iter); }
- // IPropertyTreeIterator impl.
- virtual bool first() override
- {
- cp = 0;
- iterCount = iters.ordinality();
- if (nextIterator())
- return true;
- else
- return false;
- }
- virtual bool next() override
- {
- while (currentIter)
- {
- if (currentIter->next())
- {
- current = ¤tIter->query();
- return true;
- }
- if (nextIterator())
- return true;
- }
- current = NULL;
- return false;
- }
- virtual bool isValid() override { return (NULL != current); }
- virtual IPropertyTree & query() override { assertex(current); return *current; }
- private:
- bool nextIterator()
- {
- while (cp<iterCount)
- {
- currentIter = (IPropertyTreeIterator *) &iters.item(cp++);
- if (currentIter->first())
- {
- current = ¤tIter->query();
- return true;
- }
- }
- current = NULL;
- currentIter = NULL;
- return false;
- }
- IArray iters;
- IPropertyTreeIterator *currentIter;
- IPropertyTree *current;
- unsigned cp, iterCount;
- };
- ///////////////////
- CPTValue::CPTValue(size32_t size, const void *data, bool binary, bool raw, bool _compressed)
- {
- compressed = _compressed;
- if (!raw && binary && size > PTREE_COMPRESS_THRESHOLD)
- {
- unsigned newSize = size * PTREE_COMPRESS_BOTHER_PECENTAGE / 100;
- void *newData = NULL;
- ICompressor *compressor = NULL;
- try
- {
- newData = malloc(sizeof(size32_t) + newSize);
- compressor = createLZWCompressor();
- compressor->open(((char *)newData) + sizeof(size32_t), newSize);
- if (compressor->write(data, size)==size)
- {
- compressor->close();
- memcpy(newData, &size, sizeof(size32_t));
- newSize = sizeof(size32_t) + compressor->buflen();
- compressed = true;
- set(newSize, newData);
- }
- free(newData);
- compressor->Release();
- }
- catch (...)
- {
- if (newData)
- free(newData);
- if (compressor) compressor->Release();
- throw;
- }
- }
- if (raw || !compressed)
- set(size, data);
- }
- static void *uncompress(const void *src, size32_t &sz)
- {
- IExpander *expander = NULL;
- void *uncompressedValue = NULL;
- try
- {
- memcpy(&sz, src, sizeof(size32_t));
- assertex(sz);
- expander = createLZWExpander();
- src = ((const char *)src) + sizeof(size32_t);
- uncompressedValue = malloc(sz);
- assertex(uncompressedValue);
- expander->init(src);
- expander->expand(uncompressedValue);
- expander->Release();
- return uncompressedValue;
- }
- catch (...)
- {
- if (expander) expander->Release();
- if (uncompressedValue) free(uncompressedValue);
- throw;
- }
- }
- const void *CPTValue::queryValue() const
- {
- if (compressed)
- {
- size32_t sz;
- void *uncompressedValue = uncompress(get(), sz);
- ((MemoryAttr *)this)->setOwn(sz, uncompressedValue);
- compressed = false;
- }
- return get();
- }
- void CPTValue::serialize(MemoryBuffer &tgt)
- {
- //Retain backward compatibility for the serialization format.
- size32_t serialLen = (size32_t)length();
- tgt.append(serialLen);
- if (serialLen)
- {
- tgt.append(compressed);
- tgt.append(serialLen, get());
- }
- }
- void CPTValue::deserialize(MemoryBuffer &src)
- {
- size32_t sz;
- src.read(sz);
- if (sz)
- {
- src.read(compressed);
- set(sz, src.readDirect(sz));
- }
- else
- {
- compressed = false;
- clear();
- }
- }
- MemoryBuffer &CPTValue::getValue(MemoryBuffer &tgt, bool binary) const
- {
- if (compressed)
- {
- size32_t sz;
- void *uncompressedValue = uncompress(get(), sz);
- if (!binary) sz -= 1;
- tgt.append(sz, uncompressedValue);
- if (uncompressedValue)
- free(uncompressedValue);
- }
- else
- {
- if (binary)
- tgt.append((size32_t)length(), get());
- else
- tgt.append((size32_t)length()-1, get());
- }
- return tgt;
- }
- StringBuffer &CPTValue::getValue(StringBuffer &tgt, bool binary) const
- {
- if (compressed)
- {
- size32_t sz;
- void *uncompressedValue = NULL;
- try
- {
- uncompressedValue = uncompress(get(), sz);
- if (!binary) sz -= 1;
- tgt.append(sz, (const char *)uncompressedValue);
- free(uncompressedValue);
- }
- catch (IException *)
- {
- if (uncompressedValue) free(uncompressedValue);
- throw;
- }
- }
- else
- {
- if (binary) // this should probably be an assert?
- tgt.append((size32_t)length(), (const char *)get());
- else if (length())
- tgt.append((size32_t)length()-1, (const char *)get());
- }
- return tgt;
- }
- size32_t CPTValue::queryValueSize() const
- {
- if (compressed)
- {
- size32_t sz;
- memcpy(&sz, get(), sizeof(size32_t));
- return sz;
- }
- else
- return (size32_t)length();
- }
- ///////////////////
- CPTArray::~CPTArray()
- {
- if (map.load())
- delete map.load();
- }
- CQualifierMap *CPTArray::setMap(CQualifierMap *_map)
- {
- CQualifierMap *expected = nullptr;
- if (map.compare_exchange_strong(expected, _map))
- return nullptr;
- else
- return expected;
- }
- void CPTArray::addElement(IPropertyTree *tree)
- {
- append(*tree);
- CQualifierMap *map = queryMap();
- if (map)
- {
- if (tree->getAttributeCount())
- map->addMatchingValues(tree);
- }
- }
- void CPTArray::setElement(unsigned idx, IPropertyTree *tree)
- {
- CQualifierMap *map = queryMap();
- if (map)
- {
- // remove any mappings for existing element.
- if (isItem(idx))
- {
- IPropertyTree *existing = &((IPropertyTree &)item(idx));
- map->replaceMatchingValues(existing, tree);
- }
- else
- map->addMatchingValues(tree);
- }
- add(*tree, idx);
- }
- void CPTArray::removeElement(unsigned idx)
- {
- CQualifierMap *map = queryMap();
- IPropertyTree *existing = &((IPropertyTree &)item(idx));
- if (map)
- map->removeMatchingValues(existing);
- ((PTree *)existing)->setOwner(nullptr);
- remove(idx);
- }
- unsigned CPTArray::find(const IPropertyTree *search) const
- {
- IInterface **start = getArray();
- IInterface **last = start + ordinality();
- IInterface **members = start;
- while (true)
- {
- if (*members == search)
- return members-start;
- members++;
- if (members == last)
- break;
- }
- return NotFound;
- }
- //////////////////
- PTree::PTree(byte _flags, IPTArrayValue *_value, ChildMap *_children)
- {
- flags = _flags;
- children = LINK(_children);
- value = _value;
- }
- PTree::~PTree()
- {
- if (value) delete value;
- ::Release(children);
- }
- IPropertyTree *PTree::queryChild(unsigned index)
- {
- if (!value) return NULL;
- if (!value->isArray()) return this;
- IPropertyTree *v = value->queryElement(index);
- return v;
- }
- aindex_t PTree::findChild(IPropertyTree *child, bool remove)
- {
- if (value && value->isArray())
- {
- unsigned pos = value->find(child);
- if (remove && NotFound != pos)
- value->removeElement(pos);
- return pos;
- }
- else if (checkChildren())
- {
- IPropertyTree *_child = children->query(child->queryName());
- if (_child == child)
- {
- if (remove)
- children->removeExact(_child);
- return 0;
- }
- else if (_child)
- {
- PTree *__child = (PTree *) _child;
- return __child->findChild(child, remove);
- }
- }
- return NotFound;
- }
- ChildMap *PTree::checkChildren() const
- {
- return children;
- }
- void PTree::setLocal(size32_t l, const void *data, bool _binary)
- {
- if (value) delete value;
- if (l)
- value = new CPTValue(l, data, _binary);
- else
- value = NULL;
- if (_binary)
- IptFlagSet(flags, ipt_binary);
- else
- IptFlagClr(flags, ipt_binary);
- }
- void PTree::appendLocal(size32_t l, const void *data, bool binary)
- {
- if (0 == l) return;
- MemoryBuffer mb;
- if (value)
- {
- assertex(!value->isArray());
- assertex(binary == IptFlagTst(flags, ipt_binary));
- value->getValue(mb, binary);
- mb.append(l, data);
- delete value;
-
- l = mb.length();
- data = mb.toByteArray();
- }
- if (l)
- value = new CPTValue(l, data, binary);
- else
- value = NULL;
- if (binary)
- IptFlagSet(flags, ipt_binary);
- else
- IptFlagClr(flags, ipt_binary);
- }
- // IPropertyTree impl.
- bool PTree::hasProp(const char * xpath) const
- {
- const char *prop = splitXPathX(xpath);
- if (isAttribute(prop)) // JCS - note no wildcards on attributes
- {
- if (prop != xpath)
- {
- MAKE_LSTRING(path, xpath, prop-xpath);
- Owned<IPropertyTreeIterator> iter = getElements(path);
- if (iter->first())
- {
- do
- {
- IPropertyTree &branch = iter->query();
- if (branch.hasProp(prop))
- return true;
- }
- while (iter->next());
- }
- return false;
- }
- else
- return nullptr != findAttribute(xpath);
- }
- else
- {
- IPropertyTreeIterator *iter = getElements(xpath);
- bool res = iter->first();
- iter->Release();
- return res;
- }
- }
- const char *PTree::queryProp(const char *xpath) const
- {
- if (!xpath)
- {
- if (!value) return NULL;
- return (const char *) value->queryValue();
- }
- else if (isAttribute(xpath))
- return getAttributeValue(xpath);
- else
- {
- const char *prop = splitXPathX(xpath);
- if (isAttribute(prop))
- {
- MAKE_LSTRING(path, xpath, prop-xpath);
- IPropertyTree *branch = queryPropTree(path);
- if (!branch) return NULL;
- return branch->queryProp(prop);
- }
- else
- {
- IPropertyTree *branch = queryPropTree(xpath);
- if (!branch) return NULL;
- return branch->queryProp(NULL);
- }
- }
- }
- bool PTree::getProp(const char *xpath, StringBuffer &ret) const
- {
- if (!xpath)
- {
- if (!value) return false;
- value->getValue(ret, IptFlagTst(flags, ipt_binary));
- return true;
- }
- else if (isAttribute(xpath))
- {
- const char *value = getAttributeValue(xpath);
- if (!value) return false;
- ret.append(value);
- return true;
- }
- else
- {
- const char *prop = splitXPathX(xpath);
- if (isAttribute(prop))
- {
- MAKE_LSTRING(path, xpath, prop-xpath)
- IPropertyTree *branch = queryPropTree(path);
- if (!branch) return false;
- return branch->getProp(prop, ret);
- }
- else
- {
- IPropertyTree *branch = queryPropTree(xpath);
- if (!branch) return false;
- return branch->getProp(NULL, ret);
- }
- }
- }
- void PTree::setProp(const char *xpath, const char *val)
- {
- if (!xpath || '\0' == *xpath)
- {
- if (!val)
- {
- if (value) delete value;
- value = NULL;
- }
- else
- {
- size32_t l=(size32_t)strlen(val);
- if (!l)
- {
- if (value) delete value;
- value = NULL;
- }
- else
- setLocal(l+1, val);
- }
- }
- else if (isAttribute(xpath))
- {
- if (!val)
- removeAttribute(xpath);
- else
- setAttribute(xpath, val);
- }
- else
- {
- const char *prop;
- IPropertyTree *branch = splitBranchProp(xpath, prop, true);
- if (isAttribute(prop))
- branch->setProp(prop, val);
- else
- {
- if (val)
- {
- IPropertyTree *propBranch = queryCreateBranch(branch, prop);
- propBranch->setProp(NULL, val);
- }
- else
- branch->removeProp(prop);
- }
- }
- }
- aindex_t PTree::getChildMatchPos(const char *xpath)
- {
- Owned<IPropertyTreeIterator> childIter = getElements(xpath);
- if (!childIter->first())
- return (aindex_t)-1;
- IPropertyTree &childMatch = childIter->query();
- #ifdef _DEBUG
- if (childIter->next())
- AMBIGUOUS_PATH("addPropX", xpath);
- #endif
- if (value)
- if (value->isArray())
- return findChild(&childMatch);
- else
- return 0;
- else
- return 0;
- }
- void PTree::resolveParentChild(const char *xpath, IPropertyTree *&parent, IPropertyTree *&child, StringAttr &path, StringAttr &qualifier)
- {
- parent = child = NULL;
- if (!xpath)
- throw MakeIPTException(-1, "No path to resolve parent from");
- const char *end = xpath+strlen(xpath);
- const char *prop = end;
- while (prop != xpath && *(prop-1) != '/')
- --prop;
- size32_t ps = prop-xpath;
- if (ps)
- {
- path.set(xpath, ps);
- Owned<IPropertyTreeIterator> pathIter = getElements(path);
- if (!pathIter->first())
- throw MakeIPTException(-1, "resolveParentChild: path not found %s", xpath);
- /* If 'path' resolves to iterator of this, then treat as if no leading path
- * i.e. "./x", or "././.x" is equivalent to "x"
- */
- if (this != &pathIter->query())
- {
- IPropertyTree *currentPath = NULL;
- #ifdef _DEBUG
- bool multiplePaths = false;
- #endif
- bool multipleChildMatches = false;
- for (;;)
- {
- // JCSMORE - a bit annoying has to be done again once path has been established
- currentPath = &pathIter->query();
- Owned<IPropertyTreeIterator> childIter = currentPath->getElements(prop);
- if (childIter->first())
- {
- child = &childIter->query();
- #ifdef _DEBUG
- if (parent)
- AMBIGUOUS_PATH("resolveParentChild", xpath);
- #endif
- if (!multipleChildMatches && childIter->next())
- multipleChildMatches = true;
- parent = currentPath;
- }
- if (pathIter->next())
- {
- #ifdef _DEBUG
- multiplePaths = true;
- #endif
- }
- else
- break;
- }
- if (!parent)
- {
- #ifdef _DEBUG
- if (multiplePaths) // i.e. no unique path to child found and multiple parent paths
- AMBIGUOUS_PATH("resolveParentChild", xpath);
- #endif
- parent = currentPath;
- }
- if (multipleChildMatches)
- child = NULL; // single parent, but no single child.
- path.set(prop);
- const char *pstart = prop;
- bool wild;
- readWildId(prop, wild);
- size32_t s = prop-pstart;
- if (wild)
- throw MakeXPathException(pstart, PTreeExcpt_XPath_ParseError, s-1, "Wildcards not permitted on add");
- assertex(s);
- path.set(pstart, s);
- qualifier.set(prop);
- return;
- }
- }
- assertex(prop && *prop);
- parent = this;
- const char *pstart = prop;
- bool wild;
- readWildId(prop, wild);
- assertex(!wild);
- size32_t s = prop-pstart;
- if (*prop && *prop != '[')
- throw MakeXPathException(pstart, PTreeExcpt_XPath_ParseError, s, "Qualifier expected e.g. [..]");
- path.set(pstart, s);
- if (checkChildren())
- child = children->query(path);
- if (child)
- qualifier.set(prop);
- else
- qualifier.clear();
- }
- void PTree::addProp(const char *xpath, const char *val)
- {
- if (!xpath || '\0' == *xpath)
- addLocal((size32_t)strlen(val)+1, val);
- else if (isAttribute(xpath))
- setAttribute(xpath, val);
- else if ('[' == *xpath)
- {
- aindex_t pos = getChildMatchPos(xpath);
- if ((aindex_t) -1 == pos)
- throw MakeIPTException(-1, "addProp: qualifier unmatched %s", xpath);
- addLocal((size32_t)strlen(val)+1, val, false, pos);
- }
- else
- {
- IPropertyTree *parent, *child;
- StringAttr path, qualifier;
- resolveParentChild(xpath, parent, child, path, qualifier);
- if (parent != this)
- parent->addProp(path, val);
- else if (child)
- child->addProp(qualifier, val);
- else
- setProp(path, val);
- }
- }
- void PTree::appendProp(const char *xpath, const char *val)
- {
- if (!xpath || '\0' == *xpath)
- appendLocal((size_t)strlen(val)+1, val, false);
- else if (isAttribute(xpath))
- {
- StringBuffer newVal;
- getProp(xpath, newVal);
- newVal.append(val);
- setAttribute(xpath, newVal.str());
- }
- else if ('[' == *xpath)
- {
- IPropertyTree *qualified = queryPropTree(xpath);
- if (!qualified)
- throw MakeIPTException(-1, "appendProp: qualifier unmatched %s", xpath);
- qualified->appendProp(nullptr, val);
- }
- else
- {
- IPropertyTree *parent, *child;
- StringAttr path, qualifier;
- resolveParentChild(xpath, parent, child, path, qualifier);
- if (parent != this)
- parent->appendProp(path, val);
- else if (child)
- child->appendProp(qualifier, val);
- else
- setProp(path, val);
- }
- }
- bool PTree::getPropBool(const char *xpath, bool dft) const
- {
- const char *val = queryProp(xpath);
- if (val && *val)
- return strToBool(val);
- else
- return dft;
- }
- __int64 PTree::getPropInt64(const char *xpath, __int64 dft) const
- {
- if (!xpath)
- {
- if (!value) return dft;
- else
- {
- const char *v = (const char *)value->queryValue();
- if (!v || !*v) return dft;
- else return _atoi64(v);
- }
- }
- else if (isAttribute(xpath))
- {
- const char *v = getAttributeValue(xpath);
- if (!v || !*v) // intentional return dft if attribute equals ""
- return dft;
- return _atoi64(v);
- }
- else
- {
- const char *prop = splitXPathX(xpath);
- if (isAttribute(prop))
- {
- MAKE_LSTRING(path, xpath, prop-xpath);
- IPropertyTree *branch = queryPropTree(path);
- if (!branch) return dft;
- return branch->getPropInt64(prop, dft);
- }
- else
- {
- IPropertyTree *branch = queryPropTree(xpath);
- if (!branch) return dft;
- return branch->getPropInt64(NULL, dft);
- }
- }
- }
- void PTree::setPropInt64(const char * xpath, __int64 val)
- {
- if (!xpath || '\0' == *xpath)
- {
- char buf[23];
- numtostr(buf, val);
- setLocal((size32_t)strlen(buf)+1, buf);
- }
- else if (isAttribute(xpath))
- {
- char buf[23];
- numtostr(buf, val);
- setAttribute(xpath, buf);
- }
- else
- {
- const char *prop;
- IPropertyTree *branch = splitBranchProp(xpath, prop, true);
- if (isAttribute(prop))
- branch->setPropInt64(prop, val);
- else
- {
- IPropertyTree *propBranch = queryCreateBranch(branch, prop);
- propBranch->setPropInt64(NULL, val);
- }
- }
- }
- void PTree::addPropInt64(const char *xpath, __int64 val)
- {
- if (!xpath || '\0' == *xpath)
- {
- char buf[23];
- numtostr(buf,val);
- addLocal((size32_t)strlen(buf)+1, buf);
- }
- else if (isAttribute(xpath))
- {
- char buf[23];
- numtostr(buf, val);
- setAttribute(xpath, buf);
- }
- else if ('[' == *xpath)
- {
- char buf[23];
- numtostr(buf, val);
- aindex_t pos = getChildMatchPos(xpath);
- if ((aindex_t) -1 == pos)
- throw MakeIPTException(-1, "addPropInt64: qualifier unmatched %s", xpath);
- addLocal((size32_t)strlen(buf)+1, buf, false, pos);
- }
- else
- {
- IPropertyTree *parent, *child;
- StringAttr path, qualifier;
- resolveParentChild(xpath, parent, child, path, qualifier);
- if (parent != this)
- parent->addPropInt64(path, val);
- else if (child)
- child->addPropInt64(qualifier, val);
- else
- setPropInt64(path, val);
- }
- }
- int PTree::getPropInt(const char *xpath, int dft) const
- {
- return (int) getPropInt64(xpath, dft); // underlying type always __int64 (now)
- }
- void PTree::setPropInt(const char *xpath, int val)
- {
- setPropInt64(xpath, val); // underlying type always __int64 (now)
- }
- void PTree::addPropInt(const char *xpath, int val)
- {
- addPropInt64(xpath, val); // underlying type always __int64 (now)
- }
- double PTree::getPropReal(const char *xpath, double dft) const
- {
- const char *val = queryProp(xpath);
- return val?atof(val):dft;
- }
- bool PTree::isCompressed(const char *xpath) const
- {
- if (!xpath)
- return (value && value->isCompressed());
- else if (isAttribute(xpath))
- return false;
- else
- {
- const char *prop = splitXPathX(xpath);
- if (!isAttribute(prop))
- {
- IPropertyTree *branch = queryPropTree(xpath);
- if (branch)
- return branch->isCompressed(nullptr);
- }
- }
- return false;
- }
- bool PTree::isBinary(const char *xpath) const
- {
- if (!xpath)
- return IptFlagTst(flags, ipt_binary);
- else if (isAttribute(xpath)) // still positing that attr cannot be binary for now.
- return false;
- else
- {
- const char *prop = splitXPathX(xpath);
- if (!isAttribute(prop))
- {
- IPropertyTree *branch = queryPropTree(xpath);
- if (branch)
- return branch->isBinary(nullptr);
- }
- }
- return false;
- }
- bool PTree::renameTree(IPropertyTree *child, const char *newName) // really here for hook for SDS (can substationally optimize remote action)
- {
- if (0==strcmp(newName, child->queryName()) && NotFound!=findChild(child)) return false;
- Linked<IPropertyTree> tmp = child;
- if (removeTree(child))
- {
- addPropTree(newName, child);
- tmp.getClear(); // addPropTree has taken ownership.
- return true;
- }
- return false;
- }
- bool PTree::renameProp(const char *xpath, const char *newName)
- {
- if (!xpath || '\0' == *xpath)
- throw MakeIPTException(-1, "renameProp: cannot rename self, renameProp has to rename in context of a parent");
- if (strcmp(xpath,"/")==0) // rename of self allowed assuming no parent
- setName(newName);
- else if ('[' == *xpath)
- UNIMPLEMENTED;
- else if (isAttribute(xpath))
- {
- StringBuffer val;
- if (!getProp(xpath, val))
- return false;
- removeProp(xpath);
- addProp(newName, val.str());
- }
- else
- {
- StringBuffer path;
- const char *prop = splitXPath(xpath, path);
- assertex(prop);
- if (path.length())
- {
- Owned<IPropertyTreeIterator> iter = getElements(path.str());
- if (!iter->first())
- return false;
- IPropertyTree &branch = iter->query();
- #ifdef _DEBUG
- if (iter->next())
- AMBIGUOUS_PATH("renameProp", xpath);
- #endif
- return branch.renameProp(prop, newName);
- }
- else
- {
- IPropertyTree *old = queryPropTree(xpath);
- if (!old)
- return false;
- return renameTree(old, newName);
- }
- }
- return true;
- }
- bool PTree::getPropBin(const char *xpath, MemoryBuffer &ret) const
- {
- CHECK_ATTRIBUTE(xpath);
- if (!xpath)
- {
- if (!value) return true; // exists, but no value
- value->getValue(ret, IptFlagTst(flags, ipt_binary));
- return true;
- }
- else
- {
- const char *prop = splitXPathX(xpath);
- if (isAttribute(prop))
- {
- MAKE_LSTRING(path, xpath, prop-xpath);
- IPropertyTree *branch = queryPropTree(path);
- if (!branch) return false;
- return branch->getPropBin(prop, ret);
- }
- else
- {
- IPropertyTree *branch = queryPropTree(xpath);
- if (!branch) return false;
- return branch->getPropBin(NULL, ret);
- }
- }
- }
- void PTree::setPropBin(const char * xpath, size32_t size, const void *data)
- {
- CHECK_ATTRIBUTE(xpath);
- if (!xpath || '\0' == *xpath)
- setLocal(size, data, true);
- else
- {
- const char *prop;
- IPropertyTree *branch = splitBranchProp(xpath, prop, true);
- if (isAttribute(prop))
- branch->setPropBin(prop, size, data);
- else
- {
- IPropertyTree *propBranch = queryCreateBranch(branch, prop);
- propBranch->setPropBin(NULL, size, data);
- }
- }
- }
- void PTree::addPropBin(const char *xpath, size32_t size, const void *data)
- {
- CHECK_ATTRIBUTE(xpath);
- if (!xpath || '\0' == *xpath)
- addLocal(size, data, true);
- else if ('[' == *xpath)
- {
- aindex_t pos = getChildMatchPos(xpath);
- if ((aindex_t) -1 == pos)
- throw MakeIPTException(-1, "addPropBin: qualifier unmatched %s", xpath);
- addLocal(size, data, true, pos);
- }
- else
- {
- IPropertyTree *parent, *child;
- StringAttr path, qualifier;
- resolveParentChild(xpath, parent, child, path, qualifier);
- if (parent != this)
- parent->addPropBin(path, size, data);
- else if (child)
- child->addPropBin(qualifier, size, data);
- else
- setPropBin(path, size, data);
- }
- }
- void PTree::appendPropBin(const char *xpath, size32_t size, const void *data)
- {
- CHECK_ATTRIBUTE(xpath);
- if (!xpath || '\0' == *xpath)
- appendLocal(size, data, true);
- else if ('[' == *xpath)
- {
- IPropertyTree *qualified = queryPropTree(xpath);
- if (!qualified)
- throw MakeIPTException(-1, "appendPropBin: qualifier unmatched %s", xpath);
- qualified->appendPropBin(nullptr, size, data);
- }
- else
- {
- IPropertyTree *parent, *child;
- StringAttr path, qualifier;
- resolveParentChild(xpath, parent, child, path, qualifier);
- if (parent != this)
- parent->appendPropBin(path, size, data);
- else if (child)
- child->appendPropBin(qualifier, size, data);
- else
- setPropBin(path, size, data);
- }
- }
- IPropertyTree *PTree::getPropTree(const char *xpath) const
- {
- IPropertyTree *tree = queryPropTree(xpath);
- return LINK(tree);
- }
- IPropertyTree *PTree::queryPropTree(const char *xpath) const
- {
- Owned<IPropertyTreeIterator> iter = getElements(xpath);
- IPropertyTree *element = NULL;
- if (iter->first())
- {
- element = &iter->query();
- #ifdef _DEBUG
- //The following call can double the cost of finding a match from an IPropertyTree
- if (iter->next())
- AMBIGUOUS_PATH("getProp",xpath);
- #endif
- }
- return element;
- }
- void PTree::replaceSelf(IPropertyTree *val)
- {
- Owned<IAttributeIterator> aiter = getAttributes();
- StringArray attrs;
- ForEach (*aiter)
- attrs.append(aiter->queryName());
- ForEachItemIn(a, attrs)
- removeProp(attrs.item(a));
- ICopyArrayOf<IPropertyTree> elems;
- Owned<IPropertyTreeIterator> iter = getElements("*");
- ForEach(*iter)
- elems.append(iter->query());
- ForEachItemIn(e, elems)
- removeTree(&elems.item(e));
- aiter.setown(val->getAttributes());
- ForEach(*aiter)
- setProp(aiter->queryName(), aiter->queryValue());
- iter.setown(val->getElements("*"));
- ForEach(*iter)
- {
- IPropertyTree &node = iter->query();
- node.Link();
- addPropTree(node.queryName(), &node);
- }
- val->Release();
- }
- IPropertyTree *PTree::setPropTree(const char *xpath, IPropertyTree *val)
- {
- CHECK_ATTRIBUTE(xpath);
- if (NULL == xpath)
- {
- replaceSelf(val);
- return this;
- }
- else
- {
- StringAttr prop, qualifier;
- IPropertyTree *branch, *child;
- resolveParentChild(xpath, branch, child, prop, qualifier);
- if (branch == this)
- {
- IPropertyTree *_val = ownPTree(val);
- dbgassertex(QUERYINTERFACE(_val, PTree));
- PTree *__val = static_cast<PTree *>(_val);
- __val->setName(prop);
- addingNewElement(*_val, ANE_SET);
- if (!checkChildren()) createChildMap();
- children->set(prop, _val);
- return _val;
- }
- else
- return branch->setPropTree(prop, val);
- }
- }
- bool PTree::isArray(const char *xpath) const
- {
- if (!xpath || !*xpath) //item in an array child of parent? I don't think callers ever access array container directly
- return arrayOwner && arrayOwner->isArray();
- else if (isAttribute(xpath))
- return false;
- else
- {
- StringBuffer path;
- const char *prop = splitXPath(xpath, path);
- assertex(prop);
- if (!isAttribute(prop))
- {
- if (path.length())
- {
- Owned<IPropertyTreeIterator> iter = getElements(path.str());
- if (!iter->first())
- return false;
- IPropertyTree &branch = iter->query();
- #ifdef _DEBUG
- if (iter->next())
- AMBIGUOUS_PATH("isArray", xpath);
- #endif
- return branch.isArray(prop);
- }
- else
- {
- IPropertyTree *child = children->query(xpath);
- if (child)
- {
- PTree *tree = static_cast<PTree *>(child);
- return (tree && tree->value && tree->value->isArray());
- }
- }
- }
- }
- return false;
- }
- void PTree::addPTreeArrayItem(IPropertyTree *existing, const char *xpath, PTree *val, aindex_t pos)
- {
- IPropertyTree *iptval = static_cast<IPropertyTree *>(val);
- PTree *tree = nullptr;
- if (existing)
- {
- dbgassertex(QUERYINTERFACE(existing, PTree));
- tree = static_cast<PTree *>(existing);
- if (tree->value && tree->value->isArray())
- {
- val->setOwner(tree->value);
- if ((aindex_t) -1 == pos)
- tree->value->addElement(iptval);
- else
- tree->value->setElement(pos, iptval);
- return;
- }
- }
- IPTArrayValue *array = new CPTArray();
- IPropertyTree *container = create(xpath, array);
- val->setOwner(array);
- if (existing)
- {
- array->addElement(LINK(existing));
- assertex((aindex_t) -1 == pos || 0 == pos);
- if ((aindex_t) -1 == pos)
- array->addElement(iptval);
- else
- array->setElement(0, iptval);
- tree->setOwner(array);
- children->replace(xpath, container);
- }
- else
- {
- array->addElement(iptval);
- children->set(xpath, container);
- }
- }
- IPropertyTree *PTree::addPropTree(const char *xpath, IPropertyTree *val, bool alwaysUseArray)
- {
- if (!xpath || '\0' == *xpath)
- throw MakeIPTException(PTreeExcpt_InvalidTagName, "Invalid xpath for property tree insertion specified");
- else
- {
- CHECK_ATTRIBUTE(xpath);
- const char *x = xpath;
- for (;;)
- {
- if (!*x++)
- {
- IPropertyTree *_val = ownPTree(val);
- dbgassertex(QUERYINTERFACE(_val, PTree));
- PTree *__val = static_cast<PTree *>(_val);
- /* NB: potentially param xpath is a reference to the existing name.
- * So fetch new name ptr after set.
- */
- __val->setName(xpath);
- xpath = __val->queryName();
- addingNewElement(*_val, -1);
- if (checkChildren())
- {
- IPropertyTree *child = children->query(xpath);
- if (child)
- {
- addPTreeArrayItem(child, xpath, __val);
- return _val;
- }
- }
- else
- createChildMap();
- if (alwaysUseArray)
- addPTreeArrayItem(nullptr, xpath, __val);
- else
- children->set(xpath, _val);
- return _val;
- }
- if ('/' == *x || '[' == *x)
- break;
- }
- IPropertyTree *parent, *child;
- StringAttr path, qualifier;
- resolveParentChild(xpath, parent, child, path, qualifier);
- if (parent != this)
- return parent->addPropTree(path, val);
- else
- {
- aindex_t pos = (aindex_t)-1;
- if (!qualifier.isEmpty())
- {
- pos = ((PTree *)child)->getChildMatchPos(qualifier);
- if ((aindex_t) -1 == pos)
- throw MakeIPTException(-1, "addPropTree: qualifier unmatched %s", xpath);
- }
- IPropertyTree *_val = ownPTree(val);
- dbgassertex(QUERYINTERFACE(_val, PTree));
- PTree *__val = static_cast<PTree *>(_val);
- __val->setName(path);
- addingNewElement(*_val, pos);
- if (child)
- {
- addPTreeArrayItem(child, path, __val, pos);
- }
- else
- {
- if (!checkChildren()) createChildMap();
- if (alwaysUseArray)
- addPTreeArrayItem(nullptr, path, __val);
- else
- children->set(path, _val);
- children->set(path, _val);
- }
- return _val;
- }
- }
- }
- IPropertyTree *PTree::addPropTree(const char *xpath, IPropertyTree *val)
- {
- return addPropTree(xpath, val, false);
- }
- IPropertyTree *PTree::addPropTreeArrayItem(const char *xpath, IPropertyTree *val)
- {
- return addPropTree(xpath, val, true);
- }
- bool PTree::removeTree(IPropertyTree *child)
- {
- if (child == this)
- throw MakeIPTException(-1, "Cannot remove self");
- if (checkChildren())
- {
- IPropertyTree *_child = children->query(child->queryName());
- if (_child)
- {
- if (child == _child)
- return children->removeExact(child);
- else
- {
- IPTArrayValue *value = ((PTree *)_child)->queryValue();
- if (value && value->isArray())
- {
- unsigned pos = value->find(child);
- if (NotFound != pos)
- {
- removingElement(child, pos);
- value->removeElement(pos);
- if (0 == value->elements())
- children->removeExact(_child);
- return true;
- }
- }
- }
- }
- }
- return false;
- }
- bool PTree::removeProp(const char *xpath)
- {
- if (xpath && isAttribute(xpath))
- return removeAttribute(xpath);
- StringBuffer path;
- const char *prop = splitXPath(xpath, path);
- if (!prop)
- throw MakeXPathException(xpath, PTreeExcpt_XPath_ParseError, 0, "Invalid xpath for property deletion");
- if (path.length())
- {
- Owned<IPropertyTreeIterator> iter = getElements(path.str());
- if (!iter)
- return false;
-
- bool res = false;
- if (iter->first())
- {
- do
- {
- IPropertyTree *branch = &iter->query();
- if (branch) {
- res = branch->removeProp(prop);
- if (res)
- break; // deleted first may be another
- }
- }
- while (iter->next());
- }
- return res;
- }
- else
- {
- if (!queryNextUnquoted(xpath, '[') && !strchr(prop, '*')) // have to work hard to locate qualified prop tree from parent.
- {
- if (!checkChildren()) return false;
- return children->remove(prop); // NB: might be multivalued.
- }
- const char *xxpath = prop;
- readID(xxpath, false);
- const char *idEnd = xxpath;
- if ('[' == *xxpath)
- {
- ++xxpath;
- const char *digitStart = xxpath;
- while (*xxpath && ']' != *xxpath && isdigit(*xxpath)) xxpath++;
- assertex(*xxpath != '\0');
- if (']' == *xxpath) // so it's a digit index!
- {
- StringAttr id(prop, idEnd-prop);
- PTree *child = children?(PTree *)children->query(id):NULL;
- if (child)
- {
- if (child->value && child->value->isArray() && child->value->elements()>1)
- {
- StringAttr digit(digitStart, xxpath-digitStart);
- unsigned i = atoi(digit);
- if (i && i <= child->value->elements())
- {
- removingElement(child->value->queryElement(i-1), i-1);
- child->value->removeElement(i-1);
- return true;
- }
- }
- else
- return children->removeExact(child);
- }
- return false;
- }
- }
- // JCSMORE - This is ridiculous for qualifier have to iterate to find match ok, but then finding where that *was* gees!
- Owned <IPropertyTreeIterator> iter = getElements(prop);
- if (!iter->first())
- return false;
- IPropertyTree *match = &iter->query();
- #if 0 // intentionally removes first encountered
- if (iter->next())
- {
- AMBIGUOUS_PATH("removeProp",xpath);
- }
- #endif
- return removeTree(match);
- }
- return false;
- }
- aindex_t PTree::queryChildIndex(IPropertyTree *child)
- {
- return findChild(child);
- }
- StringBuffer &PTree::getName(StringBuffer &ret) const
- {
- ret.append(queryName());
- return ret;
- }
- typedef CopyReferenceArrayOf<AttrValue> AttrArray;
- IAttributeIterator *PTree::getAttributes(bool sorted) const
- {
- class CAttributeIterator : implements IAttributeIterator, public CInterface
- {
- Linked<const PTree> parent;
- AttrValue *cur = nullptr;
- public:
- IMPLEMENT_IINTERFACE;
- CAttributeIterator(const PTree *_parent) : parent(_parent)
- {
- }
- // IAttributeIterator impl.
- virtual bool first() override
- {
- cur = parent->getNextAttribute(nullptr);
- return cur ? true : false;
- }
- virtual bool next() override
- {
- cur = parent->getNextAttribute(cur);
- return cur ? true : false;
- }
- virtual bool isValid() override { return cur ? true : false; }
- virtual const char *queryName() const override
- {
- return cur->key.get();
- }
- virtual const char *queryValue() const override
- {
- return cur->value.get();
- }
- virtual StringBuffer &getValue(StringBuffer &out) override
- {
- out.append(queryValue());
- return out;
- }
- virtual unsigned count() override { return parent->getAttributeCount(); }
- };
- class CSortedAttributeIterator : implements IAttributeIterator, public CInterface
- {
- typedef ArrayIteratorOf<AttrArray, AttrValue &> AttrIterator;
- AttrArray attrs;
- AttrValue *cur;
- AttrIterator *iter;
- Linked<const PTree> parent;
- public:
- IMPLEMENT_IINTERFACE;
- static int compareAttrs(AttrValue * const *ll, AttrValue * const *rr)
- {
- return stricmp((*ll)->key.get(), (*rr)->key.get());
- };
- CSortedAttributeIterator(const PTree *_parent) : cur(NULL), iter(NULL), parent(_parent)
- {
- AttrValue *cur = parent->getNextAttribute(nullptr);
- if (cur)
- {
- do
- {
- attrs.append(*cur);
- cur = parent->getNextAttribute(cur);
- }
- while (cur);
- attrs.sort(compareAttrs);
- iter = new AttrIterator(attrs);
- }
- }
- ~CSortedAttributeIterator()
- {
- if (iter)
- delete iter;
- }
- // IAttributeIterator impl.
- virtual bool first() override
- {
- if (!iter) return false;
- if (!iter->first()) { cur = NULL; return false; }
- cur = &iter->query();
- return true;
- }
- virtual bool next() override
- {
- if (!iter) return false;
- if (!iter->next()) { cur = NULL; return false; }
- cur = &iter->query();
- return true;
- }
- virtual bool isValid() override { return cur!=NULL; }
- virtual const char *queryName() const override
- {
- assertex(cur);
- return cur->key.get();
- }
- virtual const char *queryValue() const override
- {
- assertex(cur);
- return cur->value.get();
- }
- virtual StringBuffer &getValue(StringBuffer &out) override
- {
- assertex(cur);
- return out.append(queryValue());
- }
- virtual unsigned count() override { return attrs.ordinality(); }
- };
- if (sorted)
- return new CSortedAttributeIterator(this);
- else
- return new CAttributeIterator(this);
- }
- ///////////////////
- class CIndexIterator : implements IPropertyTreeIterator, public CInterface
- {
- Owned<IPropertyTreeIterator> subIter;
- IPropertyTree *celem;
- unsigned index, current;
- public:
- IMPLEMENT_IINTERFACE;
- CIndexIterator(IPropertyTreeIterator *_subIter, unsigned _index) : subIter(_subIter), index(_index)
- {
- }
- // IPropertyTreeIterator
- virtual bool first() override
- {
- if (!index)
- return false;
- if (!subIter->first())
- return false;
- current = 1;
- celem = NULL;
- do
- {
- if (current == index)
- {
- celem = &subIter->query();
- return true;
- }
- if (!subIter->next())
- return false;
- } while (++current <= index);
- return false;
- }
- virtual bool isValid() override
- {
- return celem && (index >= current);
- }
- virtual bool next() override
- {
- celem = NULL;
- return false;
- }
- virtual IPropertyTree & query() override
- {
- return *celem;
- }
- };
- IPropertyTreeIterator *PTree::getElements(const char *xpath, IPTIteratorCodes flags) const
- {
- // NULL iterator for local value (i.e. maybe be single value or array)
- if (NULL == xpath || '\0' == *xpath)
- return new SingleIdIterator(*this);
- Owned<IPropertyTreeIterator> iter;
- const char *_xpath = xpath;
- bool root=true;
- restart:
- switch (*xpath)
- {
- case '.':
- root=false;
- ++xpath;
- if ('\0' == *xpath)
- return new SingleIdIterator(*this);
- else if ('/' != *xpath)
- throw MakeXPathException(xpath-1, PTreeExcpt_XPath_Unsupported, 0, "\"/\" expected");
- goto restart;
- case '/':
- ++xpath;
- if ('/' == *xpath)
- {
- iter.setown(getElements(xpath+1));
- if (checkChildren())
- {
- IPropertyTreeIterator *iter2 = new PTIdMatchIterator(this, "*", isnocase(), flags & iptiter_sort);
- iter2 = new PTStackIterator(iter2, xpath-1);
- SeriesPTIterator *series = new SeriesPTIterator();
- series->addIterator(iter.getClear());
- series->addIterator(iter2);
- return series;
- }
- else
- return iter.getClear();
- }
- else if (root)
- throw MakeXPathException(xpath, PTreeExcpt_XPath_Unsupported, 0, "Root specifier \"/\" specifier is not supported");
- else if ('\0' == *xpath)
- return new SingleIdIterator(*this);
- goto restart;
- case '[':
- {
- ++xpath;
- if (isdigit(*xpath)) {
- StringAttr index;
- xpath = readIndex(xpath, index);
- unsigned i = atoi(index.get());
- if (i)
- {
- if (value && value->isArray())
- {
- IPropertyTree *element = value->queryElement(--i);
- if (element)
- {
- iter.setown(element->getElements(NULL));
- }
- }
- else if (i == 1)
- iter.setown(new SingleIdIterator(*this));
- }
- }
- else
- {
- if (checkPattern(xpath))
- iter.setown(new SingleIdIterator(*this));
- }
- if (']' != *xpath)
- throw MakeXPathException(_xpath, PTreeExcpt_XPath_ParseError, xpath-_xpath, "Qualifier brace unclosed");
- ++xpath;
- break;
- }
- default:
- {
- bool wild;
- const char *start = xpath;
- readWildId(xpath, wild);
- size32_t s = xpath-start;
- if (s)
- {
- MAKE_LSTRING(id, start, s);
- if (checkChildren())
- {
- IPropertyTree *child = NULL;
- if (!wild)
- child = children->query(id);
- if ((wild || child) && '[' == *xpath) // check for local index not iterative qualifier.
- {
- const char *xxpath = xpath+1;
- if (isdigit(*xxpath)) {
- StringAttr idxstr;
- xxpath = readIndex(xxpath, idxstr);
- if (']' != *xxpath)
- throw MakeXPathException(_xpath, PTreeExcpt_XPath_ParseError, xpath-_xpath, "Qualifier brace unclosed");
- ++xxpath;
- unsigned index = atoi(idxstr.get());
- if (index)
- {
- Owned<IPropertyTreeIterator> _iter = getElements(id);
- if (_iter->first())
- {
- do
- {
- if (0 == --index)
- {
- iter.setown(new SingleIdIterator((PTree &)_iter->query()));
- break;
- }
- }
- while (_iter->next());
- }
- }
- xpath = xxpath;
- }
- else
- {
- const char *start = xxpath-1;
- for (;;)
- {
- char quote = 0;
- while (']' != *(++xxpath) || quote)
- {
- switch (*xxpath) {
- case '\"':
- case '\'':
- {
- if (quote)
- {
- if (*xxpath == quote)
- quote = 0;
- }
- else
- quote = *xxpath;
- break;
- }
- case '\0':
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xxpath-start, "Qualifier brace unclosed");
- }
- }
- ++xxpath;
- if ('[' == *xxpath)
- {
- ++xxpath;
- if (isdigit(*xxpath))
- {
- const char *lhsStart = start+1;
- Owned<IPropertyTreeIterator> siter = checkMapIterator(lhsStart, *child);
- if (!siter)
- {
- if (wild)
- iter.setown(new PTIdMatchIterator(this, id, isnocase(), flags & iptiter_sort));
- else
- iter.setown(child->getElements(NULL));
- StringAttr qualifier(start, (xxpath-1)-start);
- siter.setown(new PTStackIterator(iter.getClear(), qualifier.get()));
- }
- StringAttr index;
- xxpath = readIndex(xxpath, index);
- unsigned i = atoi(index.get());
- iter.setown(new CIndexIterator(siter.getClear(), i));
- ++xxpath;
- break;
- }
- }
- else
- {
- if (!wild)
- {
- const char *lhsStart = start+1;
- Owned<IPropertyTreeIterator> mapIter = checkMapIterator(lhsStart, *child);
- if (mapIter)
- {
- xxpath = lhsStart;
- iter.swap(mapIter);
- break;
- }
- }
- if (wild)
- iter.setown(new PTIdMatchIterator(this, id, isnocase(), flags & iptiter_sort));
- else
- iter.setown(child->getElements(NULL));
- StringAttr qualifier(start, xxpath-start);
- iter.setown(new PTStackIterator(iter.getClear(), qualifier.get()));
- break;
- }
- }
- xpath = xxpath;
- }
- }
- else
- {
- if (wild)
- iter.setown(new PTIdMatchIterator(this, id, isnocase(), flags & iptiter_sort));
- else if (child)
- iter.setown(child->getElements(NULL));
- }
- }
- }
- break;
- }
- }
- if (!iter)
- iter.setown(LINK(nullPTreeIterator));
- if (*xpath == '\0' || (*xpath == '/' && '\0' == *(xpath+1)))
- return iter.getClear();
- else
- return new PTStackIterator(iter.getClear(), xpath);
- }
- void PTree::localizeElements(const char *xpath, bool allTail)
- {
- // null action for local ptree
- }
- unsigned PTree::numChildren() const
- {
- if (!checkChildren()) return 0;
- return children->numChildren();
- }
- unsigned PTree::getCount(const char *xpath) const
- {
- unsigned c=0;
- Owned<IPropertyTreeIterator> iter = getElements(xpath);
- ForEach(*iter)
- ++c;
- return c;
- }
- void getXPathMatchTree(IPropertyTree &parentContext, const char *xpath, IPropertyTree *&matchContainer)
- {
- if (!xpath || !*xpath)
- {
- matchContainer = createPTree(parentContext.queryName());
- return;
- }
- StringBuffer head;
- const char *str = xpath;
- const char *end = str+strlen(xpath);
- bool quote = false;
- bool inQualifier = false;
- bool done = false;
- bool recurse = false;
- while (end != str)
- {
- switch (*str) {
- case '"':
- if (quote) quote = false;
- else quote = true;
- break;
- case '[':
- if (inQualifier)
- {
- if (!quote)
- throw MakeXPathException(xpath, PTreeExcpt_XPath_ParseError, str-xpath, "Unclosed qualifier detected");
- }
- else
- inQualifier = true;
- break;
- case ']':
- if (inQualifier)
- {
- if (!quote)
- inQualifier = false;
- }
- else if (!quote)
- throw MakeXPathException(xpath, PTreeExcpt_XPath_ParseError, str-xpath, "Unopened qualifier detected");
- break;
- case '/':
- if (!quote && !inQualifier)
- {
- if ('/' == *(str+1))
- recurse = true;
- done = true;
- }
- break;
- }
- if (done) break;
- ++str;
- }
- const char *tail;
- if (str==end) // top-level matches
- {
- head.append(xpath);
- if (0 == head.length())
- {
- matchContainer = createPTree(xpath);
- return;
- }
- tail = NULL;
- }
- else
- {
- head.append(str-xpath, xpath);
- if (recurse)
- tail = str+2;
- else
- tail = str+1;
- }
- Owned<IPropertyTreeIterator> parentIter = parentContext.getElements(head.str());
- Owned<IPropertyTree> matchParent;
- ForEach (*parentIter)
- {
- IPropertyTree &parent = parentIter->query();
- if (!matchParent)
- matchParent.setown(createPTree(parentContext.queryName()));
- if (tail && *tail)
- {
- IPropertyTree *childContainer = NULL;
- getXPathMatchTree(parent, tail, childContainer);
- if (childContainer)
- {
- if (!head.length())
- matchParent.setown(childContainer);
- else
- {
- unsigned pos = ((PTree &)parentContext).findChild(&parent);
- matchParent->addPropTree(childContainer->queryName(), childContainer);
- childContainer->setPropInt("@pos", pos+1);
- }
- if (!matchContainer)
- matchContainer = LINK(matchParent);
- }
- if (recurse)
- {
- Owned<IPropertyTreeIterator> iter = parent.getElements("*");
- ForEach (*iter)
- {
- IPropertyTree *childContainer = NULL;
- IPropertyTree &child = iter->query();
- getXPathMatchTree(child, xpath, childContainer);
- if (childContainer)
- {
- unsigned pos = ((PTree &)parent).findChild(&child);
- matchParent->addPropTree(childContainer->queryName(), childContainer);
- childContainer->setPropInt("@pos", pos+1);
- if (!matchContainer)
- matchContainer = LINK(matchParent);
- }
- }
- }
- }
- else
- {
- if (&parent != &parentContext)
- {
- IPropertyTree *childContainer = matchParent->addPropTree(parent.queryName(), createPTree());
- unsigned pos = ((PTree &)parentContext).findChild(&parent);
- childContainer->setPropInt("@pos", pos+1);
- }
- if (!matchContainer)
- matchContainer = LINK(matchParent);
- }
- }
- }
- IPropertyTree *getXPathMatchTree(IPropertyTree &parent, const char *xpath)
- {
- IPropertyTree *matchTree = NULL;
- getXPathMatchTree(parent, xpath, matchTree);
- return matchTree;
- }
- void PTree::serializeAttributes(MemoryBuffer &tgt)
- {
- IAttributeIterator *aIter = getAttributes();
- if (aIter->first())
- {
- do
- {
- tgt.append(aIter->queryName());
- tgt.append(aIter->queryValue());
- }
- while (aIter->next());
- }
- tgt.append(""); // attribute terminator. i.e. blank attr name.
- aIter->Release();
- }
- void PTree::serializeSelf(MemoryBuffer &tgt)
- {
- const char *_name = queryName();
- tgt.append(_name ? _name : "");
- tgt.append(flags);
- serializeAttributes(tgt);
- if (value)
- value->serialize(tgt);
- else
- tgt.append((size32_t)0);
- }
- void PTree::serializeCutOff(MemoryBuffer &tgt, int cutoff, int depth)
- {
- serializeSelf(tgt);
- if (-1 == cutoff || depth<cutoff)
- {
- Owned<IPropertyTreeIterator> iter = getElements("*");
- if (iter->first())
- {
- do
- {
- IPropertyTree *_child = &iter->query();
- PTree *child = QUERYINTERFACE(_child, PTree); assertex(child);
- child->serializeCutOff(tgt, cutoff, depth+1);
- }
- while (iter->next());
- }
- }
- tgt.append(""); // element terminator. i.e. blank child name.
- }
- // serializable impl.
- void PTree::serialize(MemoryBuffer &tgt)
- {
- serializeCutOff(tgt, -1, 0);
- }
- void PTree::deserialize(MemoryBuffer &src)
- {
- deserializeSelf(src);
- StringAttr eName;
- for (;;)
- {
- size32_t pos = src.getPos();
- src.read(eName);
- if (eName.isEmpty())
- break;
- src.reset(pos); // reset to re-read tree name
- IPropertyTree *child = create(src);
- addPropTree(eName, child);
- }
- }
- void PTree::deserializeSelf(MemoryBuffer &src)
- {
- setName(NULL); // needs to be cleared before flags changed
- StringAttr _name;
- src.read(_name);
- src.read(flags);
- if (_name[0]==0)
- setName(NULL);
- else
- setName(_name);
- StringAttr attrName, attrValue;
- for (;;)
- {
- src.read(attrName);
- if (attrName.isEmpty())
- break;
- src.read(attrValue);
- setProp(attrName, attrValue);
- }
- size32_t size;
- unsigned pos = src.getPos();
- src.read(size);
- if (value) delete value;
- if (size)
- {
- src.reset(pos);
- value = new CPTValue(src);
- }
- else value = NULL;
- }
- IPropertyTree *PTree::clone(IPropertyTree &srcTree, bool self, bool sub)
- {
- IPropertyTree *_dstTree = self ? this : create(srcTree.queryName());
- PTree *dstTree = QUERYINTERFACE(_dstTree, PTree);
- dbgassertex(dstTree);
- if (self)
- dstTree->setName(srcTree.queryName());
- clone(srcTree, *dstTree, sub);
- return _dstTree;
- }
- void PTree::clone(IPropertyTree &srcTree, IPropertyTree &dstTree, bool sub)
- {
- PTree *_dstTree = QUERYINTERFACE((&dstTree), PTree); assertex(_dstTree); //JCSMORE
- flags = _dstTree->flags;
- if (srcTree.isBinary(NULL))
- {
- MemoryBuffer mb;
- verifyex(srcTree.getPropBin(NULL, mb));
- dstTree.setPropBin(NULL, mb.length(), mb.toByteArray());
- }
- else if (srcTree.isCompressed(NULL))
- {
- StringBuffer s;
- verifyex(srcTree.getProp(NULL, s));
- dstTree.setProp(NULL, s.str());
- }
- else
- dstTree.setProp(NULL, srcTree.queryProp(NULL));
- IAttributeIterator *attrs = srcTree.getAttributes();
- if (attrs->first())
- {
- do
- {
- dstTree.setProp(attrs->queryName(), attrs->queryValue());
- }
- while (attrs->next());
- }
- attrs->Release();
- if (sub)
- {
- Owned<IPropertyTreeIterator> iter = srcTree.getElements("*");
- if (iter->first())
- {
- do
- {
- IPropertyTree &child = iter->query();
- IPropertyTree *newChild = clone(child, false, sub);
- dstTree.addPropTree(newChild->queryName(), newChild);
- }
- while (iter->next());
- }
- }
- }
- IPropertyTree *PTree::ownPTree(IPropertyTree *tree)
- {
- if (!isEquivalent(tree) || tree->IsShared() || isCaseInsensitive() != tree->isCaseInsensitive())
- {
- IPropertyTree *newTree = clone(*tree);
- tree->Release();
- return newTree;
- }
- else
- return tree;
- }
- IPropertyTree *PTree::queryCreateBranch(IPropertyTree *branch, const char *prop, bool *newBranch)
- {
- IPropertyTree *childBranch = branch->queryPropTree(prop);
- if (!childBranch)
- {
- if (newBranch) *newBranch = true;
- childBranch = create(prop);
- branch->setPropTree(prop, childBranch);
- }
- else if (newBranch) *newBranch = false;
- return childBranch;
- }
- IPropertyTree *PTree::splitBranchProp(const char *xpath, const char *&prop, bool error)
- {
- prop = splitXPathX(xpath);
- MAKE_LSTRING(path, xpath, prop-xpath);
- IPropertyTree *branch = queryPropTree(path);
- if (!branch && error)
- throw MakeIPTException(-1, "path %s not found, when setting prop %s", path, xpath);
- return branch;
- }
- IPropertyTree *_createPropBranch(IPropertyTree *tree, const char *xpath, bool createIntermediates, IPropertyTree *&created, IPropertyTree *&createdParent)
- {
- const char *prop;
- StringBuffer path;
- prop = splitXPathUQ(xpath, path);
- IPropertyTree *branch = tree->queryPropTree(path.str());
- if (!branch)
- {
- if (path.length() == strlen(xpath))
- throw MakeIPTException(-1, "createPropBranch: cannot create path : %s", xpath);
- if (!createIntermediates)
- throw MakeIPTException(-1, "createPropBranch: no path found for : %s", path.str());
-
- if ('/' == path.charAt(path.length()-1))
- path.remove(path.length()-1, 1);
- branch = _createPropBranch(tree, path.str(), createIntermediates, created, createdParent);
- assertex(branch);
- }
- if (prop && '\0' != *prop && '@' != *prop)
- {
- IPropertyTree *_branch = branch->queryPropTree(prop);
- if (_branch)
- branch = _branch;
- else
- {
- IPropertyTree *p = branch;
- branch = branch->addPropTree(prop, createPTree());
- if (!created) { created = branch; createdParent = p; }
- }
- }
- return branch;
- }
- IPropertyTree *createPropBranch(IPropertyTree *tree, const char *xpath, bool createIntermediates, IPropertyTree **created, IPropertyTree **createdParent)
- {
- IPropertyTree *_created = NULL, *_createdParent = NULL;
- try
- {
- IPropertyTree *ret = _createPropBranch(tree, xpath, createIntermediates, _created, _createdParent);
- if (created) *created = _created;
- if (createdParent) *createdParent = _createdParent;
- return ret;
- }
- catch (...)
- {
- if (_created) (_createdParent)->removeTree(_created);
- throw;
- }
- }
- void PTree::addLocal(size32_t l, const void *data, bool _binary, int pos)
- {
- if (!l) return; // right thing to do on addProp("x", NULL) ?
- IPTArrayValue *newValue = new CPTValue(l, data, _binary);
- Owned<IPropertyTree> tree = create(queryName(), newValue);
- PTree *_tree = QUERYINTERFACE(tree.get(), PTree); assertex(_tree);
- if (_binary)
- IptFlagSet(_tree->flags, ipt_binary);
- else
- IptFlagClr(_tree->flags, ipt_binary);
- addingNewElement(*tree, pos);
- IPTArrayValue *array;
- if (value && value->isArray())
- {
- array = value;
- if (pos != -1 && ((unsigned)pos > array->elements()))
- throw MakeIPTException(-1, "Error trying to insert element at %d of %d", pos, array->elements());
- }
- else
- {
- if (pos > 0)
- throw MakeIPTException(-1, "Error trying to insert element at %d of 0", pos);
- // detach children and attributes of this branch now owned by element of newly created array.
- IPropertyTree *element1 = detach();
- array = new CPTArray();
- addingNewElement(*element1, ANE_APPEND);
- static_cast<PTree *>(element1)->setOwner(array);
- array->addElement(element1);
- value = array;
- }
- _tree->setOwner(array);
- tree->Link();
- if (-1 == pos)
- array->addElement(tree);
- else
- array->setElement(pos, tree);
- }
- enum exprType { t_none, t_equality, t_inequality, t_lteq, t_lt, t_gt, t_gteq } tType;
- inline bool match(bool wild, bool numeric, const char *xpath, exprType t, const char *value, unsigned len, const char *pat, unsigned patLen, bool nocase)
- {
- int m;
- if (numeric)
- {
- __int64 lhsN = atoi64_l(value, len);
- __int64 rhsN = atoi64_l(pat, patLen);
- m = lhsN<rhsN?-1:lhsN>rhsN?1:0;
- }
- else if (wild)
- m = false==WildMatch(value, len, pat, patLen, nocase);
- else
- {
- if (len == patLen)
- m = nocase ? memicmp(value, pat, len) : memcmp(value, pat, len);
- else if (len < patLen)
- m = -1;
- else
- m = 1;
- }
- switch (t)
- {
- case t_inequality:
- return m!=0;
- case t_lt:
- return m<0;
- case t_lteq:
- return m<=0;
- case t_equality:
- return m==0;
- case t_gteq:
- return m>=0;
- case t_gt:
- return m>0;
- }
- throw MakeXPathException(xpath, PTreeExcpt_XPath_ParseError, 0, "Invalid xpath qualifier expression in xpath: %s", xpath);
- }
- bool PTree::checkPattern(const char *&xxpath) const
- {
- // Pattern is an additional filter at the current node level
- // It can be [condition], or it can be empty (we don't support anything else)
- // supported conditions are:
- // tag - must have child called tag
- // @attr - must have attribute called attr
- // tag="value" - must have child called tag with given value
- // @attr="value" - must have attribute called attr with given value
- const char *xpath = xxpath;
- while (*xpath == ' ' || *xpath == '\t') xpath++;
- const char *start = xpath;
- bool wild = false, nocase = isnocase();
- if (*xpath=='@')
- xpath++;
- char quote = 0;
- const char *lhsEnd, *quoteBegin, *quoteEnd, *rhsBegin, *rhsEnd;
- lhsEnd = quoteBegin = quoteEnd = rhsBegin = rhsEnd = NULL;
- exprType tType = t_none;
- bool numeric=false;
- #ifdef WARNLEGACYCOMPARE
- bool legacynumeric=false;
- #endif
- for (;;)
- {
- switch (*xpath) {
- case '"':
- case '\'':
- if (quote)
- {
- if (*xpath == quote)
- {
- quote = 0;
- quoteEnd = xpath;
- }
- }
- else
- {
- if (quoteBegin)
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Quoted left hand side already seen");
- quote = *xpath;
- quoteBegin = xpath+1;
- }
- break;
- case '[':
- if (!quote)
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Unclosed qualifier detected");
- break;
- case ']':
- if (!quote)
- {
- if (!lhsEnd)
- lhsEnd = xpath;
- rhsEnd = xpath;
- }
- break;
- case ' ':
- case '\t':
- if (!lhsEnd)
- lhsEnd = xpath;
- break;
- case '!':
- if (!quote)
- {
- if (tType)
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Unexpected expression operator xpath");
- if ('=' != *(xpath+1))
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Invalid xpath qualifier expression in xpath");
- if (!lhsEnd)
- lhsEnd = xpath;
- ++xpath;
- tType = t_inequality;
- wild = true; // true by default now, introduced ~ syntax, to denote wild string
- }
- break;
- case '=':
- if (!quote)
- {
- if (wild)
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Wildcard match '~' makes no sense in this context");
- if (tType)
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Unexpected expression operator xpath");
- tType = t_equality;
- wild = true; // true by default now, introduced ~ syntax, to denote wild string
- if (!lhsEnd)
- lhsEnd = xpath;
- }
- break;
- case '>':
- if (!quote)
- {
- if (wild)
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Wildcard match '~' makes no sense in this context");
- if (tType)
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Unexpected expression operator in xpath");
- if (!lhsEnd)
- lhsEnd = xpath;
- #ifdef WARNLEGACYCOMPARE
- legacynumeric = true;
- #endif
- if ('=' == *(xpath+1))
- {
- ++xpath;
- tType = t_gteq;
- }
- else
- tType = t_gt;
- }
- break;
- case '<':
- if (!quote)
- {
- if (tType)
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Unexpected expression operator in xpath");
- if (!lhsEnd)
- lhsEnd = xpath;
- #ifdef WARNLEGACYCOMPARE
- legacynumeric = true;
- #endif
- if ('=' == *(xpath+1))
- {
- ++xpath;
- tType = t_lteq;
- }
- else
- tType = t_lt;
- }
- break;
- case '~':
- if (!quote)
- {
- if (!tType)
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Unexpected wild operator in xpath");
- wild = true;
- }
- break;
- case '?':
- if (!quote)
- {
- if (!tType)
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Unexpected case-insensitive operator in xpath");
- nocase = true;
- }
- break;
- case '\0':
- rhsEnd = xpath;
- break;
- }
- if (rhsEnd)
- break;
- xpath++;
- if (!rhsBegin && tType && !isspace(*xpath))
- rhsBegin = xpath;
- }
- if (quote)
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Parse error, unclosed quoted content");
- if (tType)
- {
- if (quoteBegin && !quoteEnd)
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Parse error, RHS missing closing quote");
- if (rhsBegin && !rhsEnd)
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Parse error, RHS missing closing quote");
- if (!quoteBegin && rhsEnd) // validate it's a numeric
- {
- const char *c = rhsBegin;
- for (;;)
- {
- if (!isdigit(*c++))
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Parse error, RHS is an unquoted string");
- if (c==rhsEnd) break;
- }
- }
- }
- MAKE_LSTRING(lhs, start, lhsEnd-start);
- bool ret = false;
- const char *tProp = splitXPathX(lhs);
- MAKE_LSTRING(head, lhs, tProp-lhs);
- Owned<IPropertyTreeIterator> iter = getElements(head);
- ForEach (*iter)
- {
- IPropertyTree &found = iter->query();
- if (t_none == tType)
- {
- if (found.hasProp(tProp))
- {
- ret = true;
- break;
- }
- }
- else
- {
- Owned<IPropertyTreeIterator> _iter2;
- IPropertyTreeIterator *iter2;
- IPropertyTree *matchElem;
- if (isAttribute(tProp))
- {
- matchElem = &found;
- iter2 = NULL;
- }
- else
- {
- _iter2.setown(found.getElements(tProp));
- iter2 = _iter2;
- if (iter2->first())
- matchElem = &iter2->query();
- else
- continue;
- tProp = NULL;
- }
- for (;;)
- {
- if (matchElem->isBinary(tProp))
- UNIMPLEMENTED;
- const char *rhs;
- unsigned rhslength;
- if (quoteEnd)
- {
- rhs = quoteBegin;
- rhslength = quoteEnd-quoteBegin;
- #ifdef WARNLEGACYCOMPARE
- if (legacynumeric)
- {
- if (isdigit(*rhs))
- IWARNLOG("Possible deprecated use of quoted numeric comparison operation: %s", xxpath);
- }
- #endif
- }
- else if (rhsEnd)
- {
- rhs = rhsBegin;
- rhslength = rhsEnd-rhsBegin;
- numeric = true;
- }
- else
- {
- rhs = NULL;
- rhslength = 0;
- }
- if (matchElem->isCompressed(tProp))
- {
- StringBuffer s;
- matchElem->getProp(tProp, s);
- ret = match(wild, numeric, xxpath, tType, s.str(), s.length(), rhs, rhslength, nocase);
- }
- else
- {
- const char *value = matchElem->queryProp(tProp);
- if (value)
- ret = match(wild, numeric, xxpath, tType, value, value?(size32_t)strlen(value):0, rhs, rhslength, nocase);
- else if (tType == t_equality)
- ret = (NULL == rhs || '\0' == *rhs);
- else if (tType == t_inequality)
- ret = (NULL != rhs && '\0' != *rhs);
- }
- if (ret)
- break;
- if (!iter2 || !iter2->next())
- break;
- matchElem = &iter2->query();
- }
- if (ret)
- break;
- }
- }
- xxpath = xpath;
- return ret;
- }
- AttrValue *PTree::findAttribute(const char *key) const
- {
- if (attrs)
- {
- AttrValue *a = attrs+numAttrs;
- if (isnocase())
- {
- while (a-- != attrs)
- {
- if (strieq(a->key.get(), key))
- return a;
- }
- }
- else
- {
- while (a-- != attrs)
- {
- if (streq(a->key.get(), key))
- return a;
- }
- }
- }
- return nullptr;
- }
- const char *PTree::getAttributeValue(const char *key) const
- {
- AttrValue *e = findAttribute(key);
- if (e)
- return e->value.get();
- return nullptr;
- }
- unsigned PTree::getAttributeCount() const
- {
- return numAttrs;
- }
- AttrValue *PTree::getNextAttribute(AttrValue *cur) const
- {
- if (0 == numAttrs)
- return nullptr;
- else if (nullptr == cur)
- return attrs;
- else
- {
- if (cur == (attrs+(numAttrs-1)))
- return nullptr;
- return ++cur;
- }
- }
- //////////////////////
- // LocalPTree
- static RelaxedAtomic<unsigned> numLocalTrees;
- unsigned queryNumLocalTrees()
- {
- return numLocalTrees;
- }
- LocalPTree::LocalPTree(const char *_name, byte _flags, IPTArrayValue *_value, ChildMap *_children) : PTree(_flags|ipt_fast, _value, _children)
- {
- if (_name)
- setName(_name);
- numLocalTrees++;
- }
- LocalPTree::~LocalPTree()
- {
- numLocalTrees--;
- name.destroy();
- if (!attrs)
- return;
- AttrValue *a = attrs+numAttrs;
- while (a--!=attrs)
- {
- a->key.destroy();
- a->value.destroy();
- }
- free(attrs);
- }
- const char *LocalPTree::queryName() const
- {
- return name.get();
- }
- void LocalPTree::setName(const char *_name)
- {
- if (_name==name.get())
- return;
- AttrStr *oname = name.getPtr(); // Don't free until after we copy - they could overlap
- if (!name.set(_name))
- name.setPtr(AttrStr::create(_name));
- if (oname)
- AttrStr::destroy(oname);
- }
- bool LocalPTree::removeAttribute(const char *key)
- {
- AttrValue *del = findAttribute(key);
- if (!del)
- return false;
- if (arrayOwner)
- {
- CQualifierMap *map = arrayOwner->queryMap();
- if (map)
- map->removeEntryIfMapped(key, del->value.get(), this);
- }
- numAttrs--;
- unsigned pos = del-attrs;
- del->key.destroy();
- del->value.destroy();
- memmove(attrs+pos, attrs+pos+1, (numAttrs-pos)*sizeof(AttrValue));
- return true;
- }
- void LocalPTree::setAttribute(const char *key, const char *val)
- {
- if (!key)
- return;
- if (!validateXMLTag(key+1))
- throw MakeIPTException(-1, "Invalid xml attribute: %s", key);
- if (!val)
- val = ""; // cannot have NULL value
- AttrValue *v = findAttribute(key);
- AttrStr *goer = nullptr;
- if (v)
- {
- if (streq(v->value.get(), val))
- return;
- goer = v->value.getPtr();
- }
- else
- {
- attrs = (AttrValue *)realloc(attrs, (numAttrs+1)*sizeof(AttrValue));
- v = new(&attrs[numAttrs++]) AttrValue; // Initialize new AttrValue
- if (!v->key.set(key))
- v->key.setPtr(isnocase() ? AttrStr::createNC(key) : AttrStr::create(key));
- }
- if (arrayOwner)
- {
- CQualifierMap *map = arrayOwner->queryMap();
- if (map)
- {
- if (goer)
- map->replaceEntryIfMapped(key, v->value.get(), val, this);
- else
- map->insertEntryIfMapped(key, val, this);
- }
- }
- if (!v->value.set(val))
- v->value.setPtr(AttrStr::create(val));
- if (goer)
- AttrStr::destroy(goer);
- }
- #ifdef TRACE_STRING_SIZE
- std::atomic<__int64> AttrStr::totsize { 0 };
- std::atomic<__int64> AttrStr::maxsize { 0 };
- #endif
- #ifdef TRACE_ATOM_SIZE
- std::atomic<__int64> AttrStrAtom::totsize { 0 };
- std::atomic<__int64> AttrStrAtom::maxsize { 0 };
- #endif
- ///////////////////
- static RelaxedAtomic<unsigned> numAtomTrees;
- unsigned queryNumAtomTrees()
- {
- return numAtomTrees;
- }
- CAtomPTree::CAtomPTree(const char *_name, byte _flags, IPTArrayValue *_value, ChildMap *_children) : PTree(_flags|ipt_lowmem, _value, _children)
- {
- numAtomTrees++;
- if (_name)
- setName(_name);
- }
- CAtomPTree::~CAtomPTree()
- {
- numAtomTrees--;
- bool nc = isnocase();
- HashKeyElement *name_ptr = name.getPtr();
- if (name_ptr)
- {
- AtomRefTable *kT = nc?keyTableNC:keyTable;
- #ifdef TRACE_ATOM_SIZE
- size_t gosize = sizeof(HashKeyElement)+strlen(name_ptr->get())+1;
- if (kT->releaseKey(name_ptr))
- AttrStrAtom::totsize -= gosize;
- #else
- kT->releaseKey(name_ptr);
- #endif
- }
- if (!attrs)
- return;
- AttrValue *a = attrs+numAttrs;
- {
- CriticalBlock block(hashcrit);
- while (a--!=attrs)
- {
- if (a->key.isPtr())
- attrHT->removekey(a->key.getPtr(), nc);
- if (a->value.isPtr())
- attrHT->removeval(a->value.getPtr());
- }
- freeAttrArray(attrs, numAttrs);
- }
- }
- void CAtomPTree::setName(const char *_name)
- {
- AtomRefTable *kT = isnocase()?keyTableNC:keyTable;
- HashKeyElement *oname = name.getPtr(); // NOTE - don't release yet as could overlap source name
- if (!_name)
- name.setPtr(nullptr);
- else
- {
- if (!validateXMLTag(_name))
- throw MakeIPTException(PTreeExcpt_InvalidTagName, ": %s", _name);
- if (!name.set(_name))
- {
- #ifdef TRACE_ALL_ATOM
- DBGLOG("TRACE_ALL_ATOM: %s", _name);
- #endif
- #ifdef TRACE_ATOM_SIZE
- bool didCreate;
- name.setPtr(kT->queryCreate(_name, didCreate));
- if (didCreate)
- {
- AttrStrAtom::totsize += sizeof(HashKeyElement)+strlen(_name)+1;
- if (AttrStrAtom::totsize > AttrStrAtom::maxsize)
- {
- AttrStrAtom::maxsize.store(AttrStrAtom::totsize);
- DBGLOG("TRACE_ATOM_SIZE: total size now %" I64F "d", AttrStrAtom::maxsize.load());
- }
- }
- #else
- name.setPtr(kT->queryCreate(_name));
- #endif
- }
- }
- if (oname)
- {
- #ifdef TRACE_ATOM_SIZE
- size_t gosize = sizeof(HashKeyElement)+strlen(oname->get())+1;
- if (kT->releaseKey(oname))
- AttrStrAtom::totsize -= gosize;
- #else
- kT->releaseKey(oname);
- #endif
- }
- }
- const char *CAtomPTree::queryName() const
- {
- return name.get();
- }
- unsigned CAtomPTree::queryHash() const
- {
- if (name.isPtr())
- {
- assert(name.getPtr());
- return name.getPtr()->queryHash();
- }
- else
- {
- const char *_name = name.get();
- size32_t nl = strlen(_name);
- return isnocase() ? hashnc((const byte *) _name, nl, 0): hashc((const byte *) _name, nl, 0);
- }
- }
- AttrValue *CAtomPTree::newAttrArray(unsigned n)
- {
- // NB crit must be locked
- if (!n)
- return nullptr;
- if (freelistmax<=n)
- {
- freelist = (AttrValue **)realloc(freelist, sizeof(AttrValue *)*(n+1));
- while (freelistmax<=n)
- freelist[freelistmax++] = nullptr;
- }
- AttrValue *&p = freelist[n];
- AttrValue *ret = p;
- if (ret)
- p = *(AttrValue **)ret;
- else
- ret = (AttrValue *)freeallocator.alloc(sizeof(AttrValue)*n);
- return ret;
- }
- void CAtomPTree::freeAttrArray(AttrValue *a, unsigned n)
- {
- // NB crit must be locked
- if (a)
- {
- AttrValue *&p = freelist[n];
- *(AttrValue **)a = p;
- p = a;
- }
- }
- void CAtomPTree::setAttribute(const char *key, const char *val)
- {
- if (!key)
- return;
- if (!validateXMLTag(key+1))
- throw MakeIPTException(-1, "Invalid xml attribute: %s", key);
- if (!val)
- val = ""; // cannot have NULL value
- AttrValue *v = findAttribute(key);
- if (v)
- {
- if (streq(v->value.get(), val))
- return;
- if (arrayOwner)
- {
- CQualifierMap *map = arrayOwner->queryMap();
- if (map)
- map->replaceEntryIfMapped(key, v->value.get(), val, this);
- }
- AttrStr * goer = v->value.getPtr();
- if (!v->value.set(val))
- {
- CriticalBlock block(hashcrit);
- if (goer)
- attrHT->removeval(goer);
- v->value.setPtr(attrHT->addval(val));
- }
- else if (goer)
- {
- CriticalBlock block(hashcrit);
- attrHT->removeval(goer);
- }
- }
- else
- {
- CriticalBlock block(hashcrit);
- AttrValue *newattrs = newAttrArray(numAttrs+1);
- if (attrs)
- {
- memcpy(newattrs, attrs, numAttrs*sizeof(AttrValue));
- freeAttrArray(attrs, numAttrs);
- }
- if (arrayOwner)
- {
- CQualifierMap *map = arrayOwner->queryMap();
- if (map)
- map->insertEntryIfMapped(key, val, this);
- }
- v = &newattrs[numAttrs];
- if (!v->key.set(key))
- v->key.setPtr(attrHT->addkey(key, isnocase()));
- if (!v->value.set(val))
- v->value.setPtr(attrHT->addval(val));
- numAttrs++;
- attrs = newattrs;
- }
- }
- bool CAtomPTree::removeAttribute(const char *key)
- {
- AttrValue *del = findAttribute(key);
- if (!del)
- return false;
- numAttrs--;
- if (arrayOwner)
- {
- CQualifierMap *map = arrayOwner->queryMap();
- if (map)
- map->removeEntryIfMapped(key, del->value.get(), this);
- }
- CriticalBlock block(hashcrit);
- if (del->key.isPtr())
- attrHT->removekey(del->key.getPtr(), isnocase());
- if (del->value.isPtr())
- attrHT->removeval(del->value.getPtr());
- AttrValue *newattrs = newAttrArray(numAttrs);
- if (newattrs)
- {
- unsigned pos = del-attrs;
- memcpy(newattrs, attrs, pos*sizeof(AttrValue));
- memcpy(newattrs+pos, attrs+pos+1, (numAttrs-pos)*sizeof(AttrValue));
- }
- freeAttrArray(attrs, numAttrs+1);
- attrs = newattrs;
- return true;
- }
- ///////////////////
- bool isEmptyPTree(const IPropertyTree *t)
- {
- if (!t)
- return true;
- if (t->numUniq())
- return false;
- Owned<IAttributeIterator> ai = t->getAttributes();
- if (ai->first())
- return false;
- const char *s = t->queryProp(NULL);
- if (s&&*s)
- return false;
- return true;
- }
- ///////////////////
- PTLocalIteratorBase::PTLocalIteratorBase(const PTree *_tree, const char *_id, bool _nocase, bool _sort) : nocase(_nocase), sort(_sort), id(_id), tree(_tree)
- {
- class CPTArrayIterator : public ArrayIIteratorOf<IArrayOf<IPropertyTree>, IPropertyTree, IPropertyTreeIterator>
- {
- public:
- CPTArrayIterator(IPropertyTreeIterator &src) : ArrayIIteratorOf<IArrayOf<IPropertyTree>, IPropertyTree, IPropertyTreeIterator>(elems)
- {
- ForEach(src)
- elems.append(src.get());
- elems.sort(comparePropTrees);
- }
- IArrayOf<IPropertyTree> elems;
- };
- tree->Link();
- baseIter = tree->checkChildren()->getIterator(sort);
- iter = NULL;
- current = NULL;
- }
- PTLocalIteratorBase::~PTLocalIteratorBase()
- {
- baseIter->Release();
- ::Release(iter);
- tree->Release();
- }
- // IPropertyTreeIterator
- bool PTLocalIteratorBase::first()
- {
- ::Release(iter); iter=NULL;
- if (!baseIter || !baseIter->first()) return false;
- return _next();
- }
- bool PTLocalIteratorBase::_next()
- {
- if (iter && iter->isValid() && iter->next())
- return true;
- for (;;)
- {
- for (;;)
- {
- if (!baseIter->isValid())
- {
- current = NULL;
- return false;
- }
- else if (match())
- break;
- baseIter->next();
- }
- IPropertyTree *element = &baseIter->query();
- baseIter->next();
- if (iter)
- iter->Release();
- iter = element->getElements(NULL);
- if (iter->first())
- {
- current = &iter->query();
- return true;
- }
- }
- }
- bool PTLocalIteratorBase::next()
- {
- return _next();
- }
- bool PTLocalIteratorBase::isValid()
- {
- return (current != NULL);
- }
- /////////////////////////////
- bool PTIdMatchIterator::match()
- {
- IPropertyTree &tree = baseIter->query();
- const char *key = tree.queryName();
- return (0 != WildMatch(key, id, nocase));
- }
- ////////////////////////////
- SingleIdIterator::SingleIdIterator(const PTree &_tree, unsigned pos, unsigned _many) : many(_many), count(0), whichNext(pos-1), start(pos-1), current(NULL), tree(_tree)
- {
- tree.Link();
- }
- SingleIdIterator::~SingleIdIterator()
- {
- tree.Release();
- }
- void SingleIdIterator::setCurrent(unsigned pos)
- {
- current = tree.value->queryElement(pos);
- }
- // IInterface impl.
- bool SingleIdIterator::first()
- {
- whichNext = start;
- if (!tree.value || !tree.value->isArray())
- {
- if (0 == whichNext)
- {
- current = const_cast<PTree*>(&tree);
- count = 1;
- }
- }
- else
- {
- count = tree.value->elements();
- if (whichNext < count)
- setCurrent(whichNext);
- else
- return false;
- }
- ++whichNext;
- return true;
- }
- bool SingleIdIterator::next()
- {
- if ((whichNext>=count) || ((unsigned) -1 != many && whichNext>start+many))
- {
- current = NULL;
- return false;
- }
- setCurrent(whichNext++);
- return true;
- }
- bool SingleIdIterator::isValid()
- {
- return (NULL != current);
- }
- //////////////
- class StackElement
- {
- public:
- void init(IPropertyTreeIterator *_iter, const char *_xpath)
- {
- xpath = (char *)strdup(_xpath);
- iter=LINK(_iter);
- }
- void clear()
- {
- ::Release(iter);
- if (xpath)
- free(xpath);
- }
- IPropertyTreeIterator *get(StringAttr &str)
- {
- str.setown(xpath); return iter; // NB used in place of pop, as element invalid after call
- }
- IPropertyTreeIterator *iter;
- char * xpath;
- };
- ///////////////////
- PTStackIterator::PTStackIterator(IPropertyTreeIterator *_iter, const char *_xpath) : rootIter(_iter), xpath(_xpath)
- {
- iter = NULL;
- xxpath = "";
- current = NULL;
- stacklen = 0;
- stackmax = 4;
- stack = (StackElement *)malloc(sizeof(StackElement)*stackmax);
- }
- PTStackIterator::~PTStackIterator()
- {
- while (stacklen)
- stack[--stacklen].clear();
- ::Release(iter);
- ::Release(rootIter);
- free(stack);
- }
- void PTStackIterator::setIterator(IPropertyTreeIterator *_iter)
- {
- assertex(_iter);
- if (iter)
- iter->Release();
- iter = _iter;
- iter->first();
- }
- // IIterator impl.
- bool PTStackIterator::first()
- {
- while (stacklen)
- stack[--stacklen].clear();
- current = NULL;
- xxpath = xpath;
- rootIter->Link();
- setIterator(rootIter);
- return next();
- }
- bool PTStackIterator::isValid()
- {
- return (current != NULL);
- }
- IPropertyTree &PTStackIterator::query()
- {
- assertex(current);
- return *current;
- }
- bool PTStackIterator::next()
- {
- bool separator = false;
- if (iter)
- {
- IPropertyTree *element = NULL;
- StringBuffer qualifierText;
- for (;;)
- {
- while (!iter->isValid())
- {
- if (iter) iter->Release();
- iter = popFromStack(stackPath); // leaves linked
- if (!iter)
- {
- current = NULL;
- return false;
- }
- xxpath = stackPath;
- element = NULL;
- }
- if (!element)
- {
- element = &iter->query();
- iter->next();
- }
- while (element)
- {
- switch (*xxpath)
- {
- case '\0':
- current = element;
- return true;
- case '.':
- if (separator) throw MakeXPathException(xpath, PTreeExcpt_XPath_ParseError, 0, "Syntax error");
- separator=false;
- ++xxpath;
- if (*xpath && '/' != *xpath)
- throw MakeXPathException(xpath-1, PTreeExcpt_XPath_Unsupported, 0, "\"/\" expected");
- break;
- case '/':
- ++xxpath;
- if ('/' == *xxpath)
- {
- --xxpath;
- if (iter->isValid())
- pushToStack(iter, xxpath);
- setIterator(element->getElements(xxpath));
- xxpath = "";
- element = NULL;
- }
- separator=true;
- break;
- default:
- separator=false;
- if (iter->isValid())
- pushToStack(iter, xxpath);
- bool wild, numeric;
- const char *start = xxpath;
- readWildIdIndex(xxpath, wild, numeric);
- size32_t s = xxpath-start;
- if (s)
- {
- // NB: actually an id not qualifier, just sharing var.
- qualifierText.clear().append(s, start);
- bool mapped = false;
- if (!wild && !numeric)
- {
- ChildMap *children = ((PTree *)element)->checkChildren();
- if (children)
- {
- IPropertyTree *child = children->query(qualifierText);
- if (child)
- {
- if ('[' == *xxpath)
- {
- const char *newXXPath = xxpath+1;
- Owned<IPropertyTreeIterator> mapIter = checkMapIterator(newXXPath, *child);
- if (mapIter)
- {
- setIterator(mapIter.getClear());
- mapped = true;
- xxpath = newXXPath;
- }
- }
- }
- }
- }
- if (!mapped)
- setIterator(element->getElements(qualifierText));
- }
- else // must be qualifier.
- {
- if ('[' != *xxpath)
- throw MakeXPathException(xxpath, PTreeExcpt_XPath_ParseError, 0, "Qualifier expected e.g. [..]");
- const char *start = xxpath;
- char quote = 0;
- while (']' != *(++xxpath) || quote)
- {
- switch (*xxpath) {
- case '\"':
- case '\'':
- {
- if (quote)
- {
- if (*xxpath == quote)
- quote = 0;
- }
- else
- quote = *xxpath;
- break;
- }
- case '\0':
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xxpath-start, "Qualifier brace unclosed");
- }
- }
- ++xxpath;
- qualifierText.clear().append(xxpath-start, start);
- setIterator(element->getElements(qualifierText.str()));
- }
- element = NULL;
- break;
- }
- }
- }
- }
- return false;
- }
- void PTStackIterator::pushToStack(IPropertyTreeIterator *iter, const char *xpath)
- {
- if (stacklen==stackmax) {
- stackmax *= 2;
- stack = (StackElement *)realloc(stack, sizeof(StackElement)*stackmax);
- }
- stack[stacklen++].init(iter, xpath);
- }
- IPropertyTreeIterator *PTStackIterator::popFromStack(StringAttr &path)
- {
- if (!stacklen)
- return NULL;
- return stack[--stacklen].get(path);
- }
- #define DEFAULT_PTREE_TYPE LocalPTree
- // factory methods
- IPropertyTree *createPTree(MemoryBuffer &src, byte flags)
- {
- IPropertyTree *tree = createPTree(nullptr, flags);
- tree->deserialize(src);
- return tree;
- }
- IPropertyTree *createPTreeFromIPT(const IPropertyTree *srcTree, ipt_flags flags)
- {
- Owned<PTree> tree = (PTree *)createPTree(NULL, flags);
- return tree->clone(*srcTree->queryBranch(NULL));
- }
- void mergePTree(IPropertyTree *target, IPropertyTree *toMerge)
- {
- Owned<IAttributeIterator> aiter = toMerge->getAttributes();
- ForEach (*aiter)
- target->addProp(aiter->queryName(), aiter->queryValue());
- Owned<IPropertyTreeIterator> iter = toMerge->getElements("*");
- ForEach (*iter)
- {
- IPropertyTree &e = iter->query();
- target->addPropTree(e.queryName(), LINK(&e));
- }
- }
- void _synchronizePTree(IPropertyTree *target, const IPropertyTree *source, bool removeTargetsNotInSource)
- {
- Owned<IAttributeIterator> aiter = target->getAttributes();
- StringArray targetAttrs;
- if (removeTargetsNotInSource)
- {
- ForEach (*aiter)
- targetAttrs.append(aiter->queryName());
- }
- aiter.setown(source->getAttributes());
- ForEach (*aiter)
- {
- const char *attr = aiter->queryName();
- if (!target->hasProp(attr))
- target->setProp(attr, aiter->queryValue());
- else
- {
- const char *sValue = aiter->queryValue();
- const char *tValue = target->queryProp(attr);
- if (NULL == sValue)
- {
- if (NULL != tValue)
- target->setProp(attr, sValue);
- }
- else if (NULL == tValue ||0 != strcmp(sValue, tValue))
- target->setProp(attr, sValue);
- if (removeTargetsNotInSource)
- targetAttrs.zap(attr);
- }
- }
- if (removeTargetsNotInSource)
- {
- // remaining
- ForEachItemIn (a, targetAttrs)
- target->removeProp(targetAttrs.item(a));
- }
-
- bool equal = true;
- MemoryBuffer srcMb;
- const char *src = NULL;
- if (target->isBinary())
- {
- MemoryBuffer tgtMb;
- target->getPropBin(NULL, tgtMb);
- source->getPropBin(NULL, srcMb);
- if (tgtMb.length() != srcMb.length())
- equal = false;
- else if (0 != memcmp(tgtMb.toByteArray(), srcMb.toByteArray(), tgtMb.length()))
- equal = false;
- }
- else
- {
- const char *tgt = target->queryProp(NULL);
- src = source->queryProp(NULL);
- unsigned lTgt = tgt?(size32_t)strlen(tgt):0;
- unsigned lSrc = src?(size32_t)strlen(src):0;
- if (lTgt != lSrc)
- equal = false;
- else if (0 != lTgt && (0 != strcmp(tgt, src)))
- equal = false;
- }
- if (!equal)
- {
- if (target->isBinary())
- target->setPropBin(NULL, srcMb.length(), srcMb.toByteArray());
- else
- target->setProp(NULL, src);
- }
-
- ICopyArrayOf<IPropertyTree> toProcess;
- Owned<IPropertyTreeIterator> iter = source->getElements("*");
- ForEach (*iter)
- toProcess.append(iter->query());
- iter.setown(target->getElements("*"));
- ICopyArrayOf<IPropertyTree> removeTreeList;
- Owned<IPropertyTreeIterator> srcTypeIter;
- StringAttr firstOfType;
- ForEach (*iter)
- {
- IPropertyTree &e = iter->query();
- const char *name = e.queryName();
- IPropertyTree *sourceCompare;
- if (!source->hasProp(name))
- {
- removeTreeList.append(e);
- firstOfType.clear();
- srcTypeIter.clear();
- }
- else
- {
- if (firstOfType.isEmpty() || 0 != strcmp(firstOfType, e.queryName()))
- {
- if (firstOfType.length() && srcTypeIter)
- {
- // add remaining
- while (srcTypeIter->next())
- {
- sourceCompare = &srcTypeIter->query();
- target->addPropTree(sourceCompare->queryName(), LINK(sourceCompare));
- toProcess.zap(*sourceCompare);
- }
- }
- srcTypeIter.setown(source->getElements(e.queryName()));
- firstOfType.set(e.queryName());
- assertex(srcTypeIter->first());
- sourceCompare = &srcTypeIter->query();
- }
- else // 2nd of type etc..
- sourceCompare = srcTypeIter->next() ? &srcTypeIter->query() : NULL;
- if (sourceCompare)
- {
- toProcess.zap(*sourceCompare);
- _synchronizePTree(&e, sourceCompare, removeTargetsNotInSource);
- }
- else
- removeTreeList.append(e);
- }
- }
- if (removeTargetsNotInSource)
- {
- ForEachItemIn (rt, removeTreeList)
- target->removeTree(&removeTreeList.item(rt));
- }
- // add unprocessed source elements, not reference by name in target
- ForEachItemIn (s, toProcess)
- {
- IPropertyTree &e = toProcess.item(s);
- target->addPropTree(e.queryName(), LINK(&e));
- }
- }
- /* ensure target is equivalent to source whilst retaining elements already present in target.
- * presevers ordering of matching elements.
- * If removeTargetsNotInSource = true (default) elements in the target not present in the source will be removed
- */
- void synchronizePTree(IPropertyTree *target, const IPropertyTree *source, bool removeTargetsNotInSource, bool rootsMustMatch)
- {
- if (rootsMustMatch)
- {
- const char *srcName = source->queryName();
- const char *tgtName = target->queryName();
- if (0 != strcmp(srcName, tgtName))
- throw MakeIPTException(PTreeExcpt_Unsupported, "Cannot synchronize if root nodes mismatch");
- }
- _synchronizePTree(target, source, removeTargetsNotInSource);
- }
- IPropertyTree *ensurePTree(IPropertyTree *root, const char *xpath)
- {
- return createPropBranch(root, xpath, true);
- }
- IPTreeReadException *createPTreeReadException(int code, const char *msg, const char *context, unsigned line, offset_t offset)
- {
- //Do not use jlib_thrown_decl because it causes problems with VS2017 - I think because of beforeDispose() in CInterfaceOf.
- //The type of the object actually thrown is IPTreeReadException which does have a jlib_thrown_decl - so it will still be caught.
- class CPTreeReadException : implements CInterfaceOf<IPTreeReadException>
- {
- int code;
- StringAttr msg;
- StringAttr context;
- unsigned line;
- offset_t offset;
- StringBuffer &getErrorMessage(StringBuffer &out) const
- {
- switch (code)
- {
- case PTreeRead_EOS:
- return out.append("Error - end of stream");
- case PTreeRead_syntax:
- return out.append("Error - syntax error");
- }
- return out;
- }
- public:
- CPTreeReadException(int _code, const char *_msg, const char *_context, unsigned _line, offset_t _offset) : code(_code), msg(_msg), context(_context), line(_line), offset(_offset) { }
- // IException
- int errorCode() const { return code; }
- StringBuffer &errorMessage(StringBuffer &str) const
- {
- getErrorMessage(str);
- if (msg.length())
- str.append(" \"").append(msg).append("\"");
- str.append(" [");
- if (line>1) // don't bother with line 1, there may be no line breaks.
- str.append("line ").append(line).append(", ");
- str.append("file offset ").append(offset).append("]");
- if (context.length())
- str.newline().append(context);
- return str;
- }
- MessageAudience errorAudience() const { return MSGAUD_user; }
-
- const char *queryDescription() { return msg; }
- unsigned queryLine() { return line; }
- offset_t queryOffset() { return offset; }
- const char *queryContext() { return context.get(); }
- };
- return new CPTreeReadException(code, msg, context, line, offset);
- }
- template <typename T>
- class CommonReaderBase : public CInterface
- {
- Linked<ISimpleReadStream> lstream;
- ISimpleReadStream *stream;
- bool bufOwned, nullTerm;
- byte *buf, *bufPtr;
- size32_t bufSize, bufRemaining;
- protected:
- PTreeReaderOptions readerOptions;
- bool ignoreWhiteSpace, noRoot;
- Linked<IPTreeNotifyEvent> iEvent;
- offset_t curOffset;
- unsigned line;
- char nextChar;
- private:
- void init()
- {
- ignoreWhiteSpace = 0 != ((unsigned)readerOptions & (unsigned)ptr_ignoreWhiteSpace);
- noRoot = 0 != ((unsigned)readerOptions & (unsigned)ptr_noRoot);
- }
- void resetState()
- {
- bufPtr = buf;
- nextChar = 0;
- if (nullTerm || stream)
- bufRemaining = 0;
- curOffset = 0;
- line = 0;
- }
- public:
- CommonReaderBase(ISimpleReadStream &_stream, IPTreeNotifyEvent &_iEvent, PTreeReaderOptions _readerOptions, size32_t _bufSize=0) :
- bufSize(_bufSize), readerOptions(_readerOptions), iEvent(&_iEvent)
- {
- if (!bufSize) bufSize = 0x20000;
- buf = new byte[bufSize];
- bufRemaining = 0;
- curOffset = 0;
- bufOwned = true;
- nullTerm = false;
- lstream.set(&_stream);
- stream = &_stream; // for efficiency
- init();
- resetState();
- }
- CommonReaderBase(const void *_buf, size32_t bufLength, IPTreeNotifyEvent &_iEvent, PTreeReaderOptions _readerOptions) :
- readerOptions(_readerOptions), iEvent(&_iEvent)
- {
- bufSize = 0; // not used for direct reads
- stream = NULL; // not used for direct reads
- bufRemaining = bufLength;
- nullTerm = false;
- buf = (byte *)_buf;
- bufOwned = false;
- init();
- resetState();
- }
- CommonReaderBase(const void *_buf, IPTreeNotifyEvent &_iEvent, PTreeReaderOptions _readerOptions) :
- readerOptions(_readerOptions), iEvent(&_iEvent)
- {
- bufSize = 0; // not used for direct reads
- stream = NULL; // not used for direct reads
- curOffset = 0;
- bufRemaining = 0;
- nullTerm = true;
- buf = (byte *)_buf;
- bufOwned = false;
- init();
- resetState();
- }
- ~CommonReaderBase()
- {
- if (bufOwned)
- delete [] buf;
- }
- protected:
- virtual void reset()
- {
- resetState();
- }
- void rewind(size32_t n)
- {
- assertex(curOffset >= n);
- if (!n) return;
- curOffset -= n;
- size32_t d = (size32_t)(bufPtr-buf);
- if (n > d) n = d;
- if (!nullTerm)
- bufRemaining += n;
- for (;;)
- {
- --bufPtr;
- if (!--n) break;
- if (10 == *bufPtr) --line;
- }
- }
- bool checkBOM()
- {
- bool utf16 = false;
- bool utf8 = false;
- // Note - technically the utf16 LE case could also be utf32 LE (utf32 BE would be 0x00 0x00 0xfe 0xff)
- // But utf32 is so rare that we ignore it for now
- switch ((unsigned char)nextChar)
- {
- case 0xff:
- readNext();
- if (0xfe == (unsigned char)nextChar)
- utf16 = true;
- break;
- case 0xfe:
- readNext();
- if (0xff == (unsigned char)nextChar)
- utf16 = true;
- break;
- case 0xef:
- readNext();
- if (0xbb == (unsigned char)nextChar)
- {
- readNext();
- if (0xbf == (unsigned char)nextChar)
- utf8 = true;
- }
- break;
- default:
- break;
- }
- if (utf8)
- return true;
- else if (utf16)
- error("Unsupported utf16 format detected in BOM header", false);
- return false;
- }
- inline void expecting(const char *str)
- {
- StringBuffer errorMsg("Expecting \"");
- error(errorMsg.append(str).append("\"").str());
- }
- inline void eos()
- {
- error("String terminator hit");
- }
- void match(const char *txt, const char *msg=NULL)
- {
- const char *c = txt;
- for (;;)
- {
- if (*c == '\0') break;
- readNext();
- if (toupper(nextChar) != toupper(*c))
- {
- if (msg)
- error(msg);
- throw c;
- }
- c++;
- }
- }
- void error(const char *msg=NULL, bool giveContext=true, PTreeReadExcptCode code=PTreeRead_syntax) __attribute__((noreturn))
- {
- StringBuffer context;
- if (giveContext)
- {
- size32_t bufPos = (size32_t)(bufPtr-buf);
- unsigned preLen = std::min(40U, bufPos);
- size32_t bR = bufRemaining;
- if (nullTerm)
- {
- byte *tPtr = bufPtr;
- while (bR<40)
- {
- if ('\0' == *tPtr++) break;
- bR++;
- }
- }
- unsigned postLen = std::min(80-preLen, bR);
- const char *bufferContext = (const char *)(bufPtr - preLen);
- context.append(preLen, bufferContext);
- context.append("*ERROR*");
- context.append(postLen, bufferContext+preLen);
- }
- throw createPTreeReadException(code, msg, context.str(), line+1, curOffset);
- }
- inline void readNext()
- {
- if (!readNextToken())
- error("End of stream encountered whilst parsing", true, PTreeRead_EOS);
- curOffset++;
- }
- inline bool checkReadNext()
- {
- if (!readNextToken())
- return false;
- curOffset++;
- return true;
- }
- inline bool checkStartReadNext()
- {
- if (curOffset || nextChar) //not at starting state
- return true;
- return readNextToken();
- }
- inline bool readNextToken();
- inline bool checkSkipWS()
- {
- while (isspace(nextChar)) if (!checkReadNext()) return false;
- return true;
- }
- inline void skipWS()
- {
- while (isspace(nextChar)) readNext();
- }
- };
- class CInstStreamReader { public: }; // only used to ensure different template definitions.
- class CInstBufferReader { public: };
- class CInstStringReader { public: };
- template <> inline bool CommonReaderBase<CInstStreamReader>::readNextToken()
- {
- // do own buffering, to have reasonable error context.
- if (0 == bufRemaining)
- {
- size32_t _bufRemaining = stream->read(bufSize, buf);
- if (!_bufRemaining)
- return false;
- bufRemaining = _bufRemaining;
- bufPtr = buf;
- }
- --bufRemaining;
- nextChar = *bufPtr++;
- if (10 == nextChar)
- line++;
- return true;
- }
- template <> inline bool CommonReaderBase<CInstBufferReader>::readNextToken()
- {
- if (0 == bufRemaining)
- return false;
- --bufRemaining;
- nextChar = *bufPtr++;
- if (10 == nextChar)
- line++;
- return true;
- }
- template <> inline bool CommonReaderBase<CInstStringReader>::readNextToken()
- {
- nextChar = *bufPtr++;
- if ('\0' == nextChar)
- {
- --bufPtr;
- return false;
- }
- if (10 == nextChar)
- line++;
- return true;
- }
- template <typename X>
- class CXMLReaderBase : public CommonReaderBase<X>, implements IEntityHelper
- {
- StringAttrMapping entityTable;
- protected:
- bool ignoreNameSpaces;
- bool hadXMLDecl;
- private:
- void init()
- {
- ignoreNameSpaces = 0 != ((unsigned) readerOptions & (unsigned)ptr_ignoreNameSpaces);
- }
- void resetState()
- {
- hadXMLDecl = false;
- }
- public:
- typedef CommonReaderBase<X> PARENT;
- using PARENT::nextChar;
- using PARENT::readNext;
- using PARENT::expecting;
- using PARENT::match;
- using PARENT::error;
- using PARENT::skipWS;
- using PARENT::rewind;
- using PARENT::readerOptions;
- CXMLReaderBase(ISimpleReadStream &_stream, IPTreeNotifyEvent &_iEvent, PTreeReaderOptions _xmlReaderOptions, size32_t _bufSize=0)
- : CommonReaderBase<X>(_stream, _iEvent, _xmlReaderOptions, _bufSize)
- {
- init();
- resetState();
- }
- CXMLReaderBase(const void *_buf, size32_t bufLength, IPTreeNotifyEvent &_iEvent, PTreeReaderOptions _xmlReaderOptions)
- : CommonReaderBase<X>(_buf, bufLength, _iEvent, _xmlReaderOptions)
- {
- init();
- resetState();
- }
- CXMLReaderBase(const void *_buf, IPTreeNotifyEvent &_iEvent, PTreeReaderOptions _xmlReaderOptions)
- : CommonReaderBase<X>(_buf, _iEvent, _xmlReaderOptions)
- {
- init();
- resetState();
- }
- protected:
- virtual void reset() override
- {
- resetState();
- PARENT::reset();
- }
- void readID(StringBuffer &id)
- {
- if (isValidXPathStartChr(nextChar))
- {
- for (;;)
- {
- id.append(nextChar);
- readNext();
- if (!isValidXPathChr(nextChar)) break;
- }
- }
- }
- void skipString()
- {
- if ('"' == nextChar)
- {
- do { readNext(); } while ('"' != nextChar);
- }
- else if ('\'' == nextChar)
- {
- do { readNext(); } while ('\'' != nextChar);
- }
- else expecting("\" or '");
- }
- bool lookupRefValue(const char *name, StringBuffer &value)
- {
- StringAttr *val = entityTable.getValue(name);
- if (!val) return false;
- value.append(*val);
- return true;
- }
- void storeEntity(const char *name, const char *value)
- {
- entityTable.setValue(name, value);
- }
- void parseEntity()
- {
- try { match("NTITY"); }
- catch (const char *) { error("Bad syntax"); }
- readNext();
- skipWS();
- StringBuffer entityName;
- if ('%' != nextChar)
- {
- readID(entityName);
- skipWS();
- if ('"' == nextChar)
- {
- StringBuffer refValue;
- for (;;)
- {
- readNext();
- if (!nextChar || '"' == nextChar)
- break;
- if ('&' == nextChar)
- {
- readNext();
- StringBuffer ref;
- if ('#' == nextChar)
- {
- ref.append("&#");
- for (;;)
- {
- readNext();
- if (!nextChar)
- expecting(";");
- if (';' == nextChar) break;
- ref.append(nextChar);
- }
- ref.append(";");
- decodeXML(ref, refValue);
- }
- else
- {
- readID(ref);
- if (';' != nextChar)
- expecting(";");
- if (!lookupRefValue(ref, refValue))
- {
- StringBuffer _ref("&");
- _ref.append(ref).append(';');
- decodeXML(ref, refValue); // try inbuilts
- }
- }
- }
- else
- refValue.append(nextChar);
- }
- storeEntity(entityName, refValue);
- }
- }
- do { readNext(); }
- while (nextChar && nextChar != '>');
- }
- void parseIntSubset()
- {
- for (;;)
- {
- readNext();
- skipWS();
- if (']'== nextChar) break;
- if ('<' == nextChar)
- {
- readNext();
- switch (nextChar)
- {
- case '!':
- {
- readNext();
- switch (nextChar)
- {
- case '-':
- parseComment();
- break;
- case 'E':
- parseEntity();
- break;
- default: // ignore anything else
- do { readNext(); }
- while (nextChar && nextChar != '>');
- break;
- }
- break;
- }
- case '?':
- {
- StringBuffer pi;
- parsePI(pi);
- break;
- }
- }
- }
- }
- }
- void parseOther()
- {
- switch (nextChar)
- {
- case '-':
- parseComment2();
- break;
- case 'D':
- {
- try { match("OCTYPE"); }
- catch (const char *) { error("Bad syntax"); }
- readNext();
- skipWS();
- StringBuffer doctypeid;
- readID(doctypeid);
- for (;;)
- {
- skipWS();
- if ('>' == nextChar) break;
- if ('[' == nextChar)
- {
- parseIntSubset();
- if (']' != nextChar)
- expecting("]");
- }
- else if ('S' == nextChar)
- {
- match("YSTEM");
- readNext();
- skipWS();
- skipString();
- }
- else if ('P' == nextChar)
- {
- match("UBLIC");
- readNext();
- skipWS();
- skipString();
- readNext();
- skipWS();
- skipString();
- }
- readNext();
- }
- break;
- }
- default:
- error("Invalid information tag");
- }
- }
- void parsePIOrDecl()
- {
- StringBuffer target;
- parsePI(target);
- if (0 == strcmp("xml", target.str()))
- {
- if (hadXMLDecl)
- error("Only one XML declartion permitted");
- hadXMLDecl = true;
- }
- }
- void parseCData(StringBuffer &text)
- {
- try { match("CDATA["); }
- catch (const char *) { error("Bad CDATA syntax"); }
- for (;;)
- {
- readNext();
- while (']' == nextChar)
- {
- readNext();
- while (']' == nextChar)
- {
- readNext();
- if ('>' == nextChar)
- return;
- else
- text.append(']');
- }
- text.append(']');
- }
- text.append(nextChar);
- }
- }
- void parsePI(StringBuffer &target)
- {
- readNext();
- if (!isValidXPathStartChr(nextChar))
- error("Invalid PI target");
- for (;;)
- {
- target.append(nextChar);
- readNext();
- if (!isValidXPathChr(nextChar))
- break;
- }
- skipWS();
- unsigned closeTag=0;
- for (;;)
- {
- if (!nextChar)
- error("Missing closing PI tag ?>");
- if (1 == closeTag)
- {
- if ('>' == nextChar)
- break;
- closeTag = 0;
- }
- else if ('?' == nextChar)
- closeTag = 1;
- readNext();
- }
- }
- void parseDirective(StringBuffer &res)
- {
- readNext();
- switch (nextChar) {
- case '-':
- parseComment2();
- break;
- case '[':
- parseCData(res);
- break;
- default:
- error("Unrecognised syntax");
- }
- }
- void parseComment()
- {
- readNext();
- if (nextChar != '-') error("Bad comment syntax");
- parseComment2();
- }
- void parseComment2()
- {
- readNext();
- if (nextChar != '-') error("Bad comment syntax");
- readNext();
- unsigned seen = 0;
- while (nextChar)
- {
- if (seen==2)
- {
- if (nextChar=='>')
- return;
- else if (nextChar != '-') // should be syntax error really.
- seen = 0;
- }
- else if (nextChar=='-')
- seen++;
- else
- seen = 0;
- readNext();
- }
- error("Bad comment syntax");
- }
- const char *_decodeXML(unsigned read, const char *startMark, StringBuffer &ret)
- {
- const char *errMark = NULL;
- try { return decodeXML(startMark, ret, &errMark, this); }
- catch (IException *e)
- {
- if (errMark)
- rewind((unsigned)(errMark-startMark));
- StringBuffer errMsg;
- e->errorMessage(errMsg);
- e->Release();
- error(errMsg.str());
- }
- return NULL; // will never get here.
- }
- // IEntityHelper impl.
- virtual bool find(const char *entity, StringBuffer &value) override
- {
- return lookupRefValue(entity, value);
- }
- };
- template <class X>
- class CXMLReader : public CXMLReaderBase<X>, implements IPTreeReader
- {
- bool rootTerminated;
- StringBuffer attrName, attrval;
- StringBuffer tmpStr;
- void init()
- {
- attrName.append('@');
- }
- void resetState()
- {
- rootTerminated = false;
- }
- public:
- typedef CXMLReaderBase<X> PARENT;
- using PARENT::nextChar;
- using PARENT::readNext;
- using PARENT::expecting;
- using PARENT::match;
- using PARENT::error;
- using PARENT::skipWS;
- using PARENT::checkBOM;
- using PARENT::checkReadNext;
- using PARENT::checkSkipWS;
- using PARENT::eos;
- using PARENT::curOffset;
- using PARENT::noRoot;
- using PARENT::ignoreWhiteSpace;
- using PARENT::iEvent;
- using PARENT::parseDirective;
- using PARENT::parseOther;
- using PARENT::parsePI;
- using PARENT::parsePIOrDecl;
- using PARENT::parseComment;
- using PARENT::_decodeXML;
- using PARENT::ignoreNameSpaces;
- using PARENT::hadXMLDecl;
- IMPLEMENT_IINTERFACE;
- CXMLReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions, size32_t bufSize=0)
- : PARENT(stream, iEvent, xmlReaderOptions, bufSize)
- {
- init();
- resetState();
- }
- CXMLReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions)
- : PARENT(buf, bufLength, iEvent, xmlReaderOptions)
- {
- init();
- resetState();
- }
- CXMLReader(const void *buf, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions)
- : PARENT(buf, iEvent, xmlReaderOptions)
- {
- init();
- resetState();
- }
- virtual void reset() override
- {
- resetState();
- PARENT::reset();
- }
- // IPTreeReader
- virtual void load() override { loadXML(); }
- virtual offset_t queryOffset() override { return curOffset; }
- void loadXML()
- {
- bool head=true;
- restart:
- if (!checkReadNext()) return;
- if (head)
- {
- head = false;
- if (checkBOM())
- if (!checkReadNext()) return;
- }
- if (!checkSkipWS()) return;
- if ('<' != nextChar)
- expecting("<");
- readNext();
- if ('!' == nextChar)
- {
- readNext();
- parseOther();
- goto restart;
- }
- else if ('?' == nextChar)
- {
- parsePIOrDecl();
- goto restart;
- }
- if (!noRoot && rootTerminated)
- {
- if (ignoreWhiteSpace)
- if (!checkSkipWS()) return;
- error("Trailing xml after close of root tag");
- }
- _loadXML();
- if (noRoot)
- {
- head = true;
- hadXMLDecl = false;
- }
- else
- rootTerminated = true;
- goto restart;
- }
- void _loadXML()
- {
- restart:
- offset_t startOffset = curOffset-2;
- if ('!' == nextChar) // not sure this branch can ever be hit.
- {
- parseComment();
- readNext();
- if ('<' != nextChar)
- expecting("<");
- goto restart;
- }
- StringBuffer tagName;
- if (ignoreWhiteSpace)
- skipWS();
- while (!isspace(nextChar) && nextChar != '>' && nextChar != '/')
- {
- tagName.append(nextChar);
- readNext();
- if ('<' == nextChar)
- error("Unmatched close tag encountered");
- }
- StringBuffer completeTagname(tagName);
- if (ignoreNameSpaces)
- {
- const char *colon;
- if ((colon = strchr(tagName.str(), ':')) != NULL)
- tagName.remove(0, (size32_t)(colon - tagName.str() + 1));
- }
- iEvent->beginNode(tagName.str(), false, startOffset);
- skipWS();
- bool endTag = false;
- bool base64 = false;
- while(nextChar != '>')
- {
- skipWS();
- if (nextChar=='/')
- {
- readNext();
- if (nextChar != '>')
- expecting(">");
- endTag = true;
- break;
- }
- attrName.setLength(1);
- attrval.clear();
- while (nextChar && !isspace(nextChar) && nextChar != '=' && nextChar != '>' && nextChar != '/')
- {
- attrName.append(nextChar);
- readNext();
- }
- skipWS();
- if (nextChar == '=') readNext(); else expecting("=");
- skipWS();
- if (nextChar == '"')
- {
- readNext();
- while (nextChar != '"')
- {
- if (!nextChar)
- eos();
- attrval.append(nextChar);
- readNext();
- }
- }
- else if (nextChar == '\'')
- {
- readNext();
- while (nextChar != '\'')
- {
- attrval.append(nextChar);
- readNext();
- }
- }
- else
- error();
- _decodeXML(0, attrval.str(), tmpStr.clear());
- if (0 == strcmp(attrName.str(), "@xsi:type") &&
- (0 == stricmp(tmpStr.str(),"SOAP-ENC:base64")))
- base64 = true;
- else
- iEvent->newAttribute(attrName.str(), tmpStr.str());
- readNext();
- skipWS();
- }
- iEvent->beginNodeContent(tagName.str());
- StringBuffer tagText;
- bool binary = base64;
- if (!endTag)
- {
- if (nextChar == '>')
- {
- for (;;)
- {
- for (;;)
- {
- readNext();
- if (ignoreWhiteSpace)
- skipWS();
- if ('\0' == nextChar)
- eos();
- StringBuffer mark;
- while (nextChar && nextChar !='<') { mark.append(nextChar); readNext(); }
- size32_t l = mark.length();
- size32_t r = l+1;
- if (l)
- {
- if (ignoreWhiteSpace)
- {
- while (l-- && isspace(mark.charAt(l)));
- mark.setLength(l+1);
- }
- tagText.ensureCapacity(mark.length());
- _decodeXML(r, mark.str(), tagText);
- }
- readNext();
- if ('!' == nextChar)
- parseDirective(tagText);
- else if ('?' == nextChar)
- {
- parsePI(tmpStr.clear());
- #ifdef STRICT_PI
- if (0 == stricmp(tmpStr.str(), "xml"))
- error("Reserved PI target used");
- #endif
- }
- else
- break;
- }
- if (nextChar=='/')
- {
- if (base64)
- {
- JBASE64_Decode(tagText.str(), tmpStr.clear());
- tagText.swapWith(tmpStr);
- }
- else
- {
- if (strlen(tagText.str()) != tagText.length())
- binary = true;
- }
- break; // exit
- }
- else
- _loadXML();
- }
- readNext();
- unsigned i = 0;
- while (!isspace(nextChar) && nextChar != '>')
- {
- if ((i >= completeTagname.length()) ||
- (nextChar != completeTagname.charAt(i++)))
- error("Mismatched opening and closing tags");
- readNext();
- }
- if (i != completeTagname.length())
- error("Mismatched opening and closing tags");
- skipWS();
- if (nextChar != '>')
- expecting(">");
- }
- }
- iEvent->endNode(tagName.str(), tagText.length(), tagText.str(), binary, curOffset);
- }
- };
- template <class X>
- class CPullXMLReader : public CXMLReaderBase<X>, implements IPullPTreeReader
- {
- typedef CXMLReaderBase<X> PARENT;
- using PARENT::nextChar;
- using PARENT::readNext;
- using PARENT::expecting;
- using PARENT::match;
- using PARENT::error;
- using PARENT::skipWS;
- using PARENT::checkBOM;
- using PARENT::checkReadNext;
- using PARENT::checkSkipWS;
- using PARENT::eos;
- using PARENT::curOffset;
- using PARENT::noRoot;
- using PARENT::ignoreWhiteSpace;
- using PARENT::iEvent;
- using PARENT::parseDirective;
- using PARENT::parseOther;
- using PARENT::parsePI;
- using PARENT::parsePIOrDecl;
- using PARENT::parseComment;
- using PARENT::_decodeXML;
- using PARENT::ignoreNameSpaces;
- using PARENT::hadXMLDecl;
- class CStateInfo : public CInterface
- {
- public:
- CStateInfo()
- {
- tag.ensureCapacity(15);
- binary = base64 = false;
- }
- inline void reset()
- {
- binary = base64 = false;
- tag.clear();
- tagText.clear();
- }
- const char *wnsTag;
- StringBuffer tag;
- StringBuffer tagText;
- bool binary, base64;
- };
- CICopyArrayOf<CStateInfo> stack, freeStateInfo;
- CStateInfo *stateInfo;
- enum ParseStates { headerStart, tagStart, tagAttributes, tagContent, tagContent2, tagClose, tagEnd, tagMarker } state;
- bool endOfRoot;
- StringBuffer attrName, attrval, mark, tmpStr;
- void resetState()
- {
- stack.kill();
- state = headerStart;
- stateInfo = NULL;
- endOfRoot = false;
- attrName.append('@');
- }
- public:
- IMPLEMENT_IINTERFACE;
- CPullXMLReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions, size32_t bufSize=0)
- : CXMLReaderBase<X>(stream, iEvent, xmlReaderOptions, bufSize)
- {
- resetState();
- }
- CPullXMLReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions)
- : CXMLReaderBase<X>(buf, bufLength, iEvent, xmlReaderOptions)
- {
- resetState();
- }
- CPullXMLReader(const void *buf, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions)
- : CXMLReaderBase<X>(buf, iEvent, xmlReaderOptions)
- {
- resetState();
- }
- ~CPullXMLReader()
- {
- ForEachItemIn(i, stack)
- delete &stack.item(i);
- ForEachItemIn(i2, freeStateInfo)
- delete &freeStateInfo.item(i2);
- }
- // IPullPTreeReader
- virtual void load() override
- {
- while (next()) {}
- }
- virtual void reset() override
- {
- PARENT::reset();
- resetState();
- }
- virtual offset_t queryOffset() override { return curOffset; }
- virtual bool next() override
- {
- switch (state)
- {
- case headerStart:
- {
- if (!checkReadNext()) return false;
- if (checkBOM())
- if (!checkReadNext()) return false;
- for (;;)
- {
- if (!checkSkipWS()) return false;
- if ('<' != nextChar)
- expecting("<");
- readNext();
- if ('!' == nextChar)
- {
- readNext();
- parseOther();
- }
- else if ('?' == nextChar)
- parsePIOrDecl();
- else
- break;
- if (!checkReadNext()) return false;
- }
- state = tagStart;
- break;
- }
- case tagStart:
- {
- offset_t startOffset;
- for (;;)
- {
- if ('!' != nextChar)
- break;
- parseComment();
- readNext();
- if ('<' != nextChar)
- expecting("<");
- }
- startOffset = curOffset-2;
- if (freeStateInfo.ordinality())
- {
- stateInfo = &freeStateInfo.popGet();
- stateInfo->reset();
- }
- else
- stateInfo = new CStateInfo;
- stack.append(*stateInfo);
- if ('/' == nextChar)
- error("Unmatched close tag encountered");
- while (!isspace(nextChar) && nextChar != '>')
- {
- stateInfo->tag.append(nextChar);
- readNext();
- if ('/' == nextChar) break;
- if ('<' == nextChar)
- error("Unmatched close tag encountered");
- }
- stateInfo->wnsTag = stateInfo->tag.str();
- if (ignoreNameSpaces)
- {
- const char *colon;
- if ((colon = strchr(stateInfo->wnsTag, ':')) != NULL)
- stateInfo->wnsTag = colon+1;
- }
- endOfRoot = false;
- try
- {
- iEvent->beginNode(stateInfo->wnsTag, false, startOffset);
- }
- catch (IPTreeException *pe)
- {
- if (PTreeExcpt_InvalidTagName == pe->errorCode())
- {
- pe->Release();
- StringBuffer msg("Expecting valid start tag, but got \"");
- error(msg.append(stateInfo->wnsTag).append("\"").str());
- }
- throw;
- }
- state = tagAttributes;
- break;
- }
- case tagAttributes:
- {
- skipWS();
- if (nextChar == '>')
- state = tagContent;
- else
- {
- skipWS();
- if (nextChar=='/')
- {
- readNext();
- if (nextChar != '>')
- expecting(">");
- // no actual content
- iEvent->beginNodeContent(stateInfo->wnsTag);
- state = tagEnd;
- break;
- }
- attrName.setLength(1);
- attrval.clear();
- while (nextChar && !isspace(nextChar) && nextChar != '=' && nextChar != '>' && nextChar != '/')
- {
- attrName.append(nextChar);
- readNext();
- }
- skipWS();
- if (nextChar == '=') readNext(); else expecting("=");
- skipWS();
- if (nextChar == '"')
- {
- readNext();
- while (nextChar != '"')
- {
- if (!nextChar)
- eos();
- attrval.append(nextChar);
- readNext();
- }
- }
- else if (nextChar == '\'')
- {
- readNext();
- while (nextChar != '\'')
- {
- attrval.append(nextChar);
- readNext();
- }
- }
- else
- error();
- _decodeXML(0, attrval.str(), tmpStr.clear());
- if (0 == strcmp(attrName.str(), "@xsi:type") &&
- (0 == stricmp(tmpStr.str(),"SOAP-ENC:base64")))
- stateInfo->base64 = true;
- else
- iEvent->newAttribute(attrName.str(), tmpStr.str());
- readNext();
- skipWS();
- }
- break;
- }
- case tagContent:
- {
- iEvent->beginNodeContent(stateInfo->wnsTag);
- if ('>' != nextChar)
- state = tagEnd;
- else
- state = tagContent2;
- break;
- }
- case tagContent2:
- {
- try
- {
- for (;;)
- {
- if (endOfRoot)
- {
- if (!checkReadNext()) return false;
- if (!checkSkipWS()) return false;
- }
- else
- {
- readNext();
- if (ignoreWhiteSpace)
- skipWS();
- }
- if ('\0' == nextChar)
- eos();
- mark.clear();
- state = tagMarker;
- while (nextChar && nextChar !='<') { mark.append(nextChar); readNext(); }
- if (!nextChar)
- break;
- size32_t l = mark.length();
- size32_t r = l+1;
- if (l && stateInfo)
- {
- if (ignoreWhiteSpace)
- {
- const char *tb = mark.str();
- const char *t = tb+l-1;
- if (isspace(*t))
- {
- while (t != tb && isspace(*(--t)));
- mark.setLength((size32_t)(t-tb+1));
- }
- }
- stateInfo->tagText.ensureCapacity(mark.length());
- _decodeXML(r, mark.str(), stateInfo->tagText);
- }
- if (endOfRoot && mark.length())
- {
- const char *m = mark.str();
- const char *e = m+mark.length();
- do { if (!isspace(*m++)) error("Trailing content after close of root tag"); }
- while (m!=e);
- }
- readNext();
- if ('!' == nextChar)
- {
- parseDirective(stateInfo->tagText);
- state = tagContent2;
- }
- else if ('?' == nextChar)
- {
- parsePI(tmpStr.clear());
- #ifdef STRICT_PI
- if (0 == stricmp(tmpStr.str(), "xml"))
- error("Reserved PI target used");
- #endif
- state = tagContent2;
- }
- else
- break;
- }
- }
- catch (IPTreeReadException *e)
- {
- if (endOfRoot && PTreeRead_EOS == e->errorCode() && (state != tagContent2 && mark.length())) // only to provide more meaningful error
- {
- const char *m = mark.str();
- const char *es = m+mark.length();
- do
- {
- if (!isspace(*m++))
- {
- e->Release();
- error("Trailing content after close of root tag");
- }
- }
- while (m!=es);
- }
- throw;
- }
- if ('/' == nextChar)
- {
- if (endOfRoot && !noRoot)
- error("Trailing tag close after close of root tag");
- if (stateInfo->base64)
- {
- JBASE64_Decode(stateInfo->tagText.str(), tmpStr.clear());
- stateInfo->tagText.swapWith(tmpStr);
- stateInfo->binary = true;
- // next state tagContent2 still
- }
- else
- {
- if (strlen(stateInfo->tagText.str()) != stateInfo->tagText.length())
- stateInfo->binary = true;
- }
- state = tagClose;
- break; // exit
- }
- else
- {
- if (endOfRoot && !noRoot)
- error("Trailing tag open after close of root tag");
- state = tagStart;
- }
- break;
- }
- case tagClose:
- {
- readNext();
- const char *t = stateInfo->tag.str();
- const char *te = t+stateInfo->tag.length();
- for (;;)
- {
- if (nextChar == '>' || isspace(nextChar))
- {
- if (t != te)
- error("Mismatched opening and closing tags");
- break;
- }
- else if (nextChar != *t++)
- error("Mismatched opening and closing tags");
- readNext();
- }
- skipWS();
- if (nextChar != '>')
- expecting(">");
- state = tagEnd;
- break;
- }
- case tagEnd:
- {
- iEvent->endNode(stateInfo->wnsTag, stateInfo->tagText.length(), stateInfo->tagText.str(), stateInfo->binary, curOffset);
- freeStateInfo.append(*stateInfo);
- stack.pop();
- endOfRoot = 0==stack.ordinality();
- stateInfo = stack.ordinality()?&stack.tos():NULL;
- if (endOfRoot && noRoot)
- {
- state = headerStart;
- hadXMLDecl = false;
- endOfRoot = false;
- }
- else
- state = tagContent2;
- break;
- }
- }
- return true;
- }
- };
- IPTreeReader *createXMLStreamReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions, size32_t bufSize)
- {
- class CXMLStreamReader : public CXMLReader<CInstStreamReader>
- {
- public:
- CXMLStreamReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions, size32_t bufSize=0) : CXMLReader<CInstStreamReader>(stream, iEvent, xmlReaderOptions, bufSize) { }
- };
- return new CXMLStreamReader(stream, iEvent, xmlReaderOptions, bufSize);
- }
- IPTreeReader *createXMLStringReader(const char *xml, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions)
- {
- class CXMLStringReader : public CXMLReader<CInstStringReader>
- {
- public:
- CXMLStringReader(const void *xml, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions) : CXMLReader<CInstStringReader>(xml, iEvent, xmlReaderOptions) { }
- };
- if (NULL == xml)
- throw createPTreeReadException(PTreeRead_syntax, "Null string passed to createXMLStringReader", NULL, 0, 0);
- return new CXMLStringReader(xml, iEvent, xmlReaderOptions);
- }
- IPTreeReader *createXMLBufferReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions)
- {
- class CXMLBufferReader : public CXMLReader<CInstBufferReader>
- {
- public:
- CXMLBufferReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions) : CXMLReader<CInstBufferReader>(buf, bufLength, iEvent, xmlReaderOptions) { }
- };
- return new CXMLBufferReader(buf, bufLength, iEvent, xmlReaderOptions);
- }
- IPullPTreeReader *createPullXMLStreamReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions, size32_t bufSize)
- {
- class CXMLStreamReader : public CPullXMLReader<CInstStreamReader>
- {
- public:
- CXMLStreamReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions, size32_t bufSize=0) : CPullXMLReader<CInstStreamReader>(stream, iEvent, xmlReaderOptions, bufSize) { }
- };
- return new CXMLStreamReader(stream, iEvent, xmlReaderOptions, bufSize);
- }
- IPullPTreeReader *createPullXMLStringReader(const char *xml, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions)
- {
- class CXMLStringReader : public CPullXMLReader<CInstStringReader>
- {
- public:
- CXMLStringReader(const void *xml, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions) : CPullXMLReader<CInstStringReader>(xml, iEvent, xmlReaderOptions) { }
- };
- return new CXMLStringReader(xml, iEvent, xmlReaderOptions);
- }
- IPullPTreeReader *createPullXMLBufferReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions)
- {
- class CXMLBufferReader : public CPullXMLReader<CInstBufferReader>
- {
- public:
- CXMLBufferReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions) : CPullXMLReader<CInstBufferReader>(buf, bufLength, iEvent, xmlReaderOptions) { }
- };
- return new CXMLBufferReader(buf, bufLength, iEvent, xmlReaderOptions);
- }
- IPTreeMaker *createPTreeMaker(byte flags, IPropertyTree *root, IPTreeNodeCreator *nodeCreator)
- {
- return new CPTreeMaker(flags, nodeCreator, root);
- }
- IPTreeMaker *createRootLessPTreeMaker(byte flags, IPropertyTree *root, IPTreeNodeCreator *nodeCreator)
- {
- return new CPTreeMaker(flags, nodeCreator, root, true);
- }
- ////////////////////////////
- ///////////////////////////
- static IPTreeMaker *createDefaultPTreeMaker(byte flags, PTreeReaderOptions readFlags)
- {
- bool noRoot = 0 != ((unsigned)readFlags & (unsigned)ptr_noRoot);
- return new CPTreeMaker(flags, NULL, NULL, noRoot);
- }
- IPropertyTree *createPTree(ISimpleReadStream &stream, byte flags, PTreeReaderOptions readFlags, IPTreeMaker *iMaker)
- {
- Owned<IPTreeMaker> _iMaker;
- if (!iMaker)
- {
- iMaker = createDefaultPTreeMaker(flags, readFlags);
- _iMaker.setown(iMaker);
- }
- Owned<IPTreeReader> reader = createXMLStreamReader(stream, *iMaker, readFlags);
- reader->load();
- if (iMaker->queryRoot())
- return LINK(iMaker->queryRoot());
- else
- return iMaker->create(NULL);
- }
- IPropertyTree *createPTree(IFileIO &ifileio, byte flags, PTreeReaderOptions readFlags, IPTreeMaker *iMaker)
- {
- OwnedIFileIOStream stream = createIOStream(&ifileio);
- return createPTree(*stream, flags, readFlags, iMaker);
- }
- IPropertyTree *createPTree(IFile &ifile, byte flags, PTreeReaderOptions readFlags, IPTreeMaker *iMaker)
- {
- OwnedIFileIO ifileio = ifile.open(IFOread);
- if (!ifileio)
- throw MakeStringException(0, "Could not locate filename: %s", ifile.queryFilename());
- return createPTree(*ifileio, flags, readFlags, iMaker);
- }
- IPropertyTree *createPTreeFromXMLFile(const char *filename, byte flags, PTreeReaderOptions readFlags, IPTreeMaker *iMaker)
- {
- OwnedIFile ifile = createIFile(filename);
- return createPTree(*ifile, flags, readFlags, iMaker);
- }
- IPropertyTree *createPTreeFromXMLString(const char *xml, byte flags, PTreeReaderOptions readFlags, IPTreeMaker *iMaker)
- {
- Owned<IPTreeMaker> _iMaker;
- if (!iMaker)
- {
- iMaker = createDefaultPTreeMaker(flags, readFlags);
- _iMaker.setown(iMaker);
- }
- Owned<IPTreeReader> reader = createXMLStringReader(xml, *iMaker, readFlags);
- reader->load();
- return LINK(iMaker->queryRoot());
- }
- IPropertyTree *createPTreeFromXMLString(unsigned len, const char *xml, byte flags, PTreeReaderOptions readFlags, IPTreeMaker *iMaker)
- {
- Owned<IPTreeMaker> _iMaker;
- if (!iMaker)
- {
- iMaker = createDefaultPTreeMaker(flags, readFlags);
- _iMaker.setown(iMaker);
- }
- Owned<IPTreeReader> reader = createXMLBufferReader(xml, len, *iMaker, readFlags);
- reader->load();
- return LINK(iMaker->queryRoot());
- }
- //////////////////////////
- /////////////////////////
- inline bool isHiddenWhenSanitized(const char *val)
- {
- if (!val || !*val)
- return false;
- return !(streq(val, "0") || streq(val, "1") || strieq(val, "true") || strieq(val, "false") || strieq(val, "yes") || strieq(val, "no"));
- }
- inline bool isSanitizedAndHidden(const char *val, byte flags, bool attribute)
- {
- bool sanitize = (attribute) ? ((flags & YAML_SanitizeAttributeValues)!=0) : ((flags & YAML_Sanitize)!=0);
- if (sanitize)
- return isHiddenWhenSanitized(val);
- return false;
- }
- static void _toXML(const IPropertyTree *tree, IIOStream &out, unsigned indent, unsigned flags)
- {
- const char *name = tree->queryName();
- if (!name) name = "__unnamed__";
- bool isBinary = tree->isBinary(NULL);
- bool inlinebody = true;
- if (flags & XML_Embed) writeCharsNToStream(out, ' ', indent);
- writeCharToStream(out, '<');
- writeStringToStream(out, name);
- Owned<IAttributeIterator> it = tree->getAttributes(true);
- if (it->first())
- {
- unsigned attributeindent = indent+2+(size32_t)strlen(name);
- bool first = true;
- do
- {
- const char *key = it->queryName();
- if (!isBinary || stricmp(key, "@xsi:type")!=0)
- {
- if (first)
- {
- if (flags & XML_LineBreak) inlinebody = false;
- first = false;
- writeCharToStream(out, ' ');
- }
- else if ((flags & XML_LineBreakAttributes) && it->count() > 3)
- {
- writeStringToStream(out, "\n");
- writeCharsNToStream(out, ' ', attributeindent);
- }
- else
- writeCharToStream(out, ' ');
- writeStringToStream(out, key+1);
- if (flags & XML_SingleQuoteAttributeValues)
- writeStringToStream(out, "='");
- else
- writeStringToStream(out, "=\"");
- const char *val = it->queryValue();
- if (val)
- {
- if (isSanitizedAndHidden(val, flags, true))
- writeCharsNToStream(out, '*', strlen(val));
- else
- encodeXML(val, out, ENCODE_NEWLINES, (unsigned)-1, true);
- }
- if (flags & XML_SingleQuoteAttributeValues)
- writeCharToStream(out, '\'');
- else
- writeCharToStream(out, '"');
- }
- }
- while (it->next());
- }
- Owned<IPropertyTreeIterator> sub = tree->getElements("*", 0 != (flags & XML_SortTags) ? iptiter_sort : iptiter_null);
- MemoryBuffer thislevelbin;
- StringBuffer _thislevel;
- const char *thislevel = NULL; // to avoid uninitialized warning
- bool empty;
- if (isBinary)
- {
- if (flags & XML_LineBreak) inlinebody = false;
- writeStringToStream(out, " xsi:type=\"SOAP-ENC:base64\"");
- empty = (!tree->getPropBin(NULL, thislevelbin))||(thislevelbin.length()==0);
- }
- else
- {
- if (tree->isCompressed(NULL))
- {
- empty = false; // can't be empty if compressed;
- verifyex(tree->getProp(NULL, _thislevel));
- thislevel = _thislevel.str();
- }
- else
- empty = (NULL == (thislevel = tree->queryProp(NULL)));
- }
- if (sub->first())
- {
- if (flags & XML_LineBreak) inlinebody = false;
- }
- else if (empty && !(flags & XML_Sanitize))
- {
- if (flags & XML_LineBreak)
- writeStringToStream(out, "/>\n");
- else
- writeStringToStream(out, "/>");
- return;
- }
- writeCharToStream(out, '>');
- if (!inlinebody)
- writeStringToStream(out, "\n");
- for(; sub->isValid(); sub->next())
- _toXML(&sub->query(), out, indent+1, flags);
- if (!empty)
- {
- if (!inlinebody)
- writeCharsNToStream(out, ' ', indent+1);
- if (flags & XML_Sanitize)
- {
- // NOTE - we don't output anything for binary.... is that ok?
- if (thislevel)
- {
- if (isHiddenWhenSanitized(thislevel))
- writeCharsNToStream(out, '*', strlen(thislevel));
- else
- writeStringToStream(out, thislevel);
- }
- }
- else if (isBinary)
- {
- if (flags & XML_NoBinaryEncode64)
- {
- if (flags & XML_NoEncode)
- {
- out.write(thislevelbin.length(), thislevelbin.toByteArray());
- }
- else
- {
- const char * buff = static_cast<const char *>(thislevelbin.toByteArray());
- const unsigned len = thislevelbin.length();
- unsigned prefix = 0;
- while ((prefix < len) && isspace(buff[prefix]))
- prefix++;
- encodeXML(buff, out, ENCODE_WHITESPACE, prefix, true);
- if (prefix != len) { // check not all spaces
- unsigned suffix = len;
- while (isspace(buff[suffix-1]))
- suffix--;
- encodeXML(buff+prefix, out, 0, suffix-prefix, true);
- encodeXML(buff+suffix, out, ENCODE_WHITESPACE, len-suffix, true);
- }
- }
- }
- else
- JBASE64_Encode(thislevelbin.toByteArray(), thislevelbin.length(), out, true);
- }
- else
- {
- if (flags & XML_NoEncode)
- {
- writeStringToStream(out, thislevel);
- }
- else
- {
- const char *m = thislevel;
- const char *p = m;
- while (isspace(*p))
- p++;
- encodeXML(m, out, ENCODE_WHITESPACE, p-m, true);
- if (*p) { // check not all spaces
- const char *s = p+strlen(p);
- while (isspace(*(s-1)))
- s--;
- assertex(s>p);
- encodeXML(p, out, 0, s-p, true);
- encodeXML(s, out, ENCODE_WHITESPACE, (unsigned)-1, true);
- }
- }
- if (!inlinebody)
- writeStringToStream(out, "\n");
- }
- }
- if (!inlinebody)
- writeCharsNToStream(out, ' ', indent);
- writeStringToStream(out, "</");
- writeStringToStream(out, name);
- if (flags & XML_LineBreak)
- writeStringToStream(out, ">\n");
- else
- writeCharToStream(out, '>');
- }
- class CStringBufferMarkupIOAdapter : public CInterfaceOf<IIOStream>
- {
- StringBuffer &out;
- public:
- CStringBufferMarkupIOAdapter(StringBuffer &_out) : out(_out) { }
- virtual void flush() override { }
- virtual size32_t read(size32_t len, void * data) override { UNIMPLEMENTED; return 0; }
- virtual size32_t write(size32_t len, const void * data) override { out.append(len, (const char *)data); return len; }
- };
- jlib_decl StringBuffer &toXML(const IPropertyTree *tree, StringBuffer &ret, unsigned indent, unsigned flags)
- {
- CStringBufferMarkupIOAdapter adapter(ret);
- _toXML(tree->queryBranch(NULL), adapter, indent, flags);
- return ret;
- }
- void toXML(const IPropertyTree *tree, IIOStream &out, unsigned indent, unsigned flags)
- {
- _toXML(tree, out, indent, flags);
- }
- void printXML(const IPropertyTree *tree, unsigned indent, unsigned flags)
- {
- StringBuffer xml;
- toXML(tree, xml, indent, flags);
- printf("%s", xml.str());
- }
- void dbglogXML(const IPropertyTree *tree, unsigned indent, unsigned flags)
- {
- StringBuffer xml;
- toXML(tree, xml, indent, flags);
- DBGLOG("%s", xml.str());
- }
- void saveXML(const char *filename, const IPropertyTree *tree, unsigned indent, unsigned flags)
- {
- OwnedIFile ifile = createIFile(filename);
- saveXML(*ifile, tree, indent, flags);
- }
- void saveXML(IFile &ifile, const IPropertyTree *tree, unsigned indent, unsigned flags)
- {
- OwnedIFileIO ifileio = ifile.open(IFOcreate);
- if (!ifileio)
- throw MakeStringException(0, "saveXML: could not find %s to open", ifile.queryFilename());
- saveXML(*ifileio, tree, indent, flags);
- }
- void saveXML(IFileIO &ifileio, const IPropertyTree *tree, unsigned indent, unsigned flags)
- {
- Owned<IIOStream> stream = createIOStream(&ifileio);
- stream.setown(createBufferedIOStream(stream));
- saveXML(*stream, tree, indent, flags);
- }
- void saveXML(IIOStream &stream, const IPropertyTree *tree, unsigned indent, unsigned flags)
- {
- toXML(tree, stream, indent, flags);
- }
- /////////////////////////
- void checkWriteJSONDelimiter(IIOStream &out, bool &delimit)
- {
- if (delimit)
- writeCharToStream(out, ',');
- delimit = false;
- }
- static void writeJSONNameToStream(IIOStream &out, const char *name, unsigned indent, bool &delimit)
- {
- if (!name || !*name)
- return;
- checkWriteJSONDelimiter(out, delimit);
- if (indent)
- {
- writeCharToStream(out, '\n');
- writeCharsNToStream(out, ' ', indent);
- }
- else
- writeCharToStream(out, ' ');
- writeCharToStream(out, '"');
- writeStringToStream(out, name);
- writeStringToStream(out, "\": ");
- delimit = false;
- }
- static void writeJSONValueToStream(IIOStream &out, const char *val, bool &delimit, bool hidden=false)
- {
- checkWriteJSONDelimiter(out, delimit);
- delimit = true;
- if (!val)
- {
- writeStringToStream(out, "null");
- return;
- }
- writeCharToStream(out, '"');
- if (hidden)
- writeCharsNToStream(out, '*', strlen(val));
- else
- {
- StringBuffer s;
- writeStringToStream(out, encodeJSON(s, val));
- }
- writeCharToStream(out, '"');
- }
- static void writeJSONBase64ValueToStream(IIOStream &out, const char *val, size32_t len, bool &delimit, bool hidden)
- {
- checkWriteJSONDelimiter(out, delimit);
- delimit = true;
- if (!val)
- {
- writeStringToStream(out, "null");
- return;
- }
- writeCharToStream(out, '"');
- if (hidden)
- JBASE64_Encode("****", strlen("****"), out, false);
- else
- JBASE64_Encode(val, len, out, false);
- writeCharToStream(out, '"');
- }
- bool isRootArrayObjectHidden(bool root, const char *name, byte flags)
- {
- return ((flags & JSON_HideRootArrayObject) && root && name && streq(name,"__array__"));
- }
- static void _toJSON(const IPropertyTree *tree, IIOStream &out, unsigned indent, byte flags, bool &delimit, bool root=false, bool isArrayItem=false)
- {
- Owned<IAttributeIterator> it = tree->getAttributes(true);
- bool hasAttributes = it->first();
- bool complex = (hasAttributes || tree->hasChildren() || tree->isBinary());
- bool isBinary = tree->isBinary(NULL);
- const char *name = tree->queryName();
- if (!root && !isArrayItem)
- {
- if (!name || !*name)
- name = "__unnamed__";
- writeJSONNameToStream(out, name, (flags & JSON_Format) ? indent : 0, delimit);
- }
- checkWriteJSONDelimiter(out, delimit);
- if (isArrayItem && (flags & JSON_Format))
- {
- writeCharToStream(out, '\n');
- writeCharsNToStream(out, ' ', indent);
- }
- bool hiddenRootArrayObject = isRootArrayObjectHidden(root, name, flags);
- if (!hiddenRootArrayObject)
- {
- if (root || complex)
- {
- writeCharToStream(out, '{');
- delimit = false;
- }
- if (hasAttributes)
- {
- ForEach(*it)
- {
- const char *key = it->queryName();
- if (!isBinary || stricmp(key, "@xsi:type")!=0)
- {
- const char *val = it->queryValue();
- if (val)
- {
- writeJSONNameToStream(out, key, (flags & JSON_Format) ? indent+1 : 0, delimit);
- if (flags & JSON_SanitizeAttributeValues)
- writeJSONValueToStream(out, val, delimit, isHiddenWhenSanitized(val));
- else
- {
- StringBuffer encoded;
- encodeJSON(encoded, val);
- writeJSONValueToStream(out, encoded.str(), delimit);
- }
- }
- }
- }
- }
- }
- MemoryBuffer thislevelbin;
- StringBuffer _thislevel;
- const char *thislevel = NULL; // to avoid uninitialized warning
- bool isNull = true;
- if (!hiddenRootArrayObject)
- {
- if (isBinary)
- {
- isNull = (!tree->getPropBin(NULL, thislevelbin))||(thislevelbin.length()==0);
- }
- else
- {
- if (tree->isCompressed(NULL))
- {
- isNull = false; // can't be empty if compressed;
- verifyex(tree->getProp(NULL, _thislevel));
- thislevel = _thislevel.str();
- }
- else
- isNull = (NULL == (thislevel = tree->queryProp(NULL)));
- }
- if (isNull && !root && !complex)
- {
- writeJSONValueToStream(out, NULL, delimit);
- return;
- }
- }
- Owned<IPropertyTreeIterator> sub = tree->getElements(hiddenRootArrayObject ? "__item__" : "*", 0 != (flags & JSON_SortTags) ? iptiter_sort : iptiter_null);
- //note that detection of repeating elements relies on the fact that ptree elements
- //of the same name will be grouped together
- bool repeatingElement = false;
- sub->first();
- while(sub->isValid())
- {
- Linked<IPropertyTree> element = &sub->query();
- const char *name = element->queryName();
- sub->next();
- if (!repeatingElement)
- {
- if (hiddenRootArrayObject)
- {
- writeCharToStream(out, '[');
- repeatingElement = true;
- delimit = false;
- }
- else if (sub->isValid() && streq(name, sub->query().queryName()))
- {
- if (flags & JSON_Format)
- indent++;
- writeJSONNameToStream(out, name, (flags & JSON_Format) ? indent : 0, delimit);
- writeCharToStream(out, '[');
- repeatingElement = true;
- delimit = false;
- }
- }
- _toJSON(element, out, indent+1, flags, delimit, false, repeatingElement);
- if (repeatingElement && (!sub->isValid() || !streq(name, sub->query().queryName())))
- {
- if (flags & JSON_Format)
- {
- writeCharToStream(out, '\n');
- writeCharsNToStream(out, ' ', indent);
- indent--;
- }
- writeCharToStream(out, ']');
- repeatingElement = false;
- delimit = true;
- }
- }
- if (!hiddenRootArrayObject && !isNull)
- {
- if (complex)
- writeJSONNameToStream(out, isBinary ? "#valuebin" : "#value", (flags & JSON_Format) ? indent+1 : 0, delimit);
- if (isBinary)
- writeJSONBase64ValueToStream(out, thislevelbin.toByteArray(), thislevelbin.length(), delimit, flags & XML_Sanitize);
- else
- {
- writeJSONValueToStream(out, thislevel, delimit, isSanitizedAndHidden(thislevel, flags, false));
- }
- }
- if (!hiddenRootArrayObject)
- {
- if (root || complex)
- {
- if (flags & JSON_Format)
- {
- writeCharToStream(out, '\n');
- writeCharsNToStream(out, ' ', indent);
- }
- writeCharToStream(out, '}');
- delimit = true;
- }
- }
- }
- jlib_decl StringBuffer &toJSON(const IPropertyTree *tree, StringBuffer &ret, unsigned indent, byte flags)
- {
- CStringBufferMarkupIOAdapter adapter(ret);
- bool delimit = false;
- _toJSON(tree->queryBranch(NULL), adapter, indent, flags, delimit, true);
- return ret;
- }
- void toJSON(const IPropertyTree *tree, IIOStream &out, unsigned indent, byte flags)
- {
- bool delimit = false;
- _toJSON(tree, out, indent, flags, delimit, true);
- }
- void printJSON(const IPropertyTree *tree, unsigned indent, byte flags)
- {
- StringBuffer json;
- toJSON(tree, json, indent, flags);
- printf("%s", json.str());
- }
- void dbglogJSON(const IPropertyTree *tree, unsigned indent, unsigned flags)
- {
- StringBuffer json;
- toJSON(tree, json, indent, flags);
- DBGLOG("%s", json.str());
- }
- static inline void skipWS(const char *&xpath)
- {
- while (isspace(*xpath)) xpath++;
- }
- static void _validateXPathSyntax(const char *xpath);
- static void validateQualifier(const char *&xxpath)
- {
- const char *xpath = xxpath;
- const char *start = xpath;
- skipWS(xpath);
- const char *lhsStart = xpath;
- for (;;)
- {
- switch (*xpath) {
- case ']':
- case '!':
- case '=':
- case '\0':
- break;
- default:
- if (!isspace(*xpath))
- {
- xpath++;
- continue;
- }
- }
- break;
- }
- StringAttr lhs(lhsStart, xpath-lhsStart);
- _validateXPathSyntax(lhs);
- skipWS(xpath);
- exprType tType = t_none;
- if ('=' == *xpath)
- {
- ++xpath;
- tType = t_equality;
- }
- else if ('!' == *xpath)
- {
- ++xpath;
- if (*xpath && '=' == *xpath)
- {
- tType = t_inequality;
- ++xpath;
- }
- else
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, 0, "Invalid qualifier expression");
- }
- if (t_none != tType)
- {
- skipWS(xpath);
- if ('~' == *xpath)
- {
- ++xpath; // Signifies wild (now always true but still accepted...)
- }
- skipWS(xpath);
- char qu = *xpath;
- if (qu != '\'' && qu != '\"')
- throw MakeXPathException(xpath, PTreeExcpt_XPath_ParseError, 0, "Syntax error - no opening \" or \'");
- ++xpath;
- while (*xpath && *xpath != qu)
- xpath++;
- if (!*xpath)
- throw MakeXPathException(xpath, PTreeExcpt_XPath_ParseError, 0, "Syntax error - no closing \" or \'");
- xpath++;
- }
- skipWS(xpath);
- if (']' != *xpath)
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, 0, "No closing brace to qualifier");
- xxpath = xpath;
- }
- static void _validateXPathSyntax(const char *xpath)
- {
- if (NULL == xpath || '\0' == *xpath)
- return;
- else
- {
- const char *_xpath = xpath;
- restart:
- if (NULL == xpath || '\0' == *xpath)
- return;
- switch (*xpath)
- {
- case '.':
- ++xpath;
- goto restart;
- case '/':
- ++xpath;
- if ('/' == *xpath)
- {
- _validateXPathSyntax(xpath+1);
- return;
- }
- goto restart;
- case '[':
- {
- ++xpath;
- if (isdigit(*xpath))
- {
- StringAttr index;
- xpath = readIndex(xpath, index);
- unsigned i = atoi(index.get());
- if (i)
- {
- }
- else
- {
- // should be syntax error
- }
- if (']' != *xpath)
- throw MakeXPathException(_xpath, PTreeExcpt_XPath_ParseError, xpath-_xpath, "Qualifier brace unclosed");
- }
- else
- validateQualifier(xpath);
- ++xpath;
- break;
- }
- default:
- {
- bool wild;
- const char *start = xpath;
- readWildId(xpath, wild); // validates also
- size32_t s = xpath-start;
- if (s)
- {
- StringAttr id(start, s);
- if ('[' == *xpath) // check for local index not iterative qualifier.
- {
- const char *xxpath = xpath+1;
- if (isdigit(*xxpath))
- {
- StringAttr idxstr;
- xxpath = readIndex(xxpath, idxstr);
- if (']' != *xxpath)
- throw MakeXPathException(_xpath, PTreeExcpt_XPath_ParseError, xpath-_xpath, "Qualifier brace unclosed");
- ++xxpath;
- unsigned index = atoi(idxstr.get());
- if (index)
- {
- }
- xpath = xxpath;
- }
- }
- }
- else if ('@' == *xpath)
- {
- ++xpath;
- const char *start = xpath;
- readID(xpath, false);
- size32_t s = xpath-start;
- if (!s)
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Missing attribute?");
- StringAttr id(start, s);
- if (!validateXMLTag(id))
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Invalid xml tag: %s", id.get());
- while (isspace(*xpath)) xpath++;
- if ('\0' != *xpath)
- throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Cannot have embedded attribute within path (must be tail component)");
- }
- else
- {
- if ('[' != *xpath)
- throw MakeXPathException(xpath, PTreeExcpt_XPath_ParseError, 0, "Qualifier expected e.g. [..]");
- validateQualifier(xpath);
- }
- break;
- }
- }
- }
- if (*xpath == '\0' || (*xpath == '/' && '\0' == *(xpath+1)))
- return;
- else
- _validateXPathSyntax(xpath);
- }
- bool validateXPathSyntax(const char *xpath, StringBuffer *error)
- {
- try
- {
- if (xpath && '/' == *xpath && *(xpath+1) != '/')
- throw MakeXPathException(xpath, PTreeExcpt_XPath_Unsupported, 0, "Root specifier \"/\" specifier is not supported");
- _validateXPathSyntax(xpath);
- return true;
- }
- catch (IException *e)
- {
- if (error)
- e->errorMessage(*error);
- e->Release();
- return false;
- }
- }
- static bool isContentXPath(const char *xpath, StringBuffer &head)
- {
- unsigned l = xpath?strlen(xpath):0;
- const char *x = xpath+l-2;
- if (l>=2 && 0==strcmp(XMLTAG_CONTENT, x))
- {
- if (x != xpath)
- head.append(x-xpath, xpath);
- return true;
- }
- return false;
- }
- bool validateXMLParseXPath(const char *xpath, StringBuffer *error)
- {
- if (!xpath || !*xpath)
- return true;
- StringBuffer head;
- if (isContentXPath(xpath, head))
- {
- if (head.length())
- {
- if ('/' == *xpath && '/' != *(xpath+1))
- {
- if (error)
- {
- Owned<IException> e = MakeStringException(0, "Invalid extract xml text '<>' usage, xpath cannot from be absolute: %s", xpath);
- e->errorMessage(*error);
- }
- return false;
- }
- return validateXPathSyntax(head.str(), error);
- }
- return true;
- }
- else
- return validateXPathSyntax('/' == *xpath && '/' != *(xpath+1) ? xpath+1 : xpath, error);
- return true;
- }
- bool areMatchingPTrees(const IPropertyTree * left, const IPropertyTree * right)
- {
- if (left == right)
- return true;
- if (!left || !right)
- return false;
- bool isCaseInsensitive = left->isCaseInsensitive();
- const char * lname = left->queryName();
- const char * rname = right->queryName();
- if (!lname || !rname)
- {
- if (lname || rname)
- return false;
- }
- else if ((isCaseInsensitive ? stricmp(lname, rname) : strcmp(lname, rname)) != 0)
- return false;
- Owned<IAttributeIterator> leftAttrIter = left->getAttributes(true);
- Owned<IAttributeIterator> rightAttrIter = right->getAttributes(true);
- rightAttrIter->first();
- ForEach(*leftAttrIter)
- {
- if (!rightAttrIter->isValid()) return false;
- const char * lname = leftAttrIter->queryName();
- const char * rname = rightAttrIter->queryName();
- if ((isCaseInsensitive ? stricmp(lname, rname) : strcmp(lname, rname)) != 0)
- return false;
- if (strcmp(leftAttrIter->queryValue(), rightAttrIter->queryValue()) != 0)
- return false;
- rightAttrIter->next();
- }
- if (rightAttrIter->isValid()) return false;
- Owned<IPropertyTreeIterator> leftElemIter = left->getElements("*", iptiter_sort);
- Owned<IPropertyTreeIterator> rightElemIter = right->getElements("*", iptiter_sort);
- rightElemIter->first();
- ForEach(*leftElemIter)
- {
- if (!rightElemIter->isValid()) return false;
- if (!areMatchingPTrees(&leftElemIter->query(), &rightElemIter->query()))
- return false;
- rightElemIter->next();
- }
- if (rightElemIter->isValid()) return false;
- return true;
- }
- /////////////////////
- static const char * skipWhitespace(const char * text)
- {
- while ((*text==' ') || (*text=='\t'))
- text++;
- return text;
- }
- static const char * skipAsterisk(const char * text)
- {
- if (*text=='*')
- return skipWhitespace(text+1);
- return text;
- }
- static const char * skipToNewline(const char * text)
- {
- while (*text && (*text != '\r') && (*text != '\n'))
- text++;
- return text;
- }
- static const char * skipNewline(const char * text)
- {
- if (*text == '\r')
- text++;
- if (*text == '\n')
- text++;
- return text;
- }
- void extractJavadoc(IPropertyTree * result, const char * text)
- {
- //Skip a leading blank line
- text = skipWhitespace(text);
- text = skipNewline(text);
- //Now process each of the parameters...
- StringBuffer tagname;
- StringBuffer tagtext;
- for (;;)
- {
- text = skipWhitespace(text);
- text = skipAsterisk(text);
- if ((*text == 0) || (*text == '@'))
- {
- if (tagtext.length())
- {
- if (tagname.length())
- result->addProp(tagname.str(), tagtext.str());
- else
- result->setProp("", tagtext.str());
- tagtext.clear();
- }
- if (*text == 0)
- return;
- text++;
- const char * start = text;
- while (isalnum(*text))
- text++;
- if (start != text)
- tagname.clear().append(text-start, start);
- text = skipWhitespace(text);
- }
- const char * start = text;
- text = skipToNewline(text);
- if (start != text)
- {
- if (tagtext.length())
- tagtext.append(" ");
- tagtext.append(text-start, start);
- }
- text = skipNewline(text);
- }
- }
- /////////////////////
- #ifdef _DEBUG
- jlib_decl void validatePTree()
- {
- Owned<IPropertyTree> testTree = createPTreeFromXMLString(
- "<ROOT>" \
- " <E a=\"av1\" b=\"bv1\" c=\"cv1\"></E>" \
- " <E a=\"av1\" b=\"bv1\" c=\"cv2\"></E>" \
- " <E a=\"av2\" b=\"bv1\"></E>" \
- " <SE c=\"cv1\"></SE>" \
- " <E a=\"av1\" b=\"bv2\"></E>" \
- " <E a=\"av2\" b=\"bv2\" c=\"cv3\">ev1</E>" \
- "</ROOT>"
- );
- Owned<IPropertyTreeIterator> iter = testTree->getElements("E[@a=\"av1\"][@b=\"bv2\"]");
- unsigned c = 0;
- ForEach (*iter)
- ++c;
- assertex(1 == c);
- int v = strcmp("bv1", testTree->queryProp("E[@a=\"av1\"][2]/@b"));
- assertex(0 == v);
- v = strcmp("cv2", testTree->queryProp("E[@a=\"av1\"][@b=\"bv1\"][2]/@c"));
- assertex(0 == v);
- v = strcmp("cv2", testTree->queryProp("E[@a=\"av1\"][2]/@c"));
- assertex(0 == v);
- v = strcmp("ev1", testTree->queryProp("E[@a=\"av2\"][@c]"));
- assertex(0 == v);
- }
- jlib_decl void testValidateXPathSyntax()
- {
- verifyex(validateXPathSyntax("@abc"));
- verifyex(validateXPathSyntax("prop"));
- verifyex(validateXPathSyntax("a/b"));
- verifyex(validateXPathSyntax("a/@b"));
- const char *s = "a[@a=\"blah\"]/b";
- verifyex(validateXPathSyntax(s));
- s = "a/b[@b=\"blah\"]";
- verifyex(validateXPathSyntax(s));
- verifyex(validateXPathSyntax(s));
- s = "a/b[b=\"blah\"]";
- verifyex(validateXPathSyntax(s));
- s = "a/b[a/b=\"blah\"]";
- verifyex(validateXPathSyntax(s));
- verifyex(validateXPathSyntax("a[1]/b[2]"));
- s = "a[b]/b[c=\"a\"]/c";
- verifyex(validateXPathSyntax(s));
- verifyex(validateXPathSyntax("//a/b/c"));
- verifyex(!validateXPathSyntax("a[b"));
- verifyex(!validateXPathSyntax("a["));
- verifyex(!validateXPathSyntax("a]"));
- verifyex(!validateXPathSyntax("a[b=blah]"));
- verifyex(!validateXPathSyntax("@a/b"));
- verifyex(!validateXPathSyntax("a[b[c]]"));
- verifyex(validateXMLParseXPath("<>"));
- verifyex(validateXMLParseXPath("a/b/c<>"));
- verifyex(validateXMLParseXPath("a/b/<>"));
- verifyex(validateXMLParseXPath("/a/b"));
- verifyex(!validateXMLParseXPath("a/b[\"]/<>"));
- verifyex(!validateXMLParseXPath("/<>"));
- }
- jlib_decl void testJdocCompare()
- {
- Owned<IPropertyTree> t1 = createPTree();
- Owned<IPropertyTree> t2 = createPTree();
- Owned<IPropertyTree> t3 = createPTree();
- Owned<IPropertyTree> t4 = createPTree();
- Owned<IPropertyTree> t5 = createPTree();
- extractJavadoc(t1, "Defines a record that contains information about a person");
- extractJavadoc(t2, "Allows the name table to be filtered.\n\n@param ages\tThe ages that are allowed to be processed.\n\t\tbadForename Forname to avoid.\n\n@return\tthe filtered dataset.");
- extractJavadoc(t3, "Allows the name table to be filtered.\n\n@param ages\tThe ages that are allowed to be processed.\n\t\tbadForename Forname to avoid.\n\n@return\tthe filtered dataset.");
- extractJavadoc(t4, "Allows the name table to be filtered.\n\n@param ages\tThe ages that are allowed to be processed.\n\t\tbadForename Forname to avoid.\n\n@return\tthe filtered dataset.");
- extractJavadoc(t5, "Allows the name table to be filtered.\n\n@param ages\tThe ages that are allowed to be processed.\n\t\tbadForename Forname to avoid.\n\n@return\tthe filtered dataset.");
- IPropertyTree * t2c = t2->addPropTree("Child1", createPTree());
- extractJavadoc(t2c, "This is some child data\n\n@param ages\tThe ages that are allowed to be processed.");
- IPropertyTree * t3c = t3->addPropTree("Child1", createPTree());
- extractJavadoc(t3c, "This is some child data\n\n@param ages\tThe ages that are allowed to be processed.");
- IPropertyTree * t4c = t4->addPropTree("Child1", createPTree());
- extractJavadoc(t4c, "This is some child data\n\n@param ages\tThe ages that are allowed to be processed, but differs.");
- IPropertyTree * t5c = t5->addPropTree("Child1", createPTree());
- extractJavadoc(t5c, "This is some child data\n\n@param ages\tThe ages that are allowed to be processed.");
- t2->setProp("@childAttr", "1");
- t3->setProp("@childAttr", "1");
- t4->setProp("@childAttr", "1");
- t5->setProp("@childAttr", "2");
- verifyex(areMatchingPTrees(NULL, NULL));
- verifyex(!areMatchingPTrees(NULL, t1));
- verifyex(!areMatchingPTrees(t1, NULL));
- verifyex(areMatchingPTrees(t1, t1));
- verifyex(areMatchingPTrees(t2, t3));
- verifyex(!areMatchingPTrees(t2, t4));
- verifyex(!areMatchingPTrees(t2, t5));
- }
- #endif
- template <class BASE_PTREE>
- class COrderedPTree : public BASE_PTREE
- {
- template <class BASECHILDMAP>
- class jlib_decl COrderedChildMap : public BASECHILDMAP
- {
- typedef COrderedChildMap<BASECHILDMAP> SELF;
- ICopyArrayOf<IPropertyTree> order;
- public:
- IMPLEMENT_SUPERHASHTABLEOF_REF_FIND(IPropertyTree, constcharptr);
- COrderedChildMap<BASECHILDMAP>() : BASECHILDMAP() { }
- ~COrderedChildMap<BASECHILDMAP>() { SELF::kill(); }
- virtual unsigned numChildren() const override { return order.ordinality(); }
- virtual IPropertyTreeIterator *getIterator(bool sort) override
- {
- class CPTArrayIterator : public ArrayIIteratorOf<IArrayOf<IPropertyTree>, IPropertyTree, IPropertyTreeIterator>
- {
- IArrayOf<IPropertyTree> elems;
- public:
- CPTArrayIterator(ICopyArrayOf<IPropertyTree> &order, bool sort) : ArrayIIteratorOf<IArrayOf<IPropertyTree>, IPropertyTree, IPropertyTreeIterator>(elems)
- {
- ForEachItemIn(e, order)
- elems.append(*LINK(&order.item(e)));
- if (sort)
- elems.sort(comparePropTrees);
- }
- };
- return new CPTArrayIterator(order, sort);
- }
- virtual bool set(const char *key, IPropertyTree *tree) override
- {
- IPropertyTree *existing = find(*key);
- if (existing)
- {
- unsigned pos = order.find(*existing);
- BASECHILDMAP::set(key, tree);
- order.replace(*tree, pos);
- }
- else
- {
- BASECHILDMAP::set(key, tree);
- order.append(*tree);
- }
- return true;
- }
- virtual bool replace(const char *key, IPropertyTree *tree) override // provides different semantics, used if element being replaced is not to be treated as deleted.
- {
- return set(key, tree);
- }
- virtual bool remove(const char *key) override
- {
- IPropertyTree *child = BASECHILDMAP::find(*key);
- if (!child)
- return false;
- order.zap(*child);
- return BASECHILDMAP::removeExact(child);
- }
- virtual bool removeExact(IPropertyTree *child) override
- {
- order.zap(*child);
- return BASECHILDMAP::removeExact(child);
- }
- };
- public:
- typedef COrderedPTree<BASE_PTREE> SELF;
- COrderedPTree<BASE_PTREE>(const char *name=NULL, byte flags=ipt_none, IPTArrayValue *value=NULL, ChildMap *children=NULL)
- : BASE_PTREE(name, flags|ipt_ordered, value, children) { }
- virtual bool isEquivalent(IPropertyTree *tree) const override { return (NULL != QUERYINTERFACE(tree, COrderedPTree<BASE_PTREE>)); }
- virtual IPropertyTree *create(const char *name=NULL, IPTArrayValue *value=NULL, ChildMap *children=NULL, bool existing=false) override
- {
- return new COrderedPTree<BASE_PTREE>(name, SELF::flags, value, children);
- }
- virtual IPropertyTree *create(MemoryBuffer &mb) override
- {
- IPropertyTree *tree = new COrderedPTree<BASE_PTREE>();
- tree->deserialize(mb);
- return tree;
- }
- virtual void createChildMap() override
- {
- if (SELF::isnocase())
- SELF::children = new COrderedChildMap<ChildMapNC>();
- else
- SELF::children = new COrderedChildMap<ChildMap>();
- }
- };
- IPropertyTree *createPTree(byte flags)
- {
- return createPTree(NULL, flags);
- }
- IPropertyTree *createPTree(const char *name, byte flags)
- {
- switch (flags & (ipt_ordered|ipt_fast|ipt_lowmem))
- {
- case ipt_ordered|ipt_fast:
- return new COrderedPTree<LocalPTree>(name, flags);
- case ipt_ordered|ipt_lowmem:
- return new COrderedPTree<CAtomPTree>(name, flags);
- case ipt_ordered:
- return new COrderedPTree<DEFAULT_PTREE_TYPE>(name, flags);
- case ipt_fast:
- return new LocalPTree(name, flags);
- case ipt_lowmem:
- return new CAtomPTree(name, flags);
- case 0:
- return new DEFAULT_PTREE_TYPE(name, flags);
- default:
- throwUnexpectedX("Invalid flags - ipt_fast and ipt_lowmem should not be specified together");
- }
- }
- typedef enum _ptElementType
- {
- elementTypeUnknown,
- elementTypeNull,
- elementTypeString,
- elementTypeBool,
- elementTypeInteger,
- elementTypeReal,
- elementTypeObject,
- elementTypeArray
- } ptElementType;
- template <typename X>
- class CJSONReaderBase : public CommonReaderBase<X>
- {
- public:
- typedef CommonReaderBase<X> PARENT;
- using PARENT::reset;
- using PARENT::nextChar;
- using PARENT::readNextToken;
- using PARENT::checkReadNext;
- using PARENT::checkStartReadNext;
- using PARENT::readNext;
- using PARENT::expecting;
- using PARENT::match;
- using PARENT::error;
- using PARENT::skipWS;
- using PARENT::rewind;
- using PARENT::ignoreWhiteSpace;
- CJSONReaderBase(ISimpleReadStream &_stream, IPTreeNotifyEvent &_iEvent, PTreeReaderOptions _readerOptions, size32_t _bufSize=0) :
- CommonReaderBase<X>(_stream, _iEvent, _readerOptions, _bufSize)
- {
- }
- CJSONReaderBase(const void *_buf, size32_t bufLength, IPTreeNotifyEvent &_iEvent, PTreeReaderOptions _readerOptions) :
- CommonReaderBase<X>(_buf, bufLength, _iEvent, _readerOptions)
- {
- }
- CJSONReaderBase(const void *_buf, IPTreeNotifyEvent &_iEvent, PTreeReaderOptions _readerOptions) :
- CommonReaderBase<X>(_buf, _iEvent, _readerOptions)
- {
- }
- ~CJSONReaderBase()
- {
- }
- protected:
- inline StringBuffer &appendChar(StringBuffer &id, char c)
- {
- int charlen = validJSONUtf8ChrLen(c);
- if (!charlen)
- error("invalid JSON character", true);
- id.append(nextChar);
- while (--charlen)
- {
- readNext();
- id.append(nextChar);
- }
- return id;
- }
- void readString(StringBuffer &value)
- {
- readNext();
- StringBuffer s;
- bool decode=false;
- while ('\"'!=nextChar)
- {
- if (nextChar=='\\')
- decode=true;
- appendChar(s, nextChar);
- readNext();
- }
- size32_t r = s.length();
- if (ignoreWhiteSpace)
- s.trimRight();
- if (decode)
- _decodeJSON(r, s.str(), value, s.length()+1);
- else
- value.swapWith(s);
- }
- void readName(StringBuffer &name)
- {
- if ('\"'!=nextChar)
- expecting("\"");
- readString(name);
- if (!name.length())
- error("empty JSON id");
- readNext();
- skipWS();
- if (':'!=nextChar)
- expecting(":");
- readNext();
- }
- ptElementType readValue(StringBuffer &value)
- {
- ptElementType type = elementTypeUnknown;
- switch (nextChar)
- {
- case '\"':
- {
- readString(value);
- type = elementTypeString;
- break;
- }
- case 't':
- match("rue", "Bad value");
- value.append("true");
- type = elementTypeBool;
- break;
- case 'f':
- match("alse", "Bad value");
- value.append("false");
- type = elementTypeBool;
- break;
- case 'n':
- match("ull", "Bad value");
- type = elementTypeNull;
- break;
- case '-':
- value.append(nextChar);
- readNext();
- //fall through
- default:
- if (!isdigit(nextChar))
- error("Bad value");
- type = elementTypeInteger;
- bool exponent = false;
- while (isdigit(nextChar) || '.'==nextChar || 'e'==nextChar || 'E'==nextChar)
- {
- if ('e'==nextChar || 'E'==nextChar)
- {
- if (exponent)
- error("Bad value");
- exponent=true;
- value.append(nextChar);
- readNext();
- if ('-'==nextChar)
- type=elementTypeReal;
- else if (!isdigit(nextChar) && '+'!=nextChar)
- error("Bad value");
- }
- if ('.'==nextChar)
- {
- if (exponent || type==elementTypeReal) //already found decimal
- error("Bad value");
- type = elementTypeReal;
- }
- value.append(nextChar);
- readNext();
- }
- rewind(1);
- break;
- }
- return type;
- }
- const char *_decodeJSON(unsigned read, const char *startMark, StringBuffer &ret, unsigned len)
- {
- const char *errMark = NULL;
- try { return decodeJSON(startMark, ret, len, &errMark); }
- catch (IException *e)
- {
- if (errMark)
- {
- if (read>(unsigned)(errMark-startMark))
- rewind((unsigned)(read - (errMark-startMark)));
- else
- rewind((unsigned)(errMark-startMark));
- }
- StringBuffer errMsg;
- e->errorMessage(errMsg);
- e->Release();
- error(errMsg.str());
- }
- return NULL; // will never get here.
- }
- };
- template <class X>
- class CJSONReader : public CJSONReaderBase<X>, implements IPTreeReader
- {
- typedef CJSONReaderBase<X> PARENT;
- using PARENT::checkBOM;
- using PARENT::rewind;
- using PARENT::readNext;
- using PARENT::readValue;
- using PARENT::readName;
- using PARENT::checkReadNext;
- using PARENT::checkSkipWS;
- using PARENT::checkStartReadNext;
- using PARENT::expecting;
- using PARENT::error;
- using PARENT::eos;
- using PARENT::_decodeJSON;
- using PARENT::skipWS;
- using PARENT::nextChar;
- using PARENT::curOffset;
- using PARENT::noRoot;
- using PARENT::ignoreWhiteSpace;
- using PARENT::iEvent;
- // StringBuffer tmpStr;
- public:
- IMPLEMENT_IINTERFACE;
- CJSONReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions, size32_t bufSize=0)
- : PARENT(stream, iEvent, readerOptions, bufSize)
- {
- }
- CJSONReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions)
- : PARENT(buf, bufLength, iEvent, readerOptions)
- {
- }
- CJSONReader(const void *buf, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions)
- : PARENT(buf, iEvent, readerOptions)
- {
- }
- void readValueNotify(const char *name, bool skipAttributes, StringBuffer *retValue, bool *isValueBinary)
- {
- offset_t startOffset = curOffset;
- StringBuffer value;
- if (readValue(value)!=elementTypeNull)
- {
- if ('@'==*name)
- {
- if (!skipAttributes)
- iEvent->newAttribute(name, value.str());
- return;
- }
- else if ('#'==*name)
- {
- dbgassertex(retValue && isValueBinary);
- *isValueBinary = false;
- if (0 == strncmp(name+1, "value", 5)) // this is a special IPT JSON prop name, representing a 'complex' value
- {
- if ('\0' == *(name+6)) // #value
- {
- retValue->swapWith(value);
- return;
- }
- else if (streq(name+6, "bin")) // #valuebin
- {
- *isValueBinary = true;
- JBASE64_Decode(value.str(), *retValue);
- return;
- }
- }
- }
- }
- iEvent->beginNode(name, false, startOffset);
- iEvent->beginNodeContent(name);
- iEvent->endNode(name, value.length(), value.str(), false, curOffset);
- }
- void readArray(const char *name)
- {
- if ('@'==*name)
- name++;
- readNext();
- skipWS();
- while (']' != nextChar)
- {
- switch (nextChar)
- {
- case '[':
- iEvent->beginNode(name, false, curOffset);
- iEvent->beginNodeContent(name);
- readArray(name);
- iEvent->endNode(name, 0, "", false, curOffset);
- break;
- case '{':
- readObject(name);
- break;
- default:
- readValueNotify(name, true, nullptr, nullptr);
- break;
- }
- readNext();
- skipWS();
- if (','==nextChar)
- readNext();
- else if (']'!=nextChar)
- error("expected ',' or ']'");
- skipWS();
- }
- }
- void readChild(const char *name, bool skipAttributes, StringBuffer *value, bool *isValueBinary)
- {
- skipWS();
- switch (nextChar)
- {
- case '}':
- {
- VStringBuffer msg("named item with no value defined %s [%d]", name, (int) curOffset);
- error(msg.str());
- }
- break;
- case '{':
- readObject(name);
- break;
- case '[':
- readArray(name);
- break;
- default:
- readValueNotify(name, skipAttributes, value, isValueBinary);
- break;
- }
- }
- void readObject(const char *name)
- {
- if ('@'==*name)
- name++;
- iEvent->beginNode(name, false, curOffset);
- readNext();
- skipWS();
- bool attributesFinalized=false;
- StringBuffer childValue; // for #value
- bool isChildValueBinary = false; // for #value
- while ('}' != nextChar)
- {
- StringBuffer tagName;
- readName(tagName);
- //internal convention so we can convert to and from xml
- //values at top of object with names starting with '@' become ptree attributes
- if (*tagName.str()!='@')
- attributesFinalized=true;
- readChild(tagName.str(), attributesFinalized, &childValue.clear(), &isChildValueBinary);
- readNext();
- skipWS();
- if (','==nextChar)
- readNext();
- else if ('}'!=nextChar)
- error("expected ',' or '}'");
- skipWS();
- }
- iEvent->endNode(name, childValue.length(), childValue.str(), isChildValueBinary, curOffset);
- }
- void loadJSON()
- {
- if (!checkStartReadNext())
- return;
- if (checkBOM() && !checkReadNext())
- return;
- if (!checkSkipWS())
- return;
- if (noRoot)
- {
- StringBuffer tagName;
- for (;;)
- {
- switch (nextChar)
- {
- case '\"': //treat named objects like we're in a noroot object
- readName(tagName.clear());
- readChild(tagName.str(), true, nullptr, nullptr);
- break;
- case '{': //treat unnamed objects like we're in a noroot array
- readObject("__object__");
- break;
- case '[': //treat unnamed arrays like we're in a noroot array
- iEvent->beginNode("__array__", false, curOffset);
- readArray("__item__");
- iEvent->endNode("__array__", 0, "", false, curOffset);
- break;
- default:
- expecting("{[ or \"");
- }
- if (!checkReadNext() || !checkSkipWS())
- break;
- switch (nextChar)
- {
- case '{': //support file formats with whitespace (usually \n) seperated objects at the root
- case '[':
- break;
- case ',':
- readNext();
- skipWS();
- break;
- default:
- expecting(",");
- break;
- }
- }
- }
- else
- {
- if ('{' == nextChar)
- readObject("__object__");
- else if ('[' == nextChar)
- {
- iEvent->beginNode("__array__", false, curOffset);
- readArray("__item__");
- iEvent->endNode("__array__", 0, "", false, curOffset);
- }
- else
- error("expected '{' or '['");
- if (checkReadNext() && checkSkipWS())
- error("trailing content after JSON closed");
- }
- }
- // IPTreeReader
- virtual void load() { loadJSON(); }
- virtual offset_t queryOffset() { return curOffset; }
- };
- template <class X>
- class CPullJSONReader : public CJSONReaderBase<X>, implements IPullPTreeReader
- {
- typedef CJSONReaderBase<X> PARENT;
- using PARENT::checkBOM;
- using PARENT::rewind;
- using PARENT::readNext;
- using PARENT::readValue;
- using PARENT::readName;
- using PARENT::checkReadNext;
- using PARENT::checkSkipWS;
- using PARENT::checkStartReadNext;
- using PARENT::expecting;
- using PARENT::error;
- using PARENT::eos;
- using PARENT::_decodeJSON;
- using PARENT::skipWS;
- using PARENT::nextChar;
- using PARENT::curOffset;
- using PARENT::noRoot;
- using PARENT::ignoreWhiteSpace;
- using PARENT::iEvent;
- class CStateInfo : public CInterface
- {
- public:
- CStateInfo()
- {
- tag.ensureCapacity(15);
- type = elementTypeUnknown;
- childCount = 0;
- wnsTag = NULL;
- }
- inline void reset()
- {
- wnsTag = NULL;
- tag.clear();
- tagText.clear();
- type = elementTypeUnknown;
- childCount = 0;
- }
- StringBuffer tag;
- StringBuffer tagText;
- ptElementType type;
- const char *wnsTag;
- unsigned childCount;
- };
- CICopyArrayOf<CStateInfo> stack, freeStateInfo;
- CStateInfo *stateInfo;
- enum ParseStates { headerStart, nameStart, valueStart, itemStart, objAttributes, itemContent, itemEnd } state;
- bool endOfRoot;
- bool preReadItemName;
- bool more;
- StringBuffer tag, value;
- void init()
- {
- state = headerStart;
- stateInfo = NULL;
- endOfRoot = false;
- preReadItemName = false;
- more = true;
- }
- virtual void resetState()
- {
- stack.kill();
- more = true;
- }
- public:
- IMPLEMENT_IINTERFACE;
- CPullJSONReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions, size32_t bufSize=0)
- : CJSONReaderBase<X>(stream, iEvent, readerOptions, bufSize)
- {
- init();
- }
- CPullJSONReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions)
- : CJSONReaderBase<X>(buf, bufLength, iEvent, readerOptions)
- {
- init();
- }
- CPullJSONReader(const void *buf, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions)
- : CJSONReaderBase<X>(buf, iEvent, readerOptions)
- {
- init();
- }
- ~CPullJSONReader()
- {
- ForEachItemIn(i, stack)
- delete &stack.item(i);
- ForEachItemIn(i2, freeStateInfo)
- delete &freeStateInfo.item(i2);
- }
- inline void checkDelimiter(const char *msg=",")
- {
- if (stateInfo && stateInfo->childCount > 0)
- {
- if (','!=nextChar)
- expecting(msg);
- readNext();
- skipWS();
- }
- }
- inline ptElementType getParentType()
- {
- if (stack.ordinality()<2)
- return stateInfo->type;
- return ((CStateInfo *)&stack.tos(1))->type;
- }
- void beginNode(const char *name, offset_t offset, ptElementType jsonType, bool notify=true)
- {
- if (stateInfo)
- stateInfo->childCount++;
- if (freeStateInfo.ordinality())
- {
- stateInfo = &freeStateInfo.popGet();
- stateInfo->reset();
- }
- else
- stateInfo = new CStateInfo;
- stack.append(*stateInfo);
- stateInfo->type=jsonType;
- if (name)
- stateInfo->tag.set(name);
- else
- stateInfo->tag.swapWith(tag);
- stateInfo->wnsTag = stateInfo->tag.str();
- if (!notify)
- return;
- try
- {
- iEvent->beginNode(stateInfo->wnsTag, false, offset);
- }
- catch (IPTreeException *pe)
- {
- if (PTreeExcpt_InvalidTagName == pe->errorCode())
- {
- pe->Release();
- StringBuffer msg("Expecting valid start tag, but got \"");
- error(msg.append(name).append("\"").str());
- }
- throw;
- }
- }
- inline const char *arrayItemName(const char *defaultName)
- {
- if (stack.ordinality()>1)
- return stateInfo->wnsTag;
- return defaultName;
- }
- bool arrayItem(offset_t offset)
- {
- skipWS();
- switch (nextChar)
- {
- case ']':
- state=itemContent;
- if (stack.ordinality()>1)
- readNext();
- if (!endNode(curOffset, getParentType()==elementTypeArray))
- return false;
- break;
- case '{':
- state=objAttributes;
- readNext();
- beginNode(arrayItemName("__object__"), offset, elementTypeObject);
- break;
- case '[':
- state=valueStart;
- readNext();
- beginNode(arrayItemName("__array__"), offset, elementTypeArray, true);
- break;
- default:
- state=valueStart;
- ptElementType type = readValue(value.clear());
- readNext();
- beginNode(arrayItemName("__item__"), offset, type, true);
- stateInfo->tagText.swapWith(value);
- break;
- }
- return true;
- }
- void namedItem()
- {
- if (!preReadItemName)
- readName(tag.clear());
- else
- preReadItemName = false;
- skipWS();
- switch (nextChar)
- {
- case '}':
- error("unexpected object close marker");
- case ']':
- error("unexpected array close marker");
- case '{':
- state=objAttributes;
- readNext();
- beginNode(NULL, curOffset, elementTypeObject);
- break;
- case '[':
- readNext();
- beginNode(NULL, curOffset, elementTypeArray, false); //false because items present events, not the array
- arrayItem(curOffset); //so process the first item so every next() has event
- break;
- default:
- state=valueStart;
- ptElementType type = readValue(value.clear());
- readNext();
- beginNode(NULL, curOffset, type);
- stateInfo->tagText.swapWith(value);
- break;
- }
- }
- void rootItem()
- {
- if ('\"'==nextChar)
- namedItem();
- else if ('{'==nextChar || '['==nextChar)
- arrayItem(curOffset);
- else
- expecting("[{ or \"");
- }
- bool rootNext()
- {
- if (!noRoot)
- return false;
- if (!checkReadNext() || !checkSkipWS())
- return true;
- switch (nextChar)
- {
- case '{': //support files where root level objects are separated by whitespace (usually \n)
- case '[':
- case ',':
- break;
- default:
- expecting(",");
- }
- return true;
- }
- void newNamedAttribute()
- {
- skipWS();
- readValue(value.clear());
- readNext();
- stateInfo->childCount++;
- iEvent->newAttribute(tag.str(), value.str());
- }
- bool endNode(offset_t offset, bool notify=true)
- {
- if (stack.ordinality()<2)
- {
- state = headerStart;
- more = rootNext();
- }
- if (notify)
- {
- if (stateInfo->type==elementTypeNull)
- iEvent->endNode(stateInfo->wnsTag, 0, "", false, offset);
- else
- iEvent->endNode(stateInfo->wnsTag, stateInfo->tagText.length(), stateInfo->tagText.str(), false, offset);
- }
- freeStateInfo.append(*stateInfo);
- stack.pop();
- stateInfo = (stack.ordinality()) ? &stack.tos() : NULL;
- return true;
- }
- // IPullPTreeReader
- virtual void load()
- {
- while (next()) {}
- }
- virtual void reset()
- {
- PARENT::reset();
- resetState();
- }
- virtual offset_t queryOffset() { return curOffset; }
- virtual bool next()
- {
- if (!more)
- return false;
- checkStartReadNext();
- checkSkipWS();
- switch (state)
- {
- case headerStart:
- {
- if (nextChar!='{' && nextChar!='[') //already positioned at start
- {
- if (!checkReadNext())
- return false;
- if (checkBOM())
- if (!checkReadNext())
- return false;
- if (!checkSkipWS())
- return false;
- }
- if (noRoot)
- rootItem();
- else
- {
- switch (nextChar)
- {
- case '{':
- state=objAttributes;
- readNext();
- beginNode("__object__", curOffset, elementTypeObject);
- break;
- case '[':
- state=valueStart;
- readNext();
- beginNode("__array__", curOffset, elementTypeArray);
- break;
- default:
- expecting("{ or [");
- break;
- }
- }
- break;
- }
- case nameStart:
- namedItem();
- break;
- case objAttributes:
- {
- if ('}'==nextChar)
- {
- state=itemEnd;
- iEvent->beginNodeContent(stateInfo->wnsTag);
- break;
- }
- checkDelimiter(", or }");
- if (nextChar != '\"')
- expecting("\"");
- readName(tag.clear());
- if (tag.charAt(0)=='@')
- newNamedAttribute();
- else
- {
- preReadItemName = true;
- state=itemContent;
- stateInfo->childCount=0;
- iEvent->beginNodeContent(stateInfo->wnsTag);
- }
- break;
- }
- case valueStart:
- state=itemContent;
- iEvent->beginNodeContent(stateInfo->wnsTag);
- break;
- case itemContent:
- {
- switch (stateInfo->type)
- {
- case elementTypeBool:
- case elementTypeString:
- case elementTypeInteger:
- case elementTypeReal:
- case elementTypeNull:
- return endNode(curOffset);
- break;
- case elementTypeArray:
- if (']'!=nextChar)
- checkDelimiter(", or ]");
- return arrayItem(curOffset);
- case elementTypeObject:
- if ('}'!=nextChar)
- {
- checkDelimiter(", or }");
- namedItem();
- }
- else
- {
- if (stack.ordinality()>1)
- readNext();
- return endNode(curOffset);
- }
- break;
- }
- break;
- }
- case itemEnd:
- {
- if (!stack.length())
- {
- if (!noRoot || !rootNext())
- return false;
- readNext();
- skipWS();
- rootItem();
- }
- else
- {
- readNext();
- state = itemContent;
- return endNode(curOffset);
- }
- break;
- }
- }
- return true;
- }
- };
- IPTreeReader *createJSONStreamReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions, size32_t bufSize)
- {
- class CJSONStreamReader : public CJSONReader<CInstStreamReader>
- {
- public:
- CJSONStreamReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions, size32_t bufSize=0) : CJSONReader<CInstStreamReader>(stream, iEvent, readerOptions, bufSize) { }
- };
- return new CJSONStreamReader(stream, iEvent, readerOptions, bufSize);
- }
- IPTreeReader *createJSONStringReader(const char *json, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions)
- {
- class CJSONStringReader : public CJSONReader<CInstStringReader>
- {
- public:
- CJSONStringReader(const void *json, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions) : CJSONReader<CInstStringReader>(json, iEvent, readerOptions) { }
- };
- if (NULL == json)
- throw createPTreeReadException(PTreeRead_syntax, "Null string passed to createJSONStringReader", NULL, 0, 0);
- return new CJSONStringReader(json, iEvent, readerOptions);
- }
- IPTreeReader *createJSONBufferReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions)
- {
- class CJSONBufferReader : public CJSONReader<CInstBufferReader>
- {
- public:
- CJSONBufferReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions) : CJSONReader<CInstBufferReader>(buf, bufLength, iEvent, readerOptions) { }
- };
- return new CJSONBufferReader(buf, bufLength, iEvent, readerOptions);
- }
- IPullPTreeReader *createPullJSONStreamReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions, size32_t bufSize)
- {
- class CJSONStreamReader : public CPullJSONReader<CInstStreamReader>
- {
- public:
- CJSONStreamReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions, size32_t bufSize=0) : CPullJSONReader<CInstStreamReader>(stream, iEvent, readerOptions, bufSize) { }
- };
- return new CJSONStreamReader(stream, iEvent, readerOptions, bufSize);
- }
- IPullPTreeReader *createPullJSONStringReader(const char *json, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions)
- {
- class CJSONStringReader : public CPullJSONReader<CInstStringReader>
- {
- public:
- CJSONStringReader(const void *json, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions) : CPullJSONReader<CInstStringReader>(json, iEvent, readerOptions) { }
- };
- return new CJSONStringReader(json, iEvent, readerOptions);
- }
- IPullPTreeReader *createPullJSONBufferReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions)
- {
- class CJSONBufferReader : public CPullJSONReader<CInstBufferReader>
- {
- public:
- CJSONBufferReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions) : CPullJSONReader<CInstBufferReader>(buf, bufLength, iEvent, readerOptions) { }
- };
- return new CJSONBufferReader(buf, bufLength, iEvent, readerOptions);
- }
- IPropertyTree *createPTreeFromJSONString(const char *json, byte flags, PTreeReaderOptions readFlags, IPTreeMaker *iMaker)
- {
- Owned<IPTreeMaker> _iMaker;
- if (!iMaker)
- {
- iMaker = createDefaultPTreeMaker(flags, readFlags);
- _iMaker.setown(iMaker);
- }
- Owned<IPTreeReader> reader = createJSONStringReader(json, *iMaker, readFlags);
- reader->load();
- return LINK(iMaker->queryRoot());
- }
- IPropertyTree *createPTreeFromJSONString(unsigned len, const char *json, byte flags, PTreeReaderOptions readFlags, IPTreeMaker *iMaker)
- {
- Owned<IPTreeMaker> _iMaker;
- if (!iMaker)
- {
- iMaker = createDefaultPTreeMaker(flags, readFlags);
- _iMaker.setown(iMaker);
- }
- Owned<IPTreeReader> reader = createJSONBufferReader(json, len, *iMaker, readFlags);
- reader->load();
- return LINK(iMaker->queryRoot());
- }
- static const char * nextHttpParameterTag(StringBuffer &tag, const char *path)
- {
- while (*path=='.')
- path++;
- const char *finger = strchr(path, '.');
- if (finger)
- {
- tag.clear().append(finger - path, path);
- finger++;
- }
- else
- tag.set(path);
- return finger;
- }
- static void ensureHttpParameter(IPropertyTree *pt, StringBuffer &tag, const char *path, const char *value, const char *fullpath)
- {
- if (!tag.length())
- return;
- unsigned idx = 1;
- if (path && isdigit(*path))
- {
- StringBuffer pos;
- path = nextHttpParameterTag(pos, path);
- idx = (unsigned) atoi(pos.str())+1;
- }
- if ('@'==*tag)
- {
- if (path && *path)
- throw MakeStringException(-1, "'@' not allowed in parent node of parameter path: %s", fullpath);
- pt->setProp(tag, value);
- return;
- }
- if (tag.charAt(tag.length()-1)=='$')
- {
- if (path && *path)
- throw MakeStringException(-1, "'$' not allowed in parent node of parameter path: %s", fullpath);
- tag.setLength(tag.length()-1);
- StringArray values;
- values.appendList(value, "\r");
- ForEachItemIn(pos, values)
- {
- const char *itemValue = values.item(pos);
- while (*itemValue=='\n')
- itemValue++;
- pt->addProp(tag, itemValue);
- }
- return;
- }
- unsigned count = pt->getCount(tag);
- while (count++ < idx)
- pt->addPropTree(tag, createPTree(tag));
- StringBuffer xpath(tag);
- xpath.append('[').append(idx).append(']');
- pt = pt->queryPropTree(xpath);
- if (!path || !*path)
- {
- pt->setProp(NULL, value);
- return;
- }
- StringBuffer nextTag;
- path = nextHttpParameterTag(nextTag, path);
- ensureHttpParameter(pt, nextTag, path, value, fullpath);
- }
- static void ensureHttpParameter(IPropertyTree *pt, const char *path, const char *value)
- {
- const char *fullpath = path;
- StringBuffer tag;
- path = nextHttpParameterTag(tag, path);
- ensureHttpParameter(pt, tag, path, value, fullpath);
- }
- bool checkParseUrlPathNodeValue(const char *s, StringBuffer &name, StringAttr &value)
- {
- s = skipWhitespace(s);
- const char *pn = strchr(s, '(');
- if (pn) //strict format param('value') so we can extend later
- {
- const char *vp = pn + 1;
- if (*vp!='\'')
- return false;
- const char *end =strchr(++vp, '\'');
- if (!end || *(end+1)!=')')
- return false;
- if (!validateXMLTag(name.append(pn-s, s).trim()))
- return false;
- value.set(vp, end-vp);
- }
- else
- {
- if (!validateXMLTag(name.append(s).trim()))
- return false;
- }
- return true;
- }
- IPropertyTree *createPTreeFromHttpPath(const char *nameWithAttrs, IPropertyTree *content, bool nestedRoot, ipt_flags flags)
- {
- StringArray nameAttrList;
- nameAttrList.appendList(nameWithAttrs, "/");
- if (!nameAttrList.ordinality())
- return NULL;
- Owned<IPropertyTree> pt = createPTree(nameAttrList.item(0), flags);
- for (aindex_t pos=1; nameAttrList.isItem(pos); pos++)
- {
- StringBuffer name;
- StringAttr value;
- if (!checkParseUrlPathNodeValue(nameAttrList.item(pos), name, value))
- throw MakeStringException(-1, "Invalid URL parameter format %s", nameAttrList.item(pos));
- StringBuffer xpath("@");
- xpath.append(name.str());
- if (!value.get())
- pt->setPropBool(xpath, true);
- else
- pt->setProp(xpath, value);
- }
- IPropertyTree *parent = pt;
- const char *input = pt->queryProp("@input");
- if (input)
- {
- StringArray inputNodes;
- inputNodes.appendList(input, ".");
- ForEachItemIn(in, inputNodes)
- {
- const char *tag = inputNodes.item(in);
- if (!validateXMLTag(tag))
- throw MakeStringException(-1, "Invalid REST query input specifier %s", input);
- parent = parent->addPropTree(tag, createPTree(tag, flags));
- }
- }
- if (streq("__array__", content->queryName()))
- {
- Owned<IAttributeIterator> aiter = content->getAttributes();
- ForEach (*aiter)
- parent->addProp(aiter->queryName(), aiter->queryValue());
- Owned<IPropertyTreeIterator> iter = content->getElements("__item__");
- ForEach (*iter)
- {
- IPropertyTree &e = iter->query();
- e.renameProp("/", "Row");
- parent->addPropTree("Row", LINK(&e));
- }
- }
- else
- mergePTree(parent, content);
- if (nestedRoot)
- {
- Owned<IPropertyTree> root = createPTree(flags);
- root->setPropTree(nameAttrList.item(0), pt.getClear());
- return root.getClear();
- }
- return pt.getClear();
- }
- //URL node nameWithAttrs is of the form: "TagName/attr1('abc')/attr2/attr3('xyz')"
- IPropertyTree *createPTreeFromHttpParameters(const char *nameWithAttrs, IProperties *parameters, bool skipLeadingDotParameters, bool nestedRoot, ipt_flags flags)
- {
- Owned<IPropertyTree> content = createPTree("content", flags);
- Owned<IPropertyIterator> iter = parameters->getIterator();
- ForEach(*iter)
- {
- StringBuffer key(iter->getPropKey());
- if (!key.length() || key.charAt(key.length()-1)=='!')
- continue;
- if (skipLeadingDotParameters && key.charAt(0)=='.')
- continue;
- const char *value = parameters->queryProp(key);
- if (!value || !*value)
- continue;
- ensureHttpParameter(content, key, value);
- }
- return createPTreeFromHttpPath(nameWithAttrs, content.getClear(), nestedRoot, flags);
- }
- IPropertyTree *createPTreeFromJSONFile(const char *filename, byte flags, PTreeReaderOptions readFlags, IPTreeMaker *iMaker)
- {
- Owned<IFile> in = createIFile(filename);
- if (!in->exists())
- return nullptr;
- StringBuffer contents;
- try
- {
- contents.loadFile(in);
- }
- catch (IException * e)
- {
- EXCLOG(e);
- e->Release();
- return nullptr;
- }
- return createPTreeFromJSONString(contents.length(), contents.str(), flags, readFlags, iMaker);
- }
- //---------------------------------------------------------------------------------------------------------------------
- static constexpr const char * currentVersion = "1.0";
- //---------------------------------------------------------------------------------------------------------------------
- /*
- * Use source to overwrite any changes in target
- * Attributes are replaced
- * Singleton elements are replaced.
- * Entire arrays of scalar elements are replaced.
- * Entire arrays of elements with no name attribute are replaced.
- * Elements with a name attribute are matched by name. If there is a match it is merged. If there is no match it is added.
- */
- static bool checkInSequence(IPropertyTree & child, StringAttr &seqname, bool &first, bool &endprior)
- {
- first = false;
- endprior = false;
- if (seqname.length() && streq(seqname, child.queryName()))
- return true;
- endprior = !seqname.isEmpty();
- if (child.isArray(nullptr))
- {
- first=true;
- seqname.set(child.queryName());
- return true;
- }
- seqname.clear();
- return false;
- }
- inline bool isScalarItem(IPropertyTree &child)
- {
- if (child.hasChildren())
- return false;
- return child.getAttributeCount()==0;
- }
- static IPropertyTree *ensureMergeConfigTarget(IPropertyTree &target, const char *tag, const char *nameAttribute, const char *name, bool sequence)
- {
- StringBuffer tempPath;
- const char * path = (sequence) ? nullptr : tag;
- if (name && nameAttribute && *nameAttribute)
- {
- tempPath.append(tag).append("[").append(nameAttribute).append("=\'").append(name).append("']");
- path = tempPath;
- }
- IPropertyTree * match = (path) ? target.queryPropTree(path) : nullptr;
- if (!match)
- {
- if (sequence)
- match = target.addPropTreeArrayItem(tag, createPTree(tag));
- else
- match = target.addPropTree(tag);
- }
- return match;
- }
- void mergeConfiguration(IPropertyTree & target, const IPropertyTree & source, const char *altNameAttribute, bool overwriteAttr)
- {
- Owned<IAttributeIterator> aiter = source.getAttributes();
- ForEach(*aiter)
- {
- if (overwriteAttr || !target.hasProp(aiter->queryName()))
- target.addProp(aiter->queryName(), aiter->queryValue());
- }
- StringAttr seqname;
- Owned<IPropertyTreeIterator> iter = source.getElements("*");
- ForEach(*iter)
- {
- IPropertyTree & child = iter->query();
- const char * tag = child.queryName();
- const char * name = child.queryProp("@name");
- bool altname = false;
- //Legacy support for old component configuration files that have repeated elements with no name tag but another unique id
- if (!name && altNameAttribute && *altNameAttribute)
- {
- name = child.queryProp(altNameAttribute);
- altname = name!=nullptr;
- }
- bool first = false;
- bool endprior = false;
- bool sequence = checkInSequence(child, seqname, first, endprior);
- if (first && (!name || isScalarItem(child))) //arrays of unamed objects or scalars are replaced
- target.removeProp(tag);
- IPropertyTree * match = ensureMergeConfigTarget(target, tag, altname ? altNameAttribute : "@name", name, sequence);
- mergeConfiguration(*match, child, altNameAttribute, overwriteAttr);
- }
- const char * sourceValue = source.queryProp("");
- target.setProp("", sourceValue);
- }
- /*
- * Load a json/yaml configuration file.
- * If there is an extends tag in the root of the file then this file is applied as a delta to the base file
- * the configuration is the contents of the tag within the file that matches the component tag.
- */
- static IPropertyTree * loadConfiguration(const char * filename, const char * componentTag, bool required, const char *altNameAttribute)
- {
- if (!checkFileExists(filename))
- throw makeStringExceptionV(99, "Configuration file %s not found", filename);
- const char * ext = pathExtension(filename);
- Owned<IPropertyTree> configTree;
- if (!ext || strieq(ext, ".yaml"))
- {
- try
- {
- configTree.setown(createPTreeFromYAMLFile(filename, 0, ptr_ignoreWhiteSpace, nullptr));
- }
- catch (IException *E)
- {
- StringBuffer msg;
- E->errorMessage(msg);
- ::Release(E);
- throw makeStringExceptionV(99, "Error loading configuration file %s (invalid yaml): %s", filename, msg.str());
- }
- }
- else
- throw makeStringExceptionV(99, "Unrecognised file extension %s", ext);
- if (!configTree)
- throw makeStringExceptionV(99, "Error loading configuration file %s", filename);
- IPropertyTree * config = configTree->queryPropTree(componentTag);
- if (!config)
- {
- if (required)
- throw makeStringExceptionV(99, "Section %s is missing from file %s", componentTag, filename);
- return nullptr;
- }
- const char * base = configTree->queryProp("@extends");
- if (!base)
- return LINK(config);
- StringBuffer baseFilename;
- splitFilename(filename, &baseFilename, &baseFilename, nullptr, nullptr, false);
- addNonEmptyPathSepChar(baseFilename);
- baseFilename.append(base);
- Owned<IPropertyTree> baseTree = loadConfiguration(baseFilename, componentTag, required, altNameAttribute);
- mergeConfiguration(*baseTree, *config, altNameAttribute);
- return LINK(baseTree);
- }
- static constexpr const char * envPrefix = "HPCC_CONFIG_";
- static void applyEnvironmentConfig(IPropertyTree & target, const char * cptPrefix, const char * value)
- {
- const char * name = value;
- if (!startsWith(name, envPrefix))
- return;
- name += strlen(envPrefix);
- if (cptPrefix)
- {
- if (!startsWith(name, cptPrefix))
- return;
- name += strlen(cptPrefix);
- if (*name++ != '_')
- return;
- }
- StringBuffer propName;
- if (startsWith(name, "PROP_"))
- {
- propName.append("@");
- name += 5;
- }
- const char * eq = strchr(value, '=');
- if (eq)
- {
- propName.append(eq - name, name);
- target.setProp(propName, eq + 1);
- }
- else
- {
- propName.append(name);
- target.setProp(propName, nullptr);
- }
- }
- IPropertyTree * createPTreeFromYAML(const char * yaml)
- {
- if (*yaml == '{')
- return createPTreeFromJSONString(yaml, 0, ptr_ignoreWhiteSpace, nullptr);
- return createPTreeFromYAMLString(yaml, 0, ptr_ignoreWhiteSpace, nullptr);
- }
- static const char * extractOption(const char * option, const char * cur)
- {
- if (startsWith(cur, option))
- {
- cur += strlen(option);
- if (*cur == '=')
- return cur + 1;
- if (*cur)
- return nullptr;
- return "1";
- }
- return nullptr;
- }
- static void applyCommandLineOption(IPropertyTree * config, const char * option, const char * value)
- {
- //Ignore -- with no following option.
- if (isEmptyString(option))
- return;
- const char *tail;
- while ((tail = strchr(option, '.')) != nullptr)
- {
- StringAttr elemName(option, tail-option);
- if (!config->hasProp(elemName))
- config = config->addPropTree(elemName);
- else
- {
- config = config->queryPropTree(elemName);
- if (!config)
- throw makeStringExceptionV(99, "Cannot overriding scalar configuration element %s with structure", elemName.get());
- }
- option = tail+1;
- }
- if (!validateXMLTag(option))
- throw makeStringExceptionV(99, "Invalid option name '%s'", option);
- StringBuffer path;
- path.append('@').append(option);
- config->setProp(path, value);
- }
- static void applyCommandLineOption(IPropertyTree * config, const char * option, std::initializer_list<const char *> ignoreOptions)
- {
- const char * eq = strchr(option, '=');
- StringBuffer name;
- const char *val = nullptr;
- if (eq)
- {
- name.append(eq - option, option);
- option = name;
- val = eq + 1;
- }
- else
- {
- //MORE: Support --x- and --x+?
- val = "1";
- }
- if (stdContains(ignoreOptions, option))
- return;
- applyCommandLineOption(config, option, val);
- }
- static Owned<IPropertyTree> componentConfiguration;
- static Owned<IPropertyTree> globalConfiguration;
- MODULE_INIT(INIT_PRIORITY_STANDARD)
- {
- return true;
- }
- MODULE_EXIT()
- {
- componentConfiguration.clear();
- globalConfiguration.clear();
- }
- IPropertyTree & queryComponentConfig()
- {
- if (!componentConfiguration)
- throw makeStringException(99, "Configuration file has not yet been processed");
- return *componentConfiguration;
- }
- IPropertyTree & queryGlobalConfig()
- {
- if (!globalConfiguration)
- throw makeStringException(99, "Configuration file has not yet been processed");
- return *globalConfiguration;
- }
- jlib_decl IPropertyTree * loadArgsIntoConfiguration(IPropertyTree *config, const char * * argv, std::initializer_list<const char *> ignoreOptions)
- {
- for (const char * * pArg = argv; *pArg; pArg++)
- {
- const char * cur = *pArg;
- if (startsWith(cur, "--"))
- applyCommandLineOption(config, cur + 2, ignoreOptions);
- }
- return config;
- }
- #ifdef _DEBUG
- static void holdLoop()
- {
- DBGLOG("Component paused for debugging purposes, attach and set held=false to release");
- bool held = true;
- while (held)
- Sleep(5);
- }
- #endif
- jlib_decl IPropertyTree * loadConfiguration(IPropertyTree *componentDefault, const char * * argv, const char * componentTag, const char * envPrefix, const char * legacyFilename, IPropertyTree * (mapper)(IPropertyTree *), const char *altNameAttribute)
- {
- if (componentConfiguration)
- throw makeStringExceptionV(99, "Configuration for component %s has already been initialised", componentTag);
- Linked<IPropertyTree> config(componentDefault);
- const char * optConfig = nullptr;
- bool outputConfig = false;
- #ifdef _DEBUG
- bool held = false;
- #endif
- for (const char * * pArg = argv; *pArg; pArg++)
- {
- const char * cur = *pArg;
- const char * matchConfig = extractOption("--config", cur);
- if (matchConfig)
- optConfig = matchConfig;
- else if (strsame(cur, "--help"))
- {
- #if 0
- //Better not to include this until it has been implemented, since it breaks eclcc
- //MORE: displayHelp(config);
- printf("%s <options>\n", argv[0]);
- exit(0);
- #endif
- }
- else if (strsame(cur, "--init"))
- {
- StringBuffer yamlText;
- toYAML(componentDefault, yamlText, 0, YAML_SortTags);
- printf("%s\n", yamlText.str());
- exit(0);
- }
- else if (strsame(cur, "--outputconfig"))
- {
- outputConfig = true;
- }
- else
- {
- matchConfig = extractOption("--componentTag", cur);
- if (matchConfig)
- componentTag = matchConfig;
- #ifdef _DEBUG
- else
- {
- const char *matchHold = extractOption("--hold", cur);
- if (matchHold)
- {
- if (strToBool(matchHold))
- {
- held = true;
- holdLoop();
- }
- }
- }
- #endif
- }
- }
- Owned<IPropertyTree> delta;
- if (optConfig)
- {
- if (streq(optConfig, "1"))
- throw makeStringExceptionV(99, "Name of configuration file omitted (use --config=<filename>)");
- //--config= with no filename can be used to ignore the legacy configuration file
- if (!isEmptyString(optConfig))
- {
- StringBuffer fullpath;
- if (!isAbsolutePath(optConfig))
- {
- appendCurrentDirectory(fullpath, false);
- addNonEmptyPathSepChar(fullpath);
- }
- fullpath.append(optConfig);
- delta.setown(loadConfiguration(fullpath, componentTag, true, altNameAttribute));
- globalConfiguration.setown(loadConfiguration(fullpath, "global", false, altNameAttribute));
- }
- }
- else
- {
- if (legacyFilename && checkFileExists(legacyFilename))
- delta.setown(createPTreeFromXMLFile(legacyFilename, ipt_caseInsensitive));
- if (delta && mapper)
- delta.setown(mapper(delta));
- }
- if (delta)
- mergeConfiguration(*config, *delta, altNameAttribute);
- const char * * environment = const_cast<const char * *>(getSystemEnv());
- for (const char * * cur = environment; *cur; cur++)
- {
- applyEnvironmentConfig(*config, envPrefix, *cur);
- }
- if (outputConfig)
- {
- loadArgsIntoConfiguration(config, argv, { "config", "outputconfig" });
- Owned<IPropertyTree> recreated = createPTree();
- recreated->setProp("@version", currentVersion);
- recreated->addPropTree(componentTag, LINK(config));
- if (globalConfiguration)
- recreated->addPropTree("global", globalConfiguration.getLink());
- StringBuffer yamlText;
- toYAML(recreated, yamlText, 0, YAML_SortTags);
- printf("%s\n", yamlText.str());
- exit(0);
- }
- else
- loadArgsIntoConfiguration(config, argv);
- //For legacy (and other weird cases) ensure there is a global section
- if (!globalConfiguration)
- globalConfiguration.setown(createPTree("global"));
- #ifdef _DEBUG
- // NB: don't re-hold, if CLI --hold already held.
- if (!held && config->getPropBool("@hold"))
- holdLoop();
- #endif
- unsigned ptreeMappingThreshold = globalConfiguration->getPropInt("@ptreeMappingThreshold", defaultSiblingMapThreshold);
- setPTreeMappingThreshold(ptreeMappingThreshold);
- componentConfiguration.set(config);
- return config.getClear();
- }
- jlib_decl IPropertyTree * loadConfiguration(const char * defaultYaml, const char * * argv, const char * componentTag, const char * envPrefix, const char * legacyFilename, IPropertyTree * (mapper)(IPropertyTree *), const char *altNameAttribute)
- {
- if (componentConfiguration)
- throw makeStringExceptionV(99, "Configuration for component %s has already been initialised", componentTag);
- Owned<IPropertyTree> componentDefault;
- if (defaultYaml)
- {
- Owned<IPropertyTree> defaultConfig = createPTreeFromYAML(defaultYaml);
- componentDefault.set(defaultConfig->queryPropTree(componentTag));
- if (!componentDefault)
- throw makeStringExceptionV(99, "Default configuration does not contain the tag %s", componentTag);
- }
- else
- componentDefault.setown(createPTree(componentTag));
- return loadConfiguration(componentDefault, argv, componentTag, envPrefix, legacyFilename, mapper, altNameAttribute);
- }
- class CYAMLBufferReader : public CInterfaceOf<IPTreeReader>
- {
- protected:
- Linked<IPTreeNotifyEvent> iEvent;
- yaml_parser_t parser;
- PTreeReaderOptions readerOptions = ptr_none;
- bool noRoot = false;
- public:
- CYAMLBufferReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &_iEvent, PTreeReaderOptions _readerOptions) :
- iEvent(&_iEvent), readerOptions(_readerOptions)
- {
- if (!yaml_parser_initialize(&parser))
- throw makeStringException(99, "Filed to initialize libyaml parser");
- yaml_parser_set_input_string(&parser, (const unsigned char *)buf, bufLength);
- noRoot = 0 != ((unsigned)readerOptions & (unsigned)ptr_noRoot);
- }
- ~CYAMLBufferReader()
- {
- yaml_parser_delete(&parser);
- }
- yaml_event_type_t nextEvent(yaml_event_t &event, yaml_event_type_t final=YAML_NO_EVENT, yaml_event_type_t expected=YAML_NO_EVENT, const char *error="")
- {
- if (!yaml_parser_parse(&parser, &event))
- throw makeStringExceptionV(99, "libyaml parser error %s", parser.problem);
- if (event.type!=final && expected!=YAML_NO_EVENT && event.type!=expected)
- throw makeStringExceptionV(99, "libyaml parser %s", error);
- return event.type;
- }
- virtual void loadSequence(const char *tagname)
- {
- if (!tagname || !*tagname) //if unmapped (unnamed) sequences are possible have to decide how to name them in the ptree, later
- throw makeStringException(99, "libyaml parser expected sequence name");
- yaml_event_t event;
- yaml_event_type_t eventType = YAML_NO_EVENT;
- while (eventType!=YAML_SEQUENCE_END_EVENT)
- {
- eventType = nextEvent(event);
- switch (eventType)
- {
- case YAML_MAPPING_START_EVENT: //child map
- loadMap(tagname, true);
- break;
- case YAML_SEQUENCE_START_EVENT:
- //todo
- break;
- case YAML_SCALAR_EVENT:
- iEvent->beginNode(tagname, true, parser.offset);
- iEvent->endNode(tagname, event.data.scalar.length, (const void *)event.data.scalar.value, false, parser.offset);
- break;
- case YAML_ALIAS_EVENT: //reference to an anchor, ignore for now
- iEvent->beginNode(tagname, true, parser.offset);
- iEvent->endNode(tagname, 0, nullptr, false, parser.offset);
- break;
- case YAML_SEQUENCE_END_EVENT: //done
- break;
- case YAML_NO_EVENT:
- case YAML_MAPPING_END_EVENT:
- case YAML_STREAM_START_EVENT:
- case YAML_STREAM_END_EVENT:
- case YAML_DOCUMENT_START_EVENT:
- case YAML_DOCUMENT_END_EVENT:
- default:
- //shouldn't be here
- break;
- }
- yaml_event_delete(&event);
- }
- }
- virtual void loadMap(const char *tagname, bool sequence)
- {
- bool binaryContent = false;
- StringBuffer content;
- if (tagname && *tagname)
- iEvent->beginNode(tagname, sequence, parser.offset);
- yaml_event_t event;
- yaml_event_type_t eventType = YAML_NO_EVENT;
- while (eventType!=YAML_MAPPING_END_EVENT)
- {
- eventType = nextEvent(event, YAML_MAPPING_END_EVENT, YAML_SCALAR_EVENT, "expected map to start with scalar name");
- if (eventType==YAML_MAPPING_END_EVENT)
- {
- yaml_event_delete(&event);
- continue;
- }
- StringBuffer attname('@');
- attname.append(event.data.scalar.length, (const char *)event.data.scalar.value);
- const char *elname = attname.str()+1;
- yaml_event_delete(&event);
- eventType = nextEvent(event);
- switch (eventType)
- {
- case YAML_MAPPING_START_EVENT: //child map
- loadMap(elname, false);
- break;
- case YAML_SEQUENCE_START_EVENT:
- loadSequence(elname);
- break;
- case YAML_SCALAR_EVENT:
- {
- //!el or !element will be our local tag (custom schema type) for an element
- //ptree toYAML should set this for element scalars, and parent text content
- const char *tag = (const char *)event.data.scalar.tag;
- if (tag && (streq(tag, "!binary") || streq(tag, "!!binary")))
- {
- if (streq(elname, "^")) //text content of parent node
- {
- binaryContent = true;
- JBASE64_Decode((const char *) event.data.scalar.value, content.clear());
- }
- else
- {
- StringBuffer decoded;
- JBASE64_Decode((const char *) event.data.scalar.value, decoded);
- iEvent->beginNode(elname, false, parser.offset);
- iEvent->endNode(elname, decoded.length(), (const void *) decoded.str(), true, parser.offset);
- }
- }
- else if (streq(elname, "^")) //text content of parent node
- {
- content.set((const char *) event.data.scalar.value);
- }
- else if (tag && (streq(tag, "!el") || streq(tag, "!element")))
- {
- iEvent->beginNode(elname, false, parser.offset);
- iEvent->endNode(elname, event.data.scalar.length, (const void *) event.data.scalar.value, false, parser.offset);
- }
- else //by default all named scalars are ptree attributes
- {
- iEvent->newAttribute(attname, (const char *)event.data.scalar.value);
- }
- break;
- }
- case YAML_ALIAS_EVENT: //reference to an anchor, ignore for now
- iEvent->beginNode(elname, false, parser.offset);
- iEvent->endNode(elname, 0, nullptr, false, parser.offset);
- break;
- case YAML_MAPPING_END_EVENT: //done
- break;
- case YAML_NO_EVENT:
- case YAML_SEQUENCE_END_EVENT:
- case YAML_STREAM_START_EVENT:
- case YAML_STREAM_END_EVENT:
- case YAML_DOCUMENT_START_EVENT:
- case YAML_DOCUMENT_END_EVENT:
- default:
- //shouldn't be here
- break;
- }
- yaml_event_delete(&event);
- }
- if (tagname && *tagname)
- iEvent->endNode(tagname, content.length(), content, binaryContent, parser.offset);
- }
- virtual void load() override
- {
- yaml_event_t event;
- yaml_event_type_t eventType = YAML_NO_EVENT;
- bool doc = false;
- bool content = false;
- while (eventType!=YAML_STREAM_END_EVENT)
- {
- eventType = nextEvent(event);
- switch (eventType)
- {
- case YAML_MAPPING_START_EVENT:
- //root content, the start of all mappings, should be only one at the root
- if (content)
- throw makeStringException(99, "YAML: Currently only support one content section (map) per stream");
- loadMap(noRoot ? nullptr : "__object__", false); //root map
- content=true;
- break;
- case YAML_SEQUENCE_START_EVENT:
- //root content, sequence (array), should be only one at the root and can't mix with mappings
- if (content)
- throw makeStringException(99, "YAML: Currently only support one content section (sequence) per stream");
- if (!noRoot)
- iEvent->beginNode("__array__", false, 0);
- loadSequence("__item__");
- if (!noRoot)
- iEvent->endNode("__array__", 0, nullptr, false, parser.offset);
- content=true;
- break;
- case YAML_STREAM_START_EVENT:
- case YAML_STREAM_END_EVENT:
- //don't think we need to do anything... unless we start saving hints
- break;
- case YAML_DOCUMENT_START_EVENT:
- //should only support one? multiple documents would imply an extra level of nesting (future flag?)
- if (doc)
- throw makeStringException(99, "YAML: Currently only support one document per stream");
- doc=true;
- break;
- case YAML_DOCUMENT_END_EVENT:
- break;
- case YAML_NO_EVENT:
- case YAML_ALIAS_EVENT: //root alias?
- case YAML_MAPPING_END_EVENT:
- case YAML_SCALAR_EVENT: //root unmapped (unnamed) scalars?
- case YAML_SEQUENCE_END_EVENT:
- //shouldn't be here
- break;
- default:
- break;
- }
- yaml_event_delete(&event);
- }
- }
- virtual offset_t queryOffset() override
- {
- return parser.offset;
- }
- };
- IPTreeReader *createYAMLBufferReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions)
- {
- return new CYAMLBufferReader(buf, bufLength, iEvent, readerOptions);
- }
- IPropertyTree *createPTreeFromYAMLString(unsigned len, const char *yaml, byte flags, PTreeReaderOptions readFlags, IPTreeMaker *iMaker)
- {
- Owned<IPTreeMaker> _iMaker;
- if (!iMaker)
- {
- iMaker = createDefaultPTreeMaker(flags, readFlags);
- _iMaker.setown(iMaker);
- }
- Owned<IPTreeReader> reader = createYAMLBufferReader(yaml, len, *iMaker, readFlags);
- reader->load();
- return LINK(iMaker->queryRoot());
- }
- IPropertyTree *createPTreeFromYAMLString(const char *yaml, byte flags, PTreeReaderOptions readFlags, IPTreeMaker *iMaker)
- {
- return createPTreeFromYAMLString(strlen(yaml), yaml, flags, readFlags, iMaker);
- }
- IPropertyTree *createPTreeFromYAMLFile(const char *filename, byte flags, PTreeReaderOptions readFlags, IPTreeMaker *iMaker)
- {
- Owned<IFile> in = createIFile(filename);
- if (!in->exists())
- return nullptr;
- StringBuffer contents;
- try
- {
- contents.loadFile(in);
- }
- catch (IException * e)
- {
- EXCLOG(e);
- e->Release();
- return nullptr;
- }
- return createPTreeFromYAMLString(contents.length(), contents, flags, readFlags, iMaker);
- }
- static int yaml_write_iiostream(void *data, unsigned char *buffer, size_t size)
- {
- IIOStream *out = (IIOStream *) data;
- out->write(size, (void *)buffer);
- out->flush();
- return 1;
- }
- class YAMLEmitter
- {
- yaml_emitter_t emitter;
- yaml_event_t event;
- IIOStream &out;
- public:
- YAMLEmitter(IIOStream &ios, int indent) : out(ios)
- {
- if (!yaml_emitter_initialize(&emitter))
- throw MakeStringException(0, "YAMLEmitter: failed to initialize");
- yaml_emitter_set_output(&emitter, yaml_write_iiostream, &out);
- yaml_emitter_set_canonical(&emitter, false);
- yaml_emitter_set_unicode(&emitter, true);
- yaml_emitter_set_indent(&emitter, indent);
- beginStream();
- beginDocument();
- }
- ~YAMLEmitter()
- {
- endDocument();
- endStream();
- yaml_emitter_delete(&emitter);
- }
- yaml_char_t *getTag(bool binary, bool element)
- {
- if (binary)
- return (yaml_char_t *) "!binary";
- if (element)
- return (yaml_char_t *) "!el";
- return nullptr;
- }
- void emit()
- {
- yaml_emitter_emit(&emitter, &event);
- }
- void checkInit(int success, const char *descr)
- {
- if (success==0)
- throw MakeStringException(0, "YAMLEmitter: %s failed", descr);
- }
- void writeValue(const char *value, bool element, bool hidden, bool binary)
- {
- yaml_scalar_style_t style = binary ? YAML_LITERAL_SCALAR_STYLE : YAML_ANY_SCALAR_STYLE;
- const yaml_char_t *tag = getTag(binary, element);
- bool implicit = tag==nullptr;
- StringBuffer s;
- if (!value)
- value = "null";
- else if (hidden)
- value = (binary) ? "KioqKg==" : s.appendN(strlen(value), '*').str(); //KioqKg== is base64 of ****
- checkInit(yaml_scalar_event_initialize(&event, nullptr, tag, (yaml_char_t *) value, -1, implicit, implicit, style), "yaml_scalar_event_initialize");
- emit();
- }
- void writeName(const char *name)
- {
- dbgassertex(name!=nullptr);
- return writeValue(name, false, false,false);
- }
- void writeNamedValue(const char *name, const char *value, bool element, bool hidden)
- {
- writeName(name);
- writeValue(value, element, hidden, false);
- }
- void writeAttribute(const char *name, const char *value, bool hidden)
- {
- writeNamedValue(name, value, false, hidden);
- }
- void beginMap()
- {
- checkInit(yaml_mapping_start_event_initialize(&event, nullptr, nullptr, 0, YAML_BLOCK_MAPPING_STYLE), "yaml_mapping_start_event_initialize");
- emit();
- }
- void endMap()
- {
- checkInit(yaml_mapping_end_event_initialize(&event), "yaml_mapping_end_event_initialize");
- emit();
- }
- void beginSequence(const char *name)
- {
- if (name)
- writeName(name);
- checkInit(yaml_sequence_start_event_initialize(&event, nullptr, nullptr, 0, YAML_ANY_SEQUENCE_STYLE), "yaml_sequence_start_event_initialize");
- emit();
- }
- void endSequence()
- {
- checkInit(yaml_sequence_end_event_initialize(&event), "yaml_sequence_end_event_initialize");
- emit();
- }
- void beginDocument()
- {
- checkInit(yaml_document_start_event_initialize(&event, nullptr, nullptr, nullptr, true), "yaml_document_start_event_initialize");
- emit();
- }
- void endDocument()
- {
- checkInit(yaml_document_end_event_initialize(&event, true), "yaml_document_end_event_initialize");
- emit();
- }
- void beginStream()
- {
- checkInit(yaml_stream_start_event_initialize(&event, YAML_UTF8_ENCODING), "yaml_stream_start_event_initialize");
- emit();
- }
- void endStream()
- {
- checkInit(yaml_stream_end_event_initialize(&event), "yaml_stream_end_event_initialize");
- emit();
- }
- };
- static void _toYAML(const IPropertyTree *tree, YAMLEmitter &yaml, byte flags, bool root=false, bool isArrayItem=false)
- {
- const char *name = tree->queryName();
- if (!root && !isArrayItem)
- {
- if (!name || !*name)
- name = "__unnamed__";
- yaml.writeName(name);
- }
- Owned<IAttributeIterator> it = tree->getAttributes(true);
- bool hasAttributes = it->first();
- bool complex = (hasAttributes || tree->hasChildren());
- bool hiddenRootArrayObject = isRootArrayObjectHidden(root, name, flags);
- if (!hiddenRootArrayObject)
- {
- if (complex)
- yaml.beginMap();
- if (hasAttributes)
- {
- ForEach(*it)
- {
- const char *key = it->queryName()+1;
- const char *val = it->queryValue();
- yaml.writeAttribute(key, val, isSanitizedAndHidden(val, flags, true));
- }
- }
- }
- StringBuffer _content;
- const char *content = nullptr; // to avoid uninitialized warning
- bool isBinary = tree->isBinary(NULL);
- bool isNull = true;
- if (!hiddenRootArrayObject)
- {
- if (isBinary)
- {
- MemoryBuffer thislevelbin;
- isNull = (!tree->getPropBin(NULL, thislevelbin))||(thislevelbin.length()==0);
- if (!isNull)
- JBASE64_Encode(thislevelbin.toByteArray(), thislevelbin.length(), _content, true);
- content = _content.str();
- }
- else if (tree->isCompressed(NULL))
- {
- isNull = false; // can't be empty if compressed;
- verifyex(tree->getProp(NULL, _content));
- content = _content.str();
- }
- else
- isNull = (NULL == (content = tree->queryProp(NULL)));
- if (isNull && !root && !complex)
- {
- yaml.writeValue("null", false, false, false);
- return;
- }
- }
- Owned<IPropertyTreeIterator> sub = tree->getElements(hiddenRootArrayObject ? "__item__" : "*", 0 != (flags & YAML_SortTags) ? iptiter_sort : iptiter_null);
- //note that detection of repeating elements relies on the fact that ptree elements
- //of the same name will be grouped together
- StringAttr seqname;
- bool sequence = false;
- ForEach(*sub)
- {
- IPropertyTree &element = sub->query();
- bool first = false;
- bool endprior = false;
- sequence = checkInSequence(element, seqname, first, endprior);
- if (endprior)
- yaml.endSequence();
- if (first)
- yaml.beginSequence(hiddenRootArrayObject ? nullptr : element.queryName());
- _toYAML(&element, yaml, flags, false, sequence);
- }
- if (sequence)
- yaml.endSequence();
- if (!isNull)
- {
- if (complex)
- yaml.writeName("^");
- //repeating/array/sequence items are implicitly elements, no need for tag
- yaml.writeValue(content, isArrayItem ? false : true, isSanitizedAndHidden(content, flags, false), isBinary);
- }
- if (!hiddenRootArrayObject && complex)
- yaml.endMap();
- }
- static void _toYAML(const IPropertyTree *tree, IIOStream &out, unsigned indent, byte flags, bool root=false, bool isArrayItem=false)
- {
- YAMLEmitter yaml(out, indent);
- _toYAML(tree, yaml, flags, true, false);
- }
- jlib_decl StringBuffer &toYAML(const IPropertyTree *tree, StringBuffer &ret, unsigned indent, byte flags)
- {
- CStringBufferMarkupIOAdapter adapter(ret);
- _toYAML(tree->queryBranch(NULL), adapter, indent, flags, true);
- return ret;
- }
- void toYAML(const IPropertyTree *tree, IIOStream &out, unsigned indent, byte flags)
- {
- _toYAML(tree, out, indent, flags, true);
- }
- void printYAML(const IPropertyTree *tree, unsigned indent, unsigned flags)
- {
- StringBuffer yaml;
- toYAML(tree, yaml, indent, flags);
- printf("%s", yaml.str());
- }
- void dbglogYAML(const IPropertyTree *tree, unsigned indent, unsigned flags)
- {
- StringBuffer yaml;
- toYAML(tree, yaml, indent, flags);
- DBGLOG("%s", yaml.str());
- }
- void saveYAML(const char *filename, const IPropertyTree *tree, unsigned indent, unsigned flags)
- {
- OwnedIFile ifile = createIFile(filename);
- saveYAML(*ifile, tree, indent, flags);
- }
- void saveYAML(IFile &ifile, const IPropertyTree *tree, unsigned indent, unsigned flags)
- {
- OwnedIFileIO ifileio = ifile.open(IFOcreate);
- if (!ifileio)
- throw MakeStringException(0, "saveXML: could not find %s to open", ifile.queryFilename());
- saveYAML(*ifileio, tree, indent, flags);
- }
- void saveYAML(IFileIO &ifileio, const IPropertyTree *tree, unsigned indent, unsigned flags)
- {
- Owned<IIOStream> stream = createIOStream(&ifileio);
- stream.setown(createBufferedIOStream(stream));
- saveYAML(*stream, tree, indent, flags);
- }
- void saveYAML(IIOStream &stream, const IPropertyTree *tree, unsigned indent, unsigned flags)
- {
- toYAML(tree, stream, indent, flags);
- }
- jlib_decl IPropertyTree * queryCostsConfiguration()
- {
- return queryComponentConfig().queryPropTree("costs");
- }
- void copyPropIfMissing(IPropertyTree & target, const char * targetName, IPropertyTree & source, const char * sourceName)
- {
- if (source.hasProp(sourceName) && !target.hasProp(targetName))
- {
- if (source.isBinary(sourceName))
- {
- MemoryBuffer value;
- source.getPropBin(sourceName, value);
- target.setPropBin(targetName, value.length(), value.toByteArray());
- }
- else
- target.setProp(targetName, source.queryProp(sourceName));
- }
- }
|