jptree.cpp 280 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853685468556856685768586859686068616862686368646865686668676868686968706871687268736874687568766877687868796880688168826883688468856886688768886889689068916892689368946895689668976898689969006901690269036904690569066907690869096910691169126913691469156916691769186919692069216922692369246925692669276928692969306931693269336934693569366937693869396940694169426943694469456946694769486949695069516952695369546955695669576958695969606961696269636964696569666967696869696970697169726973697469756976697769786979698069816982698369846985698669876988698969906991699269936994699569966997699869997000700170027003700470057006700770087009701070117012701370147015701670177018701970207021702270237024702570267027702870297030703170327033703470357036703770387039704070417042704370447045704670477048704970507051705270537054705570567057705870597060706170627063706470657066706770687069707070717072707370747075707670777078707970807081708270837084708570867087708870897090709170927093709470957096709770987099710071017102710371047105710671077108710971107111711271137114711571167117711871197120712171227123712471257126712771287129713071317132713371347135713671377138713971407141714271437144714571467147714871497150715171527153715471557156715771587159716071617162716371647165716671677168716971707171717271737174717571767177717871797180718171827183718471857186718771887189719071917192719371947195719671977198719972007201720272037204720572067207720872097210721172127213721472157216721772187219722072217222722372247225722672277228722972307231723272337234723572367237723872397240724172427243724472457246724772487249725072517252725372547255725672577258725972607261726272637264726572667267726872697270727172727273727472757276727772787279728072817282728372847285728672877288728972907291729272937294729572967297729872997300730173027303730473057306730773087309731073117312731373147315731673177318731973207321732273237324732573267327732873297330733173327333733473357336733773387339734073417342734373447345734673477348734973507351735273537354735573567357735873597360736173627363736473657366736773687369737073717372737373747375737673777378737973807381738273837384738573867387738873897390739173927393739473957396739773987399740074017402740374047405740674077408740974107411741274137414741574167417741874197420742174227423742474257426742774287429743074317432743374347435743674377438743974407441744274437444744574467447744874497450745174527453745474557456745774587459746074617462746374647465746674677468746974707471747274737474747574767477747874797480748174827483748474857486748774887489749074917492749374947495749674977498749975007501750275037504750575067507750875097510751175127513751475157516751775187519752075217522752375247525752675277528752975307531753275337534753575367537753875397540754175427543754475457546754775487549755075517552755375547555755675577558755975607561756275637564756575667567756875697570757175727573757475757576757775787579758075817582758375847585758675877588758975907591759275937594759575967597759875997600760176027603760476057606760776087609761076117612761376147615761676177618761976207621762276237624762576267627762876297630763176327633763476357636763776387639764076417642764376447645764676477648764976507651765276537654765576567657765876597660766176627663766476657666766776687669767076717672767376747675767676777678767976807681768276837684768576867687768876897690769176927693769476957696769776987699770077017702770377047705770677077708770977107711771277137714771577167717771877197720772177227723772477257726772777287729773077317732773377347735773677377738773977407741774277437744774577467747774877497750775177527753775477557756775777587759776077617762776377647765776677677768776977707771777277737774777577767777777877797780778177827783778477857786778777887789779077917792779377947795779677977798779978007801780278037804780578067807780878097810781178127813781478157816781778187819782078217822782378247825782678277828782978307831783278337834783578367837783878397840784178427843784478457846784778487849785078517852785378547855785678577858785978607861786278637864786578667867786878697870787178727873787478757876787778787879788078817882788378847885788678877888788978907891789278937894789578967897789878997900790179027903790479057906790779087909791079117912791379147915791679177918791979207921792279237924792579267927792879297930793179327933793479357936793779387939794079417942794379447945794679477948794979507951795279537954795579567957795879597960796179627963796479657966796779687969797079717972797379747975797679777978797979807981798279837984798579867987798879897990799179927993799479957996799779987999800080018002800380048005800680078008800980108011801280138014801580168017801880198020802180228023802480258026802780288029803080318032803380348035803680378038803980408041804280438044804580468047804880498050805180528053805480558056805780588059806080618062806380648065806680678068806980708071807280738074807580768077807880798080808180828083808480858086808780888089809080918092809380948095809680978098809981008101810281038104810581068107810881098110811181128113811481158116811781188119812081218122812381248125812681278128812981308131813281338134813581368137813881398140814181428143814481458146814781488149815081518152815381548155815681578158815981608161816281638164816581668167816881698170817181728173817481758176817781788179818081818182818381848185818681878188818981908191819281938194819581968197819881998200820182028203820482058206820782088209821082118212821382148215821682178218821982208221822282238224822582268227822882298230823182328233823482358236823782388239824082418242824382448245824682478248824982508251825282538254825582568257825882598260826182628263826482658266826782688269827082718272827382748275827682778278827982808281828282838284828582868287828882898290829182928293829482958296829782988299830083018302830383048305830683078308830983108311831283138314831583168317831883198320832183228323832483258326832783288329833083318332833383348335833683378338833983408341834283438344834583468347834883498350835183528353835483558356835783588359836083618362836383648365836683678368836983708371837283738374837583768377837883798380838183828383838483858386838783888389839083918392839383948395839683978398839984008401840284038404840584068407840884098410841184128413841484158416841784188419842084218422842384248425842684278428842984308431843284338434843584368437843884398440844184428443844484458446844784488449845084518452845384548455845684578458845984608461846284638464846584668467846884698470847184728473847484758476847784788479848084818482848384848485848684878488848984908491849284938494849584968497849884998500850185028503850485058506850785088509851085118512851385148515851685178518851985208521852285238524852585268527852885298530853185328533853485358536853785388539854085418542854385448545854685478548854985508551855285538554855585568557855885598560856185628563856485658566856785688569857085718572857385748575857685778578857985808581858285838584858585868587858885898590859185928593859485958596859785988599860086018602860386048605860686078608860986108611861286138614861586168617861886198620862186228623862486258626862786288629863086318632863386348635863686378638863986408641864286438644864586468647864886498650865186528653865486558656865786588659866086618662866386648665866686678668866986708671867286738674867586768677867886798680868186828683868486858686868786888689869086918692869386948695869686978698869987008701870287038704870587068707870887098710871187128713871487158716871787188719872087218722872387248725872687278728872987308731873287338734873587368737873887398740874187428743874487458746874787488749875087518752875387548755875687578758875987608761876287638764876587668767876887698770877187728773877487758776877787788779878087818782878387848785878687878788878987908791879287938794879587968797879887998800880188028803880488058806880788088809881088118812881388148815881688178818881988208821882288238824882588268827882888298830883188328833883488358836883788388839884088418842884388448845884688478848884988508851885288538854885588568857885888598860886188628863886488658866886788688869887088718872887388748875887688778878887988808881888288838884888588868887888888898890889188928893889488958896889788988899890089018902890389048905890689078908890989108911891289138914891589168917891889198920892189228923892489258926892789288929893089318932893389348935893689378938893989408941894289438944894589468947894889498950895189528953895489558956895789588959896089618962896389648965896689678968896989708971897289738974897589768977897889798980898189828983898489858986898789888989899089918992899389948995899689978998899990009001900290039004900590069007900890099010901190129013901490159016901790189019902090219022902390249025902690279028902990309031903290339034903590369037903890399040904190429043904490459046904790489049905090519052905390549055905690579058905990609061906290639064906590669067906890699070907190729073907490759076907790789079908090819082908390849085908690879088908990909091909290939094909590969097909890999100910191029103910491059106910791089109911091119112911391149115911691179118911991209121912291239124912591269127912891299130913191329133913491359136913791389139914091419142914391449145914691479148914991509151915291539154915591569157915891599160916191629163916491659166916791689169917091719172917391749175917691779178917991809181918291839184918591869187918891899190919191929193919491959196919791989199920092019202920392049205920692079208920992109211921292139214921592169217921892199220922192229223922492259226922792289229923092319232923392349235923692379238923992409241924292439244924592469247924892499250925192529253925492559256925792589259926092619262
  1. /*##############################################################################
  2. HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. ############################################################################## */
  13. #include <unordered_map>
  14. #include <unordered_set>
  15. #include <string>
  16. #include "platform.h"
  17. #include "jarray.hpp"
  18. #include "jdebug.hpp"
  19. #include "jhash.hpp"
  20. #include "jmutex.hpp"
  21. #include "jexcept.hpp"
  22. #include "jlzw.hpp"
  23. #include "jregexp.hpp"
  24. #include "jstring.hpp"
  25. #include "jutil.hpp"
  26. #include "jmisc.hpp"
  27. #include "yaml.h"
  28. #include <initializer_list>
  29. #define MAKE_LSTRING(name,src,length) \
  30. const char *name = (const char *) alloca((length)+1); \
  31. memcpy((char *) name, (src), (length)); \
  32. *(char *) (name+(length)) = '\0';
  33. #include "jfile.hpp"
  34. #include "jlog.hpp"
  35. #include "jptree.ipp"
  36. #define WARNLEGACYCOMPARE
  37. #define XMLTAG_CONTENT "<>"
  38. #undef UNIMPLEMENTED
  39. #define UNIMPLEMENTED throw MakeIPTException(-1, "UNIMPLEMENTED")
  40. #define CHECK_ATTRIBUTE(X) if (X && isAttribute(X)) throw MakeIPTException(PTreeExcpt_XPath_Unsupported, "Attribute usage invalid here");
  41. #define AMBIGUOUS_PATH(X,P) { StringBuffer buf; buf.append(X": ambiguous xpath \"").append(P).append("\""); throw MakeIPTException(PTreeExcpt_XPath_Ambiguity,"%s",buf.str()); }
  42. #define PTREE_COMPRESS_THRESHOLD (4*1024) // i.e. only use compress if > threshold
  43. #define PTREE_COMPRESS_BOTHER_PECENTAGE (80) // i.e. if it doesn't compress to <80 % of original size don't bother
  44. class NullPTreeIterator final : implements IPropertyTreeIterator
  45. {
  46. public:
  47. virtual ~NullPTreeIterator() {}
  48. virtual void Link() const override {}
  49. virtual bool Release() const override { return true; }
  50. // IPropertyTreeIterator
  51. virtual bool first() override { return false; }
  52. virtual bool next() override { return false; }
  53. virtual bool isValid() override { return false; }
  54. virtual IPropertyTree & query() override { throwUnexpected(); }
  55. } *nullPTreeIterator;
  56. IPropertyTreeIterator *createNullPTreeIterator() { return LINK(nullPTreeIterator); } // initialized in init mod below.
  57. //===================================================================
  58. #ifdef USE_READONLY_ATOMTABLE
  59. RONameTable *AttrStrUnionWithTable::roNameTable = nullptr;
  60. RONameTable *AttrStrUnionWithValueTable::roValueTable = nullptr;
  61. #endif
  62. static AtomRefTable *keyTable = nullptr;
  63. static AtomRefTable *keyTableNC = nullptr;
  64. static CriticalSection hashcrit;
  65. static CAttrValHashTable *attrHT = nullptr;
  66. static AttrValue **freelist = nullptr;
  67. static unsigned freelistmax = 0;
  68. static CLargeMemoryAllocator freeallocator((memsize_t)-1, 0x1000*sizeof(AttrValue), true);
  69. #ifdef USE_READONLY_ATOMTABLE
  70. static const char * roAttributes[] =
  71. {
  72. #include "jptree-attrs.hpp" // potentially auto-generated
  73. nullptr
  74. };
  75. static const char * roAttributeValues[] =
  76. {
  77. #include "jptree-attrvalues.hpp" // potentially auto-generated
  78. nullptr
  79. };
  80. void initializeRoTable()
  81. {
  82. for (const char **attr = roAttributes; *attr; attr++)
  83. {
  84. AttrStrUnionWithTable::roNameTable->find(*attr, true);
  85. }
  86. for (const char **value = roAttributeValues; *value; value++)
  87. {
  88. AttrStrUnionWithValueTable::roValueTable->find(*value, true);
  89. }
  90. // also populate read-only value table by generating some common constants
  91. StringBuffer constStr;
  92. for (unsigned c=0; c<1000; c++) // common unsigned values in attributes
  93. {
  94. constStr.clear().append(c);
  95. AttrStrUnionWithValueTable::roValueTable->find(constStr.str(), true);
  96. }
  97. for (unsigned c=1; c<=400; c++) // outer graphs
  98. {
  99. constStr.clear().append("graph").append(c);
  100. AttrStrUnionWithValueTable::roValueTable->find(constStr.str(), true);
  101. constStr.clear().append("Graph graph ").append(c);
  102. AttrStrUnionWithValueTable::roValueTable->find(constStr.str(), true);
  103. }
  104. for (unsigned c=1; c<=200; c++) // subgraphs
  105. {
  106. constStr.clear().append("sg").append(c);
  107. AttrStrUnionWithValueTable::roValueTable->find(constStr.str(), true);
  108. }
  109. for (unsigned c=1; c<=200; c++) // Edge 0
  110. {
  111. constStr.clear().append(c).append("_0");
  112. AttrStrUnionWithValueTable::roValueTable->find(constStr.str(), true);
  113. }
  114. for (unsigned c=0; c<35; c++)
  115. {
  116. char ch = c<9 ? ('1' + c) : ('A' + (c-9));
  117. constStr.clear().append("~spill::").append(ch); // spills
  118. AttrStrUnionWithValueTable::roValueTable->find(constStr.str(), true);
  119. constStr.clear().append("gl").append(ch); // graph results
  120. AttrStrUnionWithValueTable::roValueTable->find(constStr.str(), true);
  121. constStr.clear().append("mf").append(ch); // meta factories
  122. AttrStrUnionWithValueTable::roValueTable->find(constStr.str(), true);
  123. }
  124. for (unsigned c=1; c<=10; c++) // global auto attributes
  125. {
  126. constStr.clear().append("auto").append(c);
  127. AttrStrUnionWithValueTable::roValueTable->find(constStr.str(), true);
  128. }
  129. #ifdef TRACE_ATOM_SIZE
  130. // If you are wanting an idea of the savings from use of the RO hash table, it may be useful to reset
  131. // the counts here. But it's more correct to actually leave them in place.
  132. //AttrStrAtom::totsize = 0;
  133. //AttrStrAtom::maxsize = 0;
  134. #endif
  135. #ifdef _DEBUG
  136. for (const char **a = roAttributes; *a; a++)
  137. {
  138. // sanity check
  139. unsigned idx = AttrStrUnionWithTable::roNameTable->findIndex(*a, AttrStrC::getHash(*a));
  140. AttrStrC *val = AttrStrUnionWithTable::roNameTable->getIndex(idx);
  141. assert(val && val->eq(*a));
  142. }
  143. for (const char **v = roAttributeValues; *v; v++)
  144. {
  145. // sanity check
  146. unsigned idx = AttrStrUnionWithValueTable::roValueTable->findIndex(*v, AttrStrC::getHash(*v));
  147. AttrStrC *val = AttrStrUnionWithValueTable::roValueTable->getIndex(idx);
  148. assert(val && val->eq(*v));
  149. }
  150. #endif
  151. }
  152. #endif
  153. MODULE_INIT(INIT_PRIORITY_JPTREE)
  154. {
  155. nullPTreeIterator = new NullPTreeIterator;
  156. #ifdef USE_READONLY_ATOMTABLE
  157. AttrStrUnionWithTable::roNameTable = new RONameTable(255);
  158. AttrStrUnionWithValueTable::roValueTable = new RONameTable(4095);
  159. initializeRoTable();
  160. #endif
  161. keyTable = new AtomRefTable;
  162. keyTableNC = new AtomRefTable(true);
  163. attrHT = new CAttrValHashTable;
  164. return true;
  165. }
  166. MODULE_EXIT()
  167. {
  168. delete nullPTreeIterator;
  169. delete attrHT;
  170. keyTable->Release();
  171. keyTableNC->Release();
  172. #ifdef USE_READONLY_ATOMTABLE
  173. delete AttrStrUnionWithTable::roNameTable;
  174. delete AttrStrUnionWithValueTable::roValueTable;
  175. #endif
  176. free(freelist);
  177. freelist = NULL;
  178. }
  179. static int comparePropTrees(IInterface * const *ll, IInterface * const *rr)
  180. {
  181. IPropertyTree *l = (IPropertyTree *) *ll;
  182. IPropertyTree *r = (IPropertyTree *) *rr;
  183. return stricmp(l->queryName(), r->queryName());
  184. };
  185. class CPTArrayIterator : public ArrayIIteratorOf<IArrayOf<IPropertyTree>, IPropertyTree, IPropertyTreeIterator>
  186. {
  187. IArrayOf<IPropertyTree> elems;
  188. public:
  189. CPTArrayIterator(IPropertyTreeIterator &iter, TreeCompareFunc compare) : ArrayIIteratorOf<IArrayOf<IPropertyTree>, IPropertyTree, IPropertyTreeIterator>(elems)
  190. {
  191. ForEach(iter)
  192. elems.append(iter.get());
  193. elems.sort(compare);
  194. }
  195. CPTArrayIterator(IArrayOf<IPropertyTree> & ownedElems, TreeCompareFunc compare) : ArrayIIteratorOf<IArrayOf<IPropertyTree>, IPropertyTree, IPropertyTreeIterator>(elems)
  196. {
  197. elems.swapWith(ownedElems);
  198. elems.sort(compare);
  199. }
  200. };
  201. IPropertyTreeIterator * createSortedIterator(IPropertyTreeIterator & iter)
  202. {
  203. return new CPTArrayIterator(iter, comparePropTrees);
  204. }
  205. IPropertyTreeIterator * createSortedIterator(IPropertyTreeIterator & iter, TreeCompareFunc compare)
  206. {
  207. return new CPTArrayIterator(iter, compare);
  208. }
  209. IPropertyTreeIterator * createSortedIterator(IArrayOf<IPropertyTree> & ownedElems, TreeCompareFunc compare)
  210. {
  211. return new CPTArrayIterator(ownedElems, compare);
  212. }
  213. //////////////////
  214. unsigned ChildMap::getHashFromElement(const void *e) const
  215. {
  216. PTree &elem = (PTree &) (*(IPropertyTree *)e);
  217. return elem.queryHash();
  218. }
  219. unsigned ChildMap::numChildren() const
  220. {
  221. SuperHashIteratorOf<IPropertyTree> iter(*this);
  222. if (!iter.first()) return 0;
  223. unsigned count = 0;
  224. do
  225. {
  226. PTree *element = (PTree *) &iter.query();
  227. if (element->value && element->value->isArray())
  228. count += element->value->elements();
  229. else
  230. ++count;
  231. }
  232. while (iter.next());
  233. return count;
  234. }
  235. IPropertyTreeIterator *ChildMap::getIterator(bool sort)
  236. {
  237. class CPTHashIterator : implements IPropertyTreeIterator, public CInterface
  238. {
  239. SuperHashIteratorOf<IPropertyTree> *hiter;
  240. public:
  241. IMPLEMENT_IINTERFACE;
  242. CPTHashIterator(SuperHashTable &table) { hiter = new SuperHashIteratorOf<IPropertyTree>(table); }
  243. ~CPTHashIterator() { hiter->Release(); }
  244. // IPropertyTreeIterator
  245. virtual bool first() override { return hiter->first(); }
  246. virtual bool next() override { return hiter->next(); }
  247. virtual bool isValid() override { return hiter->isValid(); }
  248. virtual IPropertyTree & query() override { return hiter->query(); }
  249. };
  250. Owned<IPropertyTreeIterator> baseIter = new CPTHashIterator(*this);
  251. if (!sort)
  252. return baseIter.getClear();
  253. return createSortedIterator(*baseIter);
  254. }
  255. void ChildMap::onRemove(void *e)
  256. {
  257. PTree &elem = static_cast<PTree &>(*(IPropertyTree *)e);
  258. IPTArrayValue *value = elem.queryValue();
  259. if (value && value->isArray())
  260. {
  261. IPropertyTree **elems = value->getRawArray();
  262. IPropertyTree **elemsEnd = elems+value->elements();
  263. while (elems != elemsEnd)
  264. {
  265. PTree *pElem = static_cast<PTree *>(*elems++);
  266. pElem->setOwner(nullptr);
  267. }
  268. }
  269. elem.Release();
  270. }
  271. ///////////
  272. bool validateXMLTag(const char *name)
  273. {
  274. if (!isValidXPathStartChr(*name)) return false;
  275. ++name;
  276. while (*name != '\0')
  277. {
  278. if (!isValidXPathChr(*name)) return false;
  279. ++name;
  280. }
  281. return true;
  282. }
  283. class jlib_thrown_decl CPTreeException : implements IPTreeException, public CInterface
  284. {
  285. int errCode;
  286. StringBuffer errMsg;
  287. public:
  288. IMPLEMENT_IINTERFACE;
  289. CPTreeException(int _errCode, const char *_errMsg, va_list &args) __attribute__((format(printf,3,0))) : errCode(_errCode)
  290. {
  291. if (_errMsg)
  292. errMsg.valist_appendf(_errMsg, args);
  293. }
  294. StringBuffer &translateCode(StringBuffer &out) const
  295. {
  296. out.append("IPropertyTree: ");
  297. switch (errCode)
  298. {
  299. case PTreeExcpt_XPath_Ambiguity:
  300. return out.append("Ambiguous xpath used");
  301. case PTreeExcpt_XPath_ParseError:
  302. return out.append("xpath parse error");
  303. case PTreeExcpt_XPath_Unsupported:
  304. return out.append("unsupported xpath syntax used");
  305. case PTreeExcpt_InvalidTagName:
  306. return out.append("Invalid tag name");
  307. default:
  308. return out.append("UNKNOWN ERROR CODE: ").append(errCode);
  309. }
  310. }
  311. // IException
  312. int errorCode() const { return errCode; }
  313. StringBuffer &errorMessage(StringBuffer &out) const
  314. {
  315. return translateCode(out).append("\n").append(errMsg.str());
  316. }
  317. MessageAudience errorAudience() const { return MSGAUD_user; }
  318. };
  319. static IPTreeException *MakeIPTException(int code, const char *format, ...) __attribute__((format(printf,2,3)));
  320. static IPTreeException *MakeXPathException(const char *xpath, int code, size_t pos, const char *format, ...) __attribute__((format(printf,4,5)));
  321. IPTreeException *MakeIPTException(int code, const char *format, ...)
  322. {
  323. va_list args;
  324. va_start(args, format);
  325. IPTreeException *e = new CPTreeException(code, format, args);
  326. va_end(args);
  327. return e;
  328. }
  329. IPTreeException *MakeXPathException(const char *xpath, int code, size_t pos, const char *format, ...)
  330. {
  331. va_list args;
  332. va_start(args, format);
  333. StringBuffer s("XPath Exception: ");
  334. s.valist_appendf(format, args);
  335. va_end(args);
  336. #ifdef _DEBUG
  337. PrintStackReport();
  338. #endif
  339. const char *msg = "in xpath = ";
  340. s.append("\n").append(msg).append(xpath);
  341. s.append("\n").appendN((size32_t)(strlen(msg)+pos), ' ').append("^");
  342. return MakeIPTException(code, "%s", s.str());
  343. }
  344. inline static void readID(const char *&xxpath, bool started)
  345. {
  346. const char *xpath = xxpath;
  347. if (isValidXPathStartChr(*xpath) || (started && isValidXPathChr(*xpath)))
  348. {
  349. do
  350. {
  351. xpath++;
  352. } while (isValidXPathChr(*xpath));
  353. xxpath = xpath;
  354. }
  355. }
  356. inline static void readWildId(const char *&xpath, bool &wild)
  357. {
  358. wild = false;
  359. for (;;)
  360. {
  361. readID(xpath, wild);
  362. if ('*' != *xpath)
  363. break;
  364. wild = true;
  365. ++xpath;
  366. }
  367. }
  368. inline const char * readIndex(const char *xpath, StringAttr &index)
  369. {
  370. const char *start = xpath;
  371. do { xpath++; } while (isdigit(*xpath));
  372. index.set(start, (xpath - start));
  373. return xpath;
  374. }
  375. inline static void readWildIdIndex(const char *&xpath, bool &wild, bool &numeric)
  376. {
  377. const char *_xpath = xpath;
  378. readWildId(xpath, wild);
  379. if ('[' == *xpath) // check for local index not iterative qualifier.
  380. {
  381. const char *end = xpath+1;
  382. if (isdigit(*end))
  383. {
  384. StringAttr index;
  385. end = readIndex(end, index);
  386. if (']' != *end)
  387. throw MakeXPathException(_xpath, PTreeExcpt_XPath_ParseError, xpath-_xpath, "Qualifier brace unclosed");
  388. xpath = end+1;
  389. numeric = true;
  390. }
  391. else
  392. numeric = false;
  393. }
  394. else
  395. numeric = false;
  396. }
  397. inline static unsigned getTailIdLength(const char *xxpath, unsigned xxpathlength)
  398. {
  399. const char *xpath = xxpath+xxpathlength;
  400. const char *end = xpath;
  401. while (xpath != xxpath)
  402. {
  403. --xpath;
  404. if (!isValidXPathChr(*xpath)) break;
  405. }
  406. if (!isAttribute(xpath) && xpath != xxpath) ++xpath;
  407. return end-xpath;
  408. }
  409. const char *splitXPathUQ(const char *xpath, StringBuffer &path)
  410. {
  411. size32_t xpathSize = (size32_t) strlen(xpath);
  412. size32_t idSize = getTailIdLength(xpath, xpathSize);
  413. path.append(xpathSize-idSize, xpath);
  414. return xpath + (xpathSize-idSize);
  415. }
  416. const char *splitXPathX(const char *xpath)
  417. {
  418. size32_t xpathSize = (size32_t) strlen(xpath);
  419. size32_t idSize = getTailIdLength(xpath, xpathSize);
  420. return xpath + (xpathSize-idSize);
  421. }
  422. // similar to above, splitXPathUQ doesn't split if qualified
  423. const char *splitXPath(const char *xpath, StringBuffer &headPath)
  424. {
  425. StringBuffer path;
  426. const char *end = xpath+strlen(xpath);
  427. const char *prop = end;
  428. bool quote = false;
  429. bool braced = false;
  430. while (xpath != prop)
  431. {
  432. --prop;
  433. if (*prop == '"')
  434. {
  435. if (quote) quote = false;
  436. else quote = true;
  437. }
  438. else if (*prop == ']' && !quote)
  439. {
  440. assertex(!braced);
  441. braced = true;
  442. }
  443. else if (*prop == '[' && !quote)
  444. {
  445. assertex(braced);
  446. braced = false;
  447. }
  448. else if (*prop == '/' && !quote && !braced)
  449. {
  450. ++prop;
  451. break;
  452. }
  453. }
  454. if (prop == end)
  455. return NULL;
  456. else if (xpath != prop)
  457. {
  458. size32_t ps = prop-xpath-1;
  459. headPath.append(ps, xpath);
  460. }
  461. return prop;
  462. }
  463. const char *queryNextUnquoted(const char *str, char c)
  464. {
  465. bool quote = false;
  466. for (;;)
  467. {
  468. char next = *str;
  469. if (next == '\0')
  470. return NULL;
  471. if ('"' == next)
  472. quote = !quote;
  473. else if (c == next && !quote)
  474. return str;
  475. ++str;
  476. }
  477. }
  478. const char *queryHead(const char *xpath, StringBuffer &head)
  479. {
  480. if (!xpath) return NULL;
  481. const char *start = xpath;
  482. bool quote = false;
  483. bool braced = false;
  484. for (;;)
  485. {
  486. if (*xpath == '\0')
  487. return NULL;
  488. ++xpath;
  489. char next = *xpath;
  490. if ('"' == next)
  491. quote = !quote;
  492. else if (next == ']' && !quote)
  493. {
  494. assertex(braced);
  495. braced = false;
  496. }
  497. else if (next == '[' && !quote)
  498. {
  499. assertex(!braced);
  500. braced = true;
  501. }
  502. else if (next == '/' && !quote && !braced)
  503. {
  504. if ('/' == *start) // so leading '//'
  505. return start;
  506. else if ('/' == *(xpath+1)) // in middle of path
  507. {
  508. head.append(xpath-start, start);
  509. return xpath;
  510. }
  511. break;
  512. }
  513. }
  514. head.append(xpath-start, start);
  515. return xpath+1;
  516. }
  517. ///////////////////
  518. static constexpr unsigned defaultSiblingMapThreshold = 100;
  519. static unsigned siblingMapThreshold = (unsigned)-1; // off until configuration default it on.
  520. void setPTreeMappingThreshold(unsigned threshold)
  521. {
  522. /*
  523. * NB: setPTreeMappingThreshold() will automatically be called via loadConfiguration
  524. * Redefining this limit, will not effect existing maps, and should generally only be called once during startup.
  525. */
  526. if (0 == threshold)
  527. threshold = (unsigned)-1;
  528. siblingMapThreshold = threshold;
  529. }
  530. class CValueMap : public std::unordered_multimap<std::string, const IPropertyTree *>
  531. {
  532. public:
  533. CValueMap(const char *_lhs, IPTArrayValue &array)
  534. {
  535. IPropertyTree **elements = array.getRawArray();
  536. IPropertyTree **last = elements+array.elements();
  537. dbgassertex(elements != last);
  538. while (true)
  539. {
  540. const char *v = (*elements)->queryProp(_lhs);
  541. if (v)
  542. emplace(std::make_pair(std::string(v), *elements));
  543. elements++;
  544. if (last == elements)
  545. break;
  546. }
  547. }
  548. std::pair<CValueMap::iterator, CValueMap::iterator> find(const char *rhs)
  549. {
  550. return equal_range(std::string(rhs));
  551. }
  552. void insertEntry(const char *v, const IPropertyTree *tree)
  553. {
  554. emplace(std::make_pair(std::string(v), tree));
  555. }
  556. bool removeEntry(const char *v, const IPropertyTree *tree)
  557. {
  558. auto range = equal_range(std::string(v));
  559. if (range.first == range.second)
  560. return false;
  561. auto it = range.first;
  562. while (true)
  563. {
  564. if (it->second == tree)
  565. {
  566. it = erase(it);
  567. return true;
  568. }
  569. ++it;
  570. if (it == range.second)
  571. break;
  572. }
  573. throwUnexpected();
  574. }
  575. void replaceEntry(const char *oldV, const char *newV, const IPropertyTree *tree)
  576. {
  577. verifyex(removeEntry(oldV, tree));
  578. if (newV)
  579. insertEntry(newV, tree);
  580. }
  581. };
  582. class CQualifierMap
  583. {
  584. std::unordered_map<std::string, CValueMap *> attrValueMaps;
  585. CriticalSection crit;
  586. public:
  587. CQualifierMap()
  588. {
  589. }
  590. ~CQualifierMap()
  591. {
  592. for (auto &e: attrValueMaps)
  593. delete e.second;
  594. }
  595. CValueMap *addMapping(const char *lhs, IPTArrayValue &array)
  596. {
  597. CValueMap *valueMap = new CValueMap(lhs, array);
  598. attrValueMaps.emplace(std::make_pair(std::string(lhs), valueMap));
  599. return valueMap;
  600. }
  601. CValueMap *addMappingIfNew(const char *lhs, IPTArrayValue &array)
  602. {
  603. CriticalBlock b(crit);
  604. auto it = attrValueMaps.find(lhs);
  605. if (it == attrValueMaps.end())
  606. return addMapping(lhs, array);
  607. else
  608. return it->second;
  609. }
  610. void addMatchingValues(const IPropertyTree *tree)
  611. {
  612. for (auto &e: attrValueMaps)
  613. {
  614. const char *v = tree->queryProp(e.first.c_str());
  615. if (v)
  616. e.second->insertEntry(v, tree);
  617. }
  618. }
  619. void removeMatchingValues(const IPropertyTree *tree)
  620. {
  621. for (auto &e: attrValueMaps)
  622. {
  623. const char *lhsp = e.first.c_str();
  624. const char *oldV = tree->queryProp(lhsp);
  625. if (oldV)
  626. verifyex(e.second->removeEntry(oldV, tree));
  627. }
  628. }
  629. void replaceMatchingValues(const IPropertyTree *oldTree, const IPropertyTree *newTree)
  630. {
  631. for (auto &e: attrValueMaps)
  632. {
  633. const char *lhsp = e.first.c_str();
  634. const char *oldV = oldTree->queryProp(lhsp);
  635. if (oldV)
  636. {
  637. verifyex(e.second->removeEntry(oldV, oldTree));
  638. const char *newV = newTree->queryProp(lhsp);
  639. if (newV)
  640. e.second->insertEntry(newV, newTree);
  641. }
  642. }
  643. }
  644. CValueMap *find(const char *lhs)
  645. {
  646. auto it = attrValueMaps.find(lhs);
  647. if (it == attrValueMaps.end())
  648. return nullptr;
  649. return it->second;
  650. }
  651. void removeEntryIfMapped(const char *lhs, const char *v, const IPropertyTree *tree)
  652. {
  653. auto it = attrValueMaps.find(lhs);
  654. if (it != attrValueMaps.end())
  655. it->second->removeEntry(v, tree);
  656. }
  657. void insertEntryIfMapped(const char *lhs, const char *v, const IPropertyTree *tree)
  658. {
  659. auto it = attrValueMaps.find(lhs);
  660. if (it != attrValueMaps.end())
  661. it->second->insertEntry(v, tree);
  662. }
  663. void replaceEntryIfMapped(const char *lhs, const char *oldv, const char *newv, const IPropertyTree *tree)
  664. {
  665. auto it = attrValueMaps.find(lhs);
  666. if (it != attrValueMaps.end())
  667. it->second->replaceEntry(oldv, newv, tree);
  668. }
  669. };
  670. // parse qualifier, returns true if simple equality expression found
  671. static bool parseEqualityQualifier(const char *&xxpath, unsigned &lhsLen, const char *&rhsBegin, unsigned &rhsLen)
  672. {
  673. const char *xpath = xxpath;
  674. while (*xpath == ' ' || *xpath == '\t') xpath++;
  675. if ('@' != *xpath) // only attributes supported
  676. return false;
  677. const char *start = xpath;
  678. char quote = 0;
  679. const char *lhsEnd, *quoteBegin, *quoteEnd, *rhsEnd;
  680. lhsEnd = quoteBegin = quoteEnd = rhsBegin = rhsEnd = NULL;
  681. bool equalSignFound = false;
  682. for (;;)
  683. {
  684. switch (*xpath)
  685. {
  686. case '"':
  687. case '\'':
  688. if (quote)
  689. {
  690. if (*xpath == quote)
  691. {
  692. quote = 0;
  693. quoteEnd = xpath;
  694. }
  695. }
  696. else
  697. {
  698. if (quoteBegin)
  699. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Quoted left hand side already seen");
  700. quote = *xpath;
  701. quoteBegin = xpath+1;
  702. }
  703. break;
  704. case '[':
  705. if (!quote)
  706. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Unclosed qualifier detected");
  707. break;
  708. case ']':
  709. if (!quote)
  710. {
  711. if (!lhsEnd)
  712. lhsEnd = xpath;
  713. rhsEnd = xpath;
  714. }
  715. break;
  716. case ' ':
  717. case '\t':
  718. if (!lhsEnd)
  719. lhsEnd = xpath;
  720. break;
  721. case '!':
  722. case '>':
  723. case '<':
  724. case '~':
  725. case '/':
  726. if (!quote)
  727. return false;
  728. break;
  729. case '=':
  730. if (!quote)
  731. {
  732. if (equalSignFound)
  733. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Unexpected expression operator xpath");
  734. equalSignFound = true;
  735. if (!lhsEnd)
  736. lhsEnd = xpath;
  737. }
  738. break;
  739. case '?':
  740. case '*':
  741. return false;
  742. case '\0':
  743. rhsEnd = xpath;
  744. break;
  745. }
  746. if (rhsEnd)
  747. break;
  748. xpath++;
  749. if (!rhsBegin && equalSignFound && !isspace(*xpath))
  750. rhsBegin = xpath;
  751. }
  752. if (quote)
  753. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Parse error, unclosed quoted content");
  754. if (!equalSignFound)
  755. return false;
  756. lhsLen = lhsEnd-start;
  757. if (quoteBegin && !quoteEnd)
  758. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Parse error, RHS missing closing quote");
  759. if (rhsBegin && !rhsEnd)
  760. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Parse error, RHS missing closing quote");
  761. if (!quoteBegin && rhsEnd) // only if numeric
  762. return false;
  763. else // quoted
  764. {
  765. rhsBegin = quoteBegin;
  766. rhsLen = quoteEnd - rhsBegin;
  767. }
  768. if (rhsEnd && *xpath == ']')
  769. xpath++;
  770. xxpath = xpath;
  771. return true;
  772. }
  773. class CMapQualifierIterator : public CInterfaceOf<IPropertyTreeIterator>
  774. {
  775. CValueMap::iterator startRange, endRange;
  776. CValueMap::iterator currentIter;
  777. public:
  778. CMapQualifierIterator(CQualifierMap &_map, CValueMap::iterator _startRange, CValueMap::iterator _endRange)
  779. : startRange(_startRange), endRange(_endRange)
  780. {
  781. }
  782. // IPropertyTreeIterator
  783. virtual bool first() override
  784. {
  785. currentIter = startRange;
  786. return currentIter != endRange;
  787. }
  788. virtual bool next() override
  789. {
  790. currentIter++;
  791. return currentIter != endRange;
  792. }
  793. virtual bool isValid() override
  794. {
  795. return currentIter != endRange;
  796. }
  797. virtual IPropertyTree & query() override { return const_cast<IPropertyTree &>(*currentIter->second); }
  798. };
  799. IPropertyTreeIterator *checkMapIterator(const char *&xxpath, IPropertyTree &child)
  800. {
  801. /*
  802. * NB: IPT's are not thread safe. It is up to the caller to ensure multiple writers do not contend.
  803. * ( Dali for example ensures writer threads are exclusive )
  804. *
  805. * That means multiple reader threads could be here concurrently.
  806. * >1 could be constructing the qualifier map for the 1st time.
  807. * For new attr updates (where map already exists), it will block on map::crit,
  808. * so that there is at most 1 thread updating the map. The underlying unordered_multiset
  809. * is thread safe if 1 writer, and multiple readers.
  810. *
  811. * On initial map creation, allow concurrency, but only 1 will succeed to swap in the new active map.
  812. * That could mean a new attr/prop. mapping is lost, until next used.
  813. * NB: once the map is live, updates are write ops. and so, just as with the IPT
  814. * itself, it is expected that something will keep it thread safe (as Dali does)
  815. *
  816. */
  817. // NB: only support simple @<attrname>=<value> qualifiers
  818. if (((unsigned)-1) == siblingMapThreshold) // disabled
  819. return nullptr;
  820. PTree &_child = (PTree &)child;
  821. if (child.isCaseInsensitive()) // NB: could support but not worth it.
  822. return nullptr;
  823. IPTArrayValue *value = _child.queryValue();
  824. if (!value)
  825. return nullptr;
  826. CQualifierMap *map = value->queryMap();
  827. if (!map)
  828. {
  829. if (!value->isArray() || (value->elements() < siblingMapThreshold))
  830. return nullptr;
  831. }
  832. unsigned lhsLen, rhsLen;
  833. const char *rhsStart;
  834. const char *xpath = xxpath;
  835. if (!parseEqualityQualifier(xpath, lhsLen, rhsStart, rhsLen))
  836. return nullptr;
  837. MAKE_LSTRING(lhs, xxpath, lhsLen);
  838. MAKE_LSTRING(rhs, rhsStart, rhsLen);
  839. // NB: there can be a race here where >1 reader is constructing new map
  840. CValueMap *valueMap = nullptr;
  841. if (map)
  842. valueMap = map->addMappingIfNew(lhs, *value);
  843. else
  844. {
  845. OwnedPtr<CQualifierMap> newMap = new CQualifierMap();
  846. valueMap = newMap->addMapping(lhs, *value);
  847. /*
  848. * NB: it's possible another read thread got here 1st, and swapped in a map.
  849. * setMap returns the existing map, and the code below checks to see if it already
  850. * handles the 'lhs' we're adding, if it doesn't it re-adds the qualifier mappings.
  851. */
  852. map = value->setMap(newMap);
  853. if (!map) // successfully swapped newMap in.
  854. map = newMap.getClear(); // NB: setMap owns
  855. else // another thread has swapped in a map whilst I was creating new one
  856. valueMap = map->addMappingIfNew(lhs, *value);
  857. }
  858. xxpath = xpath; // update parsed position
  859. auto range = valueMap->find(rhs);
  860. if (range.first != range.second)
  861. return new CMapQualifierIterator(*map, range.first, range.second);
  862. else
  863. return LINK(nullPTreeIterator);
  864. }
  865. ///////////////////
  866. class SeriesPTIterator : implements IPropertyTreeIterator, public CInterface
  867. {
  868. public:
  869. IMPLEMENT_IINTERFACE;
  870. SeriesPTIterator() : current(NULL), cp(0)
  871. {
  872. }
  873. void addIterator(IPropertyTreeIterator *iter) { iters.append(*iter); }
  874. // IPropertyTreeIterator impl.
  875. virtual bool first() override
  876. {
  877. cp = 0;
  878. iterCount = iters.ordinality();
  879. if (nextIterator())
  880. return true;
  881. else
  882. return false;
  883. }
  884. virtual bool next() override
  885. {
  886. while (currentIter)
  887. {
  888. if (currentIter->next())
  889. {
  890. current = &currentIter->query();
  891. return true;
  892. }
  893. if (nextIterator())
  894. return true;
  895. }
  896. current = NULL;
  897. return false;
  898. }
  899. virtual bool isValid() override { return (NULL != current); }
  900. virtual IPropertyTree & query() override { assertex(current); return *current; }
  901. private:
  902. bool nextIterator()
  903. {
  904. while (cp<iterCount)
  905. {
  906. currentIter = (IPropertyTreeIterator *) &iters.item(cp++);
  907. if (currentIter->first())
  908. {
  909. current = &currentIter->query();
  910. return true;
  911. }
  912. }
  913. current = NULL;
  914. currentIter = NULL;
  915. return false;
  916. }
  917. IArray iters;
  918. IPropertyTreeIterator *currentIter;
  919. IPropertyTree *current;
  920. unsigned cp, iterCount;
  921. };
  922. ///////////////////
  923. CPTValue::CPTValue(size32_t size, const void *data, bool binary, bool raw, bool _compressed)
  924. {
  925. compressed = _compressed;
  926. if (!raw && binary && size > PTREE_COMPRESS_THRESHOLD)
  927. {
  928. unsigned newSize = size * PTREE_COMPRESS_BOTHER_PECENTAGE / 100;
  929. void *newData = NULL;
  930. ICompressor *compressor = NULL;
  931. try
  932. {
  933. newData = malloc(sizeof(size32_t) + newSize);
  934. compressor = createLZWCompressor();
  935. compressor->open(((char *)newData) + sizeof(size32_t), newSize);
  936. if (compressor->write(data, size)==size)
  937. {
  938. compressor->close();
  939. memcpy(newData, &size, sizeof(size32_t));
  940. newSize = sizeof(size32_t) + compressor->buflen();
  941. compressed = true;
  942. set(newSize, newData);
  943. }
  944. free(newData);
  945. compressor->Release();
  946. }
  947. catch (...)
  948. {
  949. if (newData)
  950. free(newData);
  951. if (compressor) compressor->Release();
  952. throw;
  953. }
  954. }
  955. if (raw || !compressed)
  956. set(size, data);
  957. }
  958. static void *uncompress(const void *src, size32_t &sz)
  959. {
  960. IExpander *expander = NULL;
  961. void *uncompressedValue = NULL;
  962. try
  963. {
  964. memcpy(&sz, src, sizeof(size32_t));
  965. assertex(sz);
  966. expander = createLZWExpander();
  967. src = ((const char *)src) + sizeof(size32_t);
  968. uncompressedValue = malloc(sz);
  969. assertex(uncompressedValue);
  970. expander->init(src);
  971. expander->expand(uncompressedValue);
  972. expander->Release();
  973. return uncompressedValue;
  974. }
  975. catch (...)
  976. {
  977. if (expander) expander->Release();
  978. if (uncompressedValue) free(uncompressedValue);
  979. throw;
  980. }
  981. }
  982. const void *CPTValue::queryValue() const
  983. {
  984. if (compressed)
  985. {
  986. size32_t sz;
  987. void *uncompressedValue = uncompress(get(), sz);
  988. ((MemoryAttr *)this)->setOwn(sz, uncompressedValue);
  989. compressed = false;
  990. }
  991. return get();
  992. }
  993. void CPTValue::serialize(MemoryBuffer &tgt)
  994. {
  995. //Retain backward compatibility for the serialization format.
  996. size32_t serialLen = (size32_t)length();
  997. tgt.append(serialLen);
  998. if (serialLen)
  999. {
  1000. tgt.append(compressed);
  1001. tgt.append(serialLen, get());
  1002. }
  1003. }
  1004. void CPTValue::deserialize(MemoryBuffer &src)
  1005. {
  1006. size32_t sz;
  1007. src.read(sz);
  1008. if (sz)
  1009. {
  1010. src.read(compressed);
  1011. set(sz, src.readDirect(sz));
  1012. }
  1013. else
  1014. {
  1015. compressed = false;
  1016. clear();
  1017. }
  1018. }
  1019. MemoryBuffer &CPTValue::getValue(MemoryBuffer &tgt, bool binary) const
  1020. {
  1021. if (compressed)
  1022. {
  1023. size32_t sz;
  1024. void *uncompressedValue = uncompress(get(), sz);
  1025. if (!binary) sz -= 1;
  1026. tgt.append(sz, uncompressedValue);
  1027. if (uncompressedValue)
  1028. free(uncompressedValue);
  1029. }
  1030. else
  1031. {
  1032. if (binary)
  1033. tgt.append((size32_t)length(), get());
  1034. else
  1035. tgt.append((size32_t)length()-1, get());
  1036. }
  1037. return tgt;
  1038. }
  1039. StringBuffer &CPTValue::getValue(StringBuffer &tgt, bool binary) const
  1040. {
  1041. if (compressed)
  1042. {
  1043. size32_t sz;
  1044. void *uncompressedValue = NULL;
  1045. try
  1046. {
  1047. uncompressedValue = uncompress(get(), sz);
  1048. if (!binary) sz -= 1;
  1049. tgt.append(sz, (const char *)uncompressedValue);
  1050. free(uncompressedValue);
  1051. }
  1052. catch (IException *)
  1053. {
  1054. if (uncompressedValue) free(uncompressedValue);
  1055. throw;
  1056. }
  1057. }
  1058. else
  1059. {
  1060. if (binary) // this should probably be an assert?
  1061. tgt.append((size32_t)length(), (const char *)get());
  1062. else if (length())
  1063. tgt.append((size32_t)length()-1, (const char *)get());
  1064. }
  1065. return tgt;
  1066. }
  1067. size32_t CPTValue::queryValueSize() const
  1068. {
  1069. if (compressed)
  1070. {
  1071. size32_t sz;
  1072. memcpy(&sz, get(), sizeof(size32_t));
  1073. return sz;
  1074. }
  1075. else
  1076. return (size32_t)length();
  1077. }
  1078. ///////////////////
  1079. CPTArray::~CPTArray()
  1080. {
  1081. if (map.load())
  1082. delete map.load();
  1083. }
  1084. CQualifierMap *CPTArray::setMap(CQualifierMap *_map)
  1085. {
  1086. CQualifierMap *expected = nullptr;
  1087. if (map.compare_exchange_strong(expected, _map))
  1088. return nullptr;
  1089. else
  1090. return expected;
  1091. }
  1092. void CPTArray::addElement(IPropertyTree *tree)
  1093. {
  1094. append(*tree);
  1095. CQualifierMap *map = queryMap();
  1096. if (map)
  1097. {
  1098. if (tree->getAttributeCount())
  1099. map->addMatchingValues(tree);
  1100. }
  1101. }
  1102. void CPTArray::setElement(unsigned idx, IPropertyTree *tree)
  1103. {
  1104. CQualifierMap *map = queryMap();
  1105. if (map)
  1106. {
  1107. // remove any mappings for existing element.
  1108. if (isItem(idx))
  1109. {
  1110. IPropertyTree *existing = &((IPropertyTree &)item(idx));
  1111. map->replaceMatchingValues(existing, tree);
  1112. }
  1113. else
  1114. map->addMatchingValues(tree);
  1115. }
  1116. add(*tree, idx);
  1117. }
  1118. void CPTArray::removeElement(unsigned idx)
  1119. {
  1120. CQualifierMap *map = queryMap();
  1121. IPropertyTree *existing = &((IPropertyTree &)item(idx));
  1122. if (map)
  1123. map->removeMatchingValues(existing);
  1124. ((PTree *)existing)->setOwner(nullptr);
  1125. remove(idx);
  1126. }
  1127. unsigned CPTArray::find(const IPropertyTree *search) const
  1128. {
  1129. IInterface **start = getArray();
  1130. IInterface **last = start + ordinality();
  1131. IInterface **members = start;
  1132. while (true)
  1133. {
  1134. if (*members == search)
  1135. return members-start;
  1136. members++;
  1137. if (members == last)
  1138. break;
  1139. }
  1140. return NotFound;
  1141. }
  1142. //////////////////
  1143. PTree::PTree(byte _flags, IPTArrayValue *_value, ChildMap *_children)
  1144. {
  1145. flags = _flags;
  1146. children = LINK(_children);
  1147. value = _value;
  1148. }
  1149. PTree::~PTree()
  1150. {
  1151. if (value) delete value;
  1152. ::Release(children);
  1153. }
  1154. IPropertyTree *PTree::queryChild(unsigned index)
  1155. {
  1156. if (!value) return NULL;
  1157. if (!value->isArray()) return this;
  1158. IPropertyTree *v = value->queryElement(index);
  1159. return v;
  1160. }
  1161. aindex_t PTree::findChild(IPropertyTree *child, bool remove)
  1162. {
  1163. if (value && value->isArray())
  1164. {
  1165. unsigned pos = value->find(child);
  1166. if (remove && NotFound != pos)
  1167. value->removeElement(pos);
  1168. return pos;
  1169. }
  1170. else if (checkChildren())
  1171. {
  1172. IPropertyTree *_child = children->query(child->queryName());
  1173. if (_child == child)
  1174. {
  1175. if (remove)
  1176. children->removeExact(_child);
  1177. return 0;
  1178. }
  1179. else if (_child)
  1180. {
  1181. PTree *__child = (PTree *) _child;
  1182. return __child->findChild(child, remove);
  1183. }
  1184. }
  1185. return NotFound;
  1186. }
  1187. ChildMap *PTree::checkChildren() const
  1188. {
  1189. return children;
  1190. }
  1191. void PTree::setLocal(size32_t l, const void *data, bool _binary)
  1192. {
  1193. if (value) delete value;
  1194. if (l)
  1195. value = new CPTValue(l, data, _binary);
  1196. else
  1197. value = NULL;
  1198. if (_binary)
  1199. IptFlagSet(flags, ipt_binary);
  1200. else
  1201. IptFlagClr(flags, ipt_binary);
  1202. }
  1203. void PTree::appendLocal(size32_t l, const void *data, bool binary)
  1204. {
  1205. if (0 == l) return;
  1206. MemoryBuffer mb;
  1207. if (value)
  1208. {
  1209. assertex(!value->isArray());
  1210. assertex(binary == IptFlagTst(flags, ipt_binary));
  1211. value->getValue(mb, binary);
  1212. mb.append(l, data);
  1213. delete value;
  1214. l = mb.length();
  1215. data = mb.toByteArray();
  1216. }
  1217. if (l)
  1218. value = new CPTValue(l, data, binary);
  1219. else
  1220. value = NULL;
  1221. if (binary)
  1222. IptFlagSet(flags, ipt_binary);
  1223. else
  1224. IptFlagClr(flags, ipt_binary);
  1225. }
  1226. // IPropertyTree impl.
  1227. bool PTree::hasProp(const char * xpath) const
  1228. {
  1229. const char *prop = splitXPathX(xpath);
  1230. if (isAttribute(prop)) // JCS - note no wildcards on attributes
  1231. {
  1232. if (prop != xpath)
  1233. {
  1234. MAKE_LSTRING(path, xpath, prop-xpath);
  1235. Owned<IPropertyTreeIterator> iter = getElements(path);
  1236. if (iter->first())
  1237. {
  1238. do
  1239. {
  1240. IPropertyTree &branch = iter->query();
  1241. if (branch.hasProp(prop))
  1242. return true;
  1243. }
  1244. while (iter->next());
  1245. }
  1246. return false;
  1247. }
  1248. else
  1249. return nullptr != findAttribute(xpath);
  1250. }
  1251. else
  1252. {
  1253. IPropertyTreeIterator *iter = getElements(xpath);
  1254. bool res = iter->first();
  1255. iter->Release();
  1256. return res;
  1257. }
  1258. }
  1259. const char *PTree::queryProp(const char *xpath) const
  1260. {
  1261. if (!xpath)
  1262. {
  1263. if (!value) return NULL;
  1264. return (const char *) value->queryValue();
  1265. }
  1266. else if (isAttribute(xpath))
  1267. return getAttributeValue(xpath);
  1268. else
  1269. {
  1270. const char *prop = splitXPathX(xpath);
  1271. if (isAttribute(prop))
  1272. {
  1273. MAKE_LSTRING(path, xpath, prop-xpath);
  1274. IPropertyTree *branch = queryPropTree(path);
  1275. if (!branch) return NULL;
  1276. return branch->queryProp(prop);
  1277. }
  1278. else
  1279. {
  1280. IPropertyTree *branch = queryPropTree(xpath);
  1281. if (!branch) return NULL;
  1282. return branch->queryProp(NULL);
  1283. }
  1284. }
  1285. }
  1286. bool PTree::getProp(const char *xpath, StringBuffer &ret) const
  1287. {
  1288. if (!xpath)
  1289. {
  1290. if (!value) return false;
  1291. value->getValue(ret, IptFlagTst(flags, ipt_binary));
  1292. return true;
  1293. }
  1294. else if (isAttribute(xpath))
  1295. {
  1296. const char *value = getAttributeValue(xpath);
  1297. if (!value) return false;
  1298. ret.append(value);
  1299. return true;
  1300. }
  1301. else
  1302. {
  1303. const char *prop = splitXPathX(xpath);
  1304. if (isAttribute(prop))
  1305. {
  1306. MAKE_LSTRING(path, xpath, prop-xpath)
  1307. IPropertyTree *branch = queryPropTree(path);
  1308. if (!branch) return false;
  1309. return branch->getProp(prop, ret);
  1310. }
  1311. else
  1312. {
  1313. IPropertyTree *branch = queryPropTree(xpath);
  1314. if (!branch) return false;
  1315. return branch->getProp(NULL, ret);
  1316. }
  1317. }
  1318. }
  1319. void PTree::setProp(const char *xpath, const char *val)
  1320. {
  1321. if (!xpath || '\0' == *xpath)
  1322. {
  1323. if (!val)
  1324. {
  1325. if (value) delete value;
  1326. value = NULL;
  1327. }
  1328. else
  1329. {
  1330. size32_t l=(size32_t)strlen(val);
  1331. if (!l)
  1332. {
  1333. if (value) delete value;
  1334. value = NULL;
  1335. }
  1336. else
  1337. setLocal(l+1, val);
  1338. }
  1339. }
  1340. else if (isAttribute(xpath))
  1341. {
  1342. if (!val)
  1343. removeAttribute(xpath);
  1344. else
  1345. setAttribute(xpath, val);
  1346. }
  1347. else
  1348. {
  1349. const char *prop;
  1350. IPropertyTree *branch = splitBranchProp(xpath, prop, true);
  1351. if (isAttribute(prop))
  1352. branch->setProp(prop, val);
  1353. else
  1354. {
  1355. if (val)
  1356. {
  1357. IPropertyTree *propBranch = queryCreateBranch(branch, prop);
  1358. propBranch->setProp(NULL, val);
  1359. }
  1360. else
  1361. branch->removeProp(prop);
  1362. }
  1363. }
  1364. }
  1365. aindex_t PTree::getChildMatchPos(const char *xpath)
  1366. {
  1367. Owned<IPropertyTreeIterator> childIter = getElements(xpath);
  1368. if (!childIter->first())
  1369. return (aindex_t)-1;
  1370. IPropertyTree &childMatch = childIter->query();
  1371. #ifdef _DEBUG
  1372. if (childIter->next())
  1373. AMBIGUOUS_PATH("addPropX", xpath);
  1374. #endif
  1375. if (value)
  1376. if (value->isArray())
  1377. return findChild(&childMatch);
  1378. else
  1379. return 0;
  1380. else
  1381. return 0;
  1382. }
  1383. void PTree::resolveParentChild(const char *xpath, IPropertyTree *&parent, IPropertyTree *&child, StringAttr &path, StringAttr &qualifier)
  1384. {
  1385. parent = child = NULL;
  1386. if (!xpath)
  1387. throw MakeIPTException(-1, "No path to resolve parent from");
  1388. const char *end = xpath+strlen(xpath);
  1389. const char *prop = end;
  1390. while (prop != xpath && *(prop-1) != '/')
  1391. --prop;
  1392. size32_t ps = prop-xpath;
  1393. if (ps)
  1394. {
  1395. path.set(xpath, ps);
  1396. Owned<IPropertyTreeIterator> pathIter = getElements(path);
  1397. if (!pathIter->first())
  1398. throw MakeIPTException(-1, "resolveParentChild: path not found %s", xpath);
  1399. /* If 'path' resolves to iterator of this, then treat as if no leading path
  1400. * i.e. "./x", or "././.x" is equivalent to "x"
  1401. */
  1402. if (this != &pathIter->query())
  1403. {
  1404. IPropertyTree *currentPath = NULL;
  1405. #ifdef _DEBUG
  1406. bool multiplePaths = false;
  1407. #endif
  1408. bool multipleChildMatches = false;
  1409. for (;;)
  1410. {
  1411. // JCSMORE - a bit annoying has to be done again once path has been established
  1412. currentPath = &pathIter->query();
  1413. Owned<IPropertyTreeIterator> childIter = currentPath->getElements(prop);
  1414. if (childIter->first())
  1415. {
  1416. child = &childIter->query();
  1417. #ifdef _DEBUG
  1418. if (parent)
  1419. AMBIGUOUS_PATH("resolveParentChild", xpath);
  1420. #endif
  1421. if (!multipleChildMatches && childIter->next())
  1422. multipleChildMatches = true;
  1423. parent = currentPath;
  1424. }
  1425. if (pathIter->next())
  1426. {
  1427. #ifdef _DEBUG
  1428. multiplePaths = true;
  1429. #endif
  1430. }
  1431. else
  1432. break;
  1433. }
  1434. if (!parent)
  1435. {
  1436. #ifdef _DEBUG
  1437. if (multiplePaths) // i.e. no unique path to child found and multiple parent paths
  1438. AMBIGUOUS_PATH("resolveParentChild", xpath);
  1439. #endif
  1440. parent = currentPath;
  1441. }
  1442. if (multipleChildMatches)
  1443. child = NULL; // single parent, but no single child.
  1444. path.set(prop);
  1445. const char *pstart = prop;
  1446. bool wild;
  1447. readWildId(prop, wild);
  1448. size32_t s = prop-pstart;
  1449. if (wild)
  1450. throw MakeXPathException(pstart, PTreeExcpt_XPath_ParseError, s-1, "Wildcards not permitted on add");
  1451. assertex(s);
  1452. path.set(pstart, s);
  1453. qualifier.set(prop);
  1454. return;
  1455. }
  1456. }
  1457. assertex(prop && *prop);
  1458. parent = this;
  1459. const char *pstart = prop;
  1460. bool wild;
  1461. readWildId(prop, wild);
  1462. assertex(!wild);
  1463. size32_t s = prop-pstart;
  1464. if (*prop && *prop != '[')
  1465. throw MakeXPathException(pstart, PTreeExcpt_XPath_ParseError, s, "Qualifier expected e.g. [..]");
  1466. path.set(pstart, s);
  1467. if (checkChildren())
  1468. child = children->query(path);
  1469. if (child)
  1470. qualifier.set(prop);
  1471. else
  1472. qualifier.clear();
  1473. }
  1474. void PTree::addProp(const char *xpath, const char *val)
  1475. {
  1476. if (!xpath || '\0' == *xpath)
  1477. addLocal((size32_t)strlen(val)+1, val);
  1478. else if (isAttribute(xpath))
  1479. setAttribute(xpath, val);
  1480. else if ('[' == *xpath)
  1481. {
  1482. aindex_t pos = getChildMatchPos(xpath);
  1483. if ((aindex_t) -1 == pos)
  1484. throw MakeIPTException(-1, "addProp: qualifier unmatched %s", xpath);
  1485. addLocal((size32_t)strlen(val)+1, val, false, pos);
  1486. }
  1487. else
  1488. {
  1489. IPropertyTree *parent, *child;
  1490. StringAttr path, qualifier;
  1491. resolveParentChild(xpath, parent, child, path, qualifier);
  1492. if (parent != this)
  1493. parent->addProp(path, val);
  1494. else if (child)
  1495. child->addProp(qualifier, val);
  1496. else
  1497. setProp(path, val);
  1498. }
  1499. }
  1500. void PTree::appendProp(const char *xpath, const char *val)
  1501. {
  1502. if (!xpath || '\0' == *xpath)
  1503. appendLocal((size_t)strlen(val)+1, val, false);
  1504. else if (isAttribute(xpath))
  1505. {
  1506. StringBuffer newVal;
  1507. getProp(xpath, newVal);
  1508. newVal.append(val);
  1509. setAttribute(xpath, newVal.str());
  1510. }
  1511. else if ('[' == *xpath)
  1512. {
  1513. IPropertyTree *qualified = queryPropTree(xpath);
  1514. if (!qualified)
  1515. throw MakeIPTException(-1, "appendProp: qualifier unmatched %s", xpath);
  1516. qualified->appendProp(nullptr, val);
  1517. }
  1518. else
  1519. {
  1520. IPropertyTree *parent, *child;
  1521. StringAttr path, qualifier;
  1522. resolveParentChild(xpath, parent, child, path, qualifier);
  1523. if (parent != this)
  1524. parent->appendProp(path, val);
  1525. else if (child)
  1526. child->appendProp(qualifier, val);
  1527. else
  1528. setProp(path, val);
  1529. }
  1530. }
  1531. bool PTree::getPropBool(const char *xpath, bool dft) const
  1532. {
  1533. const char *val = queryProp(xpath);
  1534. if (val && *val)
  1535. return strToBool(val);
  1536. else
  1537. return dft;
  1538. }
  1539. __int64 PTree::getPropInt64(const char *xpath, __int64 dft) const
  1540. {
  1541. if (!xpath)
  1542. {
  1543. if (!value) return dft;
  1544. else
  1545. {
  1546. const char *v = (const char *)value->queryValue();
  1547. if (!v || !*v) return dft;
  1548. else return _atoi64(v);
  1549. }
  1550. }
  1551. else if (isAttribute(xpath))
  1552. {
  1553. const char *v = getAttributeValue(xpath);
  1554. if (!v || !*v) // intentional return dft if attribute equals ""
  1555. return dft;
  1556. return _atoi64(v);
  1557. }
  1558. else
  1559. {
  1560. const char *prop = splitXPathX(xpath);
  1561. if (isAttribute(prop))
  1562. {
  1563. MAKE_LSTRING(path, xpath, prop-xpath);
  1564. IPropertyTree *branch = queryPropTree(path);
  1565. if (!branch) return dft;
  1566. return branch->getPropInt64(prop, dft);
  1567. }
  1568. else
  1569. {
  1570. IPropertyTree *branch = queryPropTree(xpath);
  1571. if (!branch) return dft;
  1572. return branch->getPropInt64(NULL, dft);
  1573. }
  1574. }
  1575. }
  1576. void PTree::setPropInt64(const char * xpath, __int64 val)
  1577. {
  1578. if (!xpath || '\0' == *xpath)
  1579. {
  1580. char buf[23];
  1581. numtostr(buf, val);
  1582. setLocal((size32_t)strlen(buf)+1, buf);
  1583. }
  1584. else if (isAttribute(xpath))
  1585. {
  1586. char buf[23];
  1587. numtostr(buf, val);
  1588. setAttribute(xpath, buf);
  1589. }
  1590. else
  1591. {
  1592. const char *prop;
  1593. IPropertyTree *branch = splitBranchProp(xpath, prop, true);
  1594. if (isAttribute(prop))
  1595. branch->setPropInt64(prop, val);
  1596. else
  1597. {
  1598. IPropertyTree *propBranch = queryCreateBranch(branch, prop);
  1599. propBranch->setPropInt64(NULL, val);
  1600. }
  1601. }
  1602. }
  1603. void PTree::addPropInt64(const char *xpath, __int64 val)
  1604. {
  1605. if (!xpath || '\0' == *xpath)
  1606. {
  1607. char buf[23];
  1608. numtostr(buf,val);
  1609. addLocal((size32_t)strlen(buf)+1, buf);
  1610. }
  1611. else if (isAttribute(xpath))
  1612. {
  1613. char buf[23];
  1614. numtostr(buf, val);
  1615. setAttribute(xpath, buf);
  1616. }
  1617. else if ('[' == *xpath)
  1618. {
  1619. char buf[23];
  1620. numtostr(buf, val);
  1621. aindex_t pos = getChildMatchPos(xpath);
  1622. if ((aindex_t) -1 == pos)
  1623. throw MakeIPTException(-1, "addPropInt64: qualifier unmatched %s", xpath);
  1624. addLocal((size32_t)strlen(buf)+1, buf, false, pos);
  1625. }
  1626. else
  1627. {
  1628. IPropertyTree *parent, *child;
  1629. StringAttr path, qualifier;
  1630. resolveParentChild(xpath, parent, child, path, qualifier);
  1631. if (parent != this)
  1632. parent->addPropInt64(path, val);
  1633. else if (child)
  1634. child->addPropInt64(qualifier, val);
  1635. else
  1636. setPropInt64(path, val);
  1637. }
  1638. }
  1639. int PTree::getPropInt(const char *xpath, int dft) const
  1640. {
  1641. return (int) getPropInt64(xpath, dft); // underlying type always __int64 (now)
  1642. }
  1643. void PTree::setPropInt(const char *xpath, int val)
  1644. {
  1645. setPropInt64(xpath, val); // underlying type always __int64 (now)
  1646. }
  1647. void PTree::addPropInt(const char *xpath, int val)
  1648. {
  1649. addPropInt64(xpath, val); // underlying type always __int64 (now)
  1650. }
  1651. double PTree::getPropReal(const char *xpath, double dft) const
  1652. {
  1653. const char *val = queryProp(xpath);
  1654. return val?atof(val):dft;
  1655. }
  1656. bool PTree::isCompressed(const char *xpath) const
  1657. {
  1658. if (!xpath)
  1659. return (value && value->isCompressed());
  1660. else if (isAttribute(xpath))
  1661. return false;
  1662. else
  1663. {
  1664. const char *prop = splitXPathX(xpath);
  1665. if (!isAttribute(prop))
  1666. {
  1667. IPropertyTree *branch = queryPropTree(xpath);
  1668. if (branch)
  1669. return branch->isCompressed(nullptr);
  1670. }
  1671. }
  1672. return false;
  1673. }
  1674. bool PTree::isBinary(const char *xpath) const
  1675. {
  1676. if (!xpath)
  1677. return IptFlagTst(flags, ipt_binary);
  1678. else if (isAttribute(xpath)) // still positing that attr cannot be binary for now.
  1679. return false;
  1680. else
  1681. {
  1682. const char *prop = splitXPathX(xpath);
  1683. if (!isAttribute(prop))
  1684. {
  1685. IPropertyTree *branch = queryPropTree(xpath);
  1686. if (branch)
  1687. return branch->isBinary(nullptr);
  1688. }
  1689. }
  1690. return false;
  1691. }
  1692. bool PTree::renameTree(IPropertyTree *child, const char *newName) // really here for hook for SDS (can substationally optimize remote action)
  1693. {
  1694. if (0==strcmp(newName, child->queryName()) && NotFound!=findChild(child)) return false;
  1695. Linked<IPropertyTree> tmp = child;
  1696. if (removeTree(child))
  1697. {
  1698. addPropTree(newName, child);
  1699. tmp.getClear(); // addPropTree has taken ownership.
  1700. return true;
  1701. }
  1702. return false;
  1703. }
  1704. bool PTree::renameProp(const char *xpath, const char *newName)
  1705. {
  1706. if (!xpath || '\0' == *xpath)
  1707. throw MakeIPTException(-1, "renameProp: cannot rename self, renameProp has to rename in context of a parent");
  1708. if (strcmp(xpath,"/")==0) // rename of self allowed assuming no parent
  1709. setName(newName);
  1710. else if ('[' == *xpath)
  1711. UNIMPLEMENTED;
  1712. else if (isAttribute(xpath))
  1713. {
  1714. StringBuffer val;
  1715. if (!getProp(xpath, val))
  1716. return false;
  1717. removeProp(xpath);
  1718. addProp(newName, val.str());
  1719. }
  1720. else
  1721. {
  1722. StringBuffer path;
  1723. const char *prop = splitXPath(xpath, path);
  1724. assertex(prop);
  1725. if (path.length())
  1726. {
  1727. Owned<IPropertyTreeIterator> iter = getElements(path.str());
  1728. if (!iter->first())
  1729. return false;
  1730. IPropertyTree &branch = iter->query();
  1731. #ifdef _DEBUG
  1732. if (iter->next())
  1733. AMBIGUOUS_PATH("renameProp", xpath);
  1734. #endif
  1735. return branch.renameProp(prop, newName);
  1736. }
  1737. else
  1738. {
  1739. IPropertyTree *old = queryPropTree(xpath);
  1740. if (!old)
  1741. return false;
  1742. return renameTree(old, newName);
  1743. }
  1744. }
  1745. return true;
  1746. }
  1747. bool PTree::getPropBin(const char *xpath, MemoryBuffer &ret) const
  1748. {
  1749. CHECK_ATTRIBUTE(xpath);
  1750. if (!xpath)
  1751. {
  1752. if (!value) return true; // exists, but no value
  1753. value->getValue(ret, IptFlagTst(flags, ipt_binary));
  1754. return true;
  1755. }
  1756. else
  1757. {
  1758. const char *prop = splitXPathX(xpath);
  1759. if (isAttribute(prop))
  1760. {
  1761. MAKE_LSTRING(path, xpath, prop-xpath);
  1762. IPropertyTree *branch = queryPropTree(path);
  1763. if (!branch) return false;
  1764. return branch->getPropBin(prop, ret);
  1765. }
  1766. else
  1767. {
  1768. IPropertyTree *branch = queryPropTree(xpath);
  1769. if (!branch) return false;
  1770. return branch->getPropBin(NULL, ret);
  1771. }
  1772. }
  1773. }
  1774. void PTree::setPropBin(const char * xpath, size32_t size, const void *data)
  1775. {
  1776. CHECK_ATTRIBUTE(xpath);
  1777. if (!xpath || '\0' == *xpath)
  1778. setLocal(size, data, true);
  1779. else
  1780. {
  1781. const char *prop;
  1782. IPropertyTree *branch = splitBranchProp(xpath, prop, true);
  1783. if (isAttribute(prop))
  1784. branch->setPropBin(prop, size, data);
  1785. else
  1786. {
  1787. IPropertyTree *propBranch = queryCreateBranch(branch, prop);
  1788. propBranch->setPropBin(NULL, size, data);
  1789. }
  1790. }
  1791. }
  1792. void PTree::addPropBin(const char *xpath, size32_t size, const void *data)
  1793. {
  1794. CHECK_ATTRIBUTE(xpath);
  1795. if (!xpath || '\0' == *xpath)
  1796. addLocal(size, data, true);
  1797. else if ('[' == *xpath)
  1798. {
  1799. aindex_t pos = getChildMatchPos(xpath);
  1800. if ((aindex_t) -1 == pos)
  1801. throw MakeIPTException(-1, "addPropBin: qualifier unmatched %s", xpath);
  1802. addLocal(size, data, true, pos);
  1803. }
  1804. else
  1805. {
  1806. IPropertyTree *parent, *child;
  1807. StringAttr path, qualifier;
  1808. resolveParentChild(xpath, parent, child, path, qualifier);
  1809. if (parent != this)
  1810. parent->addPropBin(path, size, data);
  1811. else if (child)
  1812. child->addPropBin(qualifier, size, data);
  1813. else
  1814. setPropBin(path, size, data);
  1815. }
  1816. }
  1817. void PTree::appendPropBin(const char *xpath, size32_t size, const void *data)
  1818. {
  1819. CHECK_ATTRIBUTE(xpath);
  1820. if (!xpath || '\0' == *xpath)
  1821. appendLocal(size, data, true);
  1822. else if ('[' == *xpath)
  1823. {
  1824. IPropertyTree *qualified = queryPropTree(xpath);
  1825. if (!qualified)
  1826. throw MakeIPTException(-1, "appendPropBin: qualifier unmatched %s", xpath);
  1827. qualified->appendPropBin(nullptr, size, data);
  1828. }
  1829. else
  1830. {
  1831. IPropertyTree *parent, *child;
  1832. StringAttr path, qualifier;
  1833. resolveParentChild(xpath, parent, child, path, qualifier);
  1834. if (parent != this)
  1835. parent->appendPropBin(path, size, data);
  1836. else if (child)
  1837. child->appendPropBin(qualifier, size, data);
  1838. else
  1839. setPropBin(path, size, data);
  1840. }
  1841. }
  1842. IPropertyTree *PTree::getPropTree(const char *xpath) const
  1843. {
  1844. IPropertyTree *tree = queryPropTree(xpath);
  1845. return LINK(tree);
  1846. }
  1847. IPropertyTree *PTree::queryPropTree(const char *xpath) const
  1848. {
  1849. Owned<IPropertyTreeIterator> iter = getElements(xpath);
  1850. IPropertyTree *element = NULL;
  1851. if (iter->first())
  1852. {
  1853. element = &iter->query();
  1854. #ifdef _DEBUG
  1855. //The following call can double the cost of finding a match from an IPropertyTree
  1856. if (iter->next())
  1857. AMBIGUOUS_PATH("getProp",xpath);
  1858. #endif
  1859. }
  1860. return element;
  1861. }
  1862. void PTree::replaceSelf(IPropertyTree *val)
  1863. {
  1864. Owned<IAttributeIterator> aiter = getAttributes();
  1865. StringArray attrs;
  1866. ForEach (*aiter)
  1867. attrs.append(aiter->queryName());
  1868. ForEachItemIn(a, attrs)
  1869. removeProp(attrs.item(a));
  1870. ICopyArrayOf<IPropertyTree> elems;
  1871. Owned<IPropertyTreeIterator> iter = getElements("*");
  1872. ForEach(*iter)
  1873. elems.append(iter->query());
  1874. ForEachItemIn(e, elems)
  1875. removeTree(&elems.item(e));
  1876. aiter.setown(val->getAttributes());
  1877. ForEach(*aiter)
  1878. setProp(aiter->queryName(), aiter->queryValue());
  1879. iter.setown(val->getElements("*"));
  1880. ForEach(*iter)
  1881. {
  1882. IPropertyTree &node = iter->query();
  1883. node.Link();
  1884. addPropTree(node.queryName(), &node);
  1885. }
  1886. val->Release();
  1887. }
  1888. IPropertyTree *PTree::setPropTree(const char *xpath, IPropertyTree *val)
  1889. {
  1890. CHECK_ATTRIBUTE(xpath);
  1891. if (NULL == xpath)
  1892. {
  1893. replaceSelf(val);
  1894. return this;
  1895. }
  1896. else
  1897. {
  1898. StringAttr prop, qualifier;
  1899. IPropertyTree *branch, *child;
  1900. resolveParentChild(xpath, branch, child, prop, qualifier);
  1901. if (branch == this)
  1902. {
  1903. IPropertyTree *_val = ownPTree(val);
  1904. dbgassertex(QUERYINTERFACE(_val, PTree));
  1905. PTree *__val = static_cast<PTree *>(_val);
  1906. __val->setName(prop);
  1907. addingNewElement(*_val, ANE_SET);
  1908. if (!checkChildren()) createChildMap();
  1909. children->set(prop, _val);
  1910. return _val;
  1911. }
  1912. else
  1913. return branch->setPropTree(prop, val);
  1914. }
  1915. }
  1916. bool PTree::isArray(const char *xpath) const
  1917. {
  1918. if (!xpath || !*xpath) //item in an array child of parent? I don't think callers ever access array container directly
  1919. return arrayOwner && arrayOwner->isArray();
  1920. else if (isAttribute(xpath))
  1921. return false;
  1922. else
  1923. {
  1924. StringBuffer path;
  1925. const char *prop = splitXPath(xpath, path);
  1926. assertex(prop);
  1927. if (!isAttribute(prop))
  1928. {
  1929. if (path.length())
  1930. {
  1931. Owned<IPropertyTreeIterator> iter = getElements(path.str());
  1932. if (!iter->first())
  1933. return false;
  1934. IPropertyTree &branch = iter->query();
  1935. #ifdef _DEBUG
  1936. if (iter->next())
  1937. AMBIGUOUS_PATH("isArray", xpath);
  1938. #endif
  1939. return branch.isArray(prop);
  1940. }
  1941. else
  1942. {
  1943. IPropertyTree *child = children->query(xpath);
  1944. if (child)
  1945. {
  1946. PTree *tree = static_cast<PTree *>(child);
  1947. return (tree && tree->value && tree->value->isArray());
  1948. }
  1949. }
  1950. }
  1951. }
  1952. return false;
  1953. }
  1954. void PTree::addPTreeArrayItem(IPropertyTree *existing, const char *xpath, PTree *val, aindex_t pos)
  1955. {
  1956. IPropertyTree *iptval = static_cast<IPropertyTree *>(val);
  1957. PTree *tree = nullptr;
  1958. if (existing)
  1959. {
  1960. dbgassertex(QUERYINTERFACE(existing, PTree));
  1961. tree = static_cast<PTree *>(existing);
  1962. if (tree->value && tree->value->isArray())
  1963. {
  1964. val->setOwner(tree->value);
  1965. if ((aindex_t) -1 == pos)
  1966. tree->value->addElement(iptval);
  1967. else
  1968. tree->value->setElement(pos, iptval);
  1969. return;
  1970. }
  1971. }
  1972. IPTArrayValue *array = new CPTArray();
  1973. IPropertyTree *container = create(xpath, array);
  1974. val->setOwner(array);
  1975. if (existing)
  1976. {
  1977. array->addElement(LINK(existing));
  1978. assertex((aindex_t) -1 == pos || 0 == pos);
  1979. if ((aindex_t) -1 == pos)
  1980. array->addElement(iptval);
  1981. else
  1982. array->setElement(0, iptval);
  1983. tree->setOwner(array);
  1984. children->replace(xpath, container);
  1985. }
  1986. else
  1987. {
  1988. array->addElement(iptval);
  1989. children->set(xpath, container);
  1990. }
  1991. }
  1992. IPropertyTree *PTree::addPropTree(const char *xpath, IPropertyTree *val, bool alwaysUseArray)
  1993. {
  1994. if (!xpath || '\0' == *xpath)
  1995. throw MakeIPTException(PTreeExcpt_InvalidTagName, "Invalid xpath for property tree insertion specified");
  1996. else
  1997. {
  1998. CHECK_ATTRIBUTE(xpath);
  1999. const char *x = xpath;
  2000. for (;;)
  2001. {
  2002. if (!*x++)
  2003. {
  2004. IPropertyTree *_val = ownPTree(val);
  2005. dbgassertex(QUERYINTERFACE(_val, PTree));
  2006. PTree *__val = static_cast<PTree *>(_val);
  2007. /* NB: potentially param xpath is a reference to the existing name.
  2008. * So fetch new name ptr after set.
  2009. */
  2010. __val->setName(xpath);
  2011. xpath = __val->queryName();
  2012. addingNewElement(*_val, -1);
  2013. if (checkChildren())
  2014. {
  2015. IPropertyTree *child = children->query(xpath);
  2016. if (child)
  2017. {
  2018. addPTreeArrayItem(child, xpath, __val);
  2019. return _val;
  2020. }
  2021. }
  2022. else
  2023. createChildMap();
  2024. if (alwaysUseArray)
  2025. addPTreeArrayItem(nullptr, xpath, __val);
  2026. else
  2027. children->set(xpath, _val);
  2028. return _val;
  2029. }
  2030. if ('/' == *x || '[' == *x)
  2031. break;
  2032. }
  2033. IPropertyTree *parent, *child;
  2034. StringAttr path, qualifier;
  2035. resolveParentChild(xpath, parent, child, path, qualifier);
  2036. if (parent != this)
  2037. return parent->addPropTree(path, val);
  2038. else
  2039. {
  2040. aindex_t pos = (aindex_t)-1;
  2041. if (!qualifier.isEmpty())
  2042. {
  2043. pos = ((PTree *)child)->getChildMatchPos(qualifier);
  2044. if ((aindex_t) -1 == pos)
  2045. throw MakeIPTException(-1, "addPropTree: qualifier unmatched %s", xpath);
  2046. }
  2047. IPropertyTree *_val = ownPTree(val);
  2048. dbgassertex(QUERYINTERFACE(_val, PTree));
  2049. PTree *__val = static_cast<PTree *>(_val);
  2050. __val->setName(path);
  2051. addingNewElement(*_val, pos);
  2052. if (child)
  2053. {
  2054. addPTreeArrayItem(child, path, __val, pos);
  2055. }
  2056. else
  2057. {
  2058. if (!checkChildren()) createChildMap();
  2059. if (alwaysUseArray)
  2060. addPTreeArrayItem(nullptr, path, __val);
  2061. else
  2062. children->set(path, _val);
  2063. children->set(path, _val);
  2064. }
  2065. return _val;
  2066. }
  2067. }
  2068. }
  2069. IPropertyTree *PTree::addPropTree(const char *xpath, IPropertyTree *val)
  2070. {
  2071. return addPropTree(xpath, val, false);
  2072. }
  2073. IPropertyTree *PTree::addPropTreeArrayItem(const char *xpath, IPropertyTree *val)
  2074. {
  2075. return addPropTree(xpath, val, true);
  2076. }
  2077. bool PTree::removeTree(IPropertyTree *child)
  2078. {
  2079. if (child == this)
  2080. throw MakeIPTException(-1, "Cannot remove self");
  2081. if (checkChildren())
  2082. {
  2083. IPropertyTree *_child = children->query(child->queryName());
  2084. if (_child)
  2085. {
  2086. if (child == _child)
  2087. return children->removeExact(child);
  2088. else
  2089. {
  2090. IPTArrayValue *value = ((PTree *)_child)->queryValue();
  2091. if (value && value->isArray())
  2092. {
  2093. unsigned pos = value->find(child);
  2094. if (NotFound != pos)
  2095. {
  2096. removingElement(child, pos);
  2097. value->removeElement(pos);
  2098. if (0 == value->elements())
  2099. children->removeExact(_child);
  2100. return true;
  2101. }
  2102. }
  2103. }
  2104. }
  2105. }
  2106. return false;
  2107. }
  2108. bool PTree::removeProp(const char *xpath)
  2109. {
  2110. if (xpath && isAttribute(xpath))
  2111. return removeAttribute(xpath);
  2112. StringBuffer path;
  2113. const char *prop = splitXPath(xpath, path);
  2114. if (!prop)
  2115. throw MakeXPathException(xpath, PTreeExcpt_XPath_ParseError, 0, "Invalid xpath for property deletion");
  2116. if (path.length())
  2117. {
  2118. Owned<IPropertyTreeIterator> iter = getElements(path.str());
  2119. if (!iter)
  2120. return false;
  2121. bool res = false;
  2122. if (iter->first())
  2123. {
  2124. do
  2125. {
  2126. IPropertyTree *branch = &iter->query();
  2127. if (branch) {
  2128. res = branch->removeProp(prop);
  2129. if (res)
  2130. break; // deleted first may be another
  2131. }
  2132. }
  2133. while (iter->next());
  2134. }
  2135. return res;
  2136. }
  2137. else
  2138. {
  2139. if (!queryNextUnquoted(xpath, '[') && !strchr(prop, '*')) // have to work hard to locate qualified prop tree from parent.
  2140. {
  2141. if (!checkChildren()) return false;
  2142. return children->remove(prop); // NB: might be multivalued.
  2143. }
  2144. const char *xxpath = prop;
  2145. readID(xxpath, false);
  2146. const char *idEnd = xxpath;
  2147. if ('[' == *xxpath)
  2148. {
  2149. ++xxpath;
  2150. const char *digitStart = xxpath;
  2151. while (*xxpath && ']' != *xxpath && isdigit(*xxpath)) xxpath++;
  2152. assertex(*xxpath != '\0');
  2153. if (']' == *xxpath) // so it's a digit index!
  2154. {
  2155. StringAttr id(prop, idEnd-prop);
  2156. PTree *child = children?(PTree *)children->query(id):NULL;
  2157. if (child)
  2158. {
  2159. if (child->value && child->value->isArray() && child->value->elements()>1)
  2160. {
  2161. StringAttr digit(digitStart, xxpath-digitStart);
  2162. unsigned i = atoi(digit);
  2163. if (i && i <= child->value->elements())
  2164. {
  2165. removingElement(child->value->queryElement(i-1), i-1);
  2166. child->value->removeElement(i-1);
  2167. return true;
  2168. }
  2169. }
  2170. else
  2171. return children->removeExact(child);
  2172. }
  2173. return false;
  2174. }
  2175. }
  2176. // JCSMORE - This is ridiculous for qualifier have to iterate to find match ok, but then finding where that *was* gees!
  2177. Owned <IPropertyTreeIterator> iter = getElements(prop);
  2178. if (!iter->first())
  2179. return false;
  2180. IPropertyTree *match = &iter->query();
  2181. #if 0 // intentionally removes first encountered
  2182. if (iter->next())
  2183. {
  2184. AMBIGUOUS_PATH("removeProp",xpath);
  2185. }
  2186. #endif
  2187. return removeTree(match);
  2188. }
  2189. return false;
  2190. }
  2191. aindex_t PTree::queryChildIndex(IPropertyTree *child)
  2192. {
  2193. return findChild(child);
  2194. }
  2195. StringBuffer &PTree::getName(StringBuffer &ret) const
  2196. {
  2197. ret.append(queryName());
  2198. return ret;
  2199. }
  2200. typedef CopyReferenceArrayOf<AttrValue> AttrArray;
  2201. IAttributeIterator *PTree::getAttributes(bool sorted) const
  2202. {
  2203. class CAttributeIterator : implements IAttributeIterator, public CInterface
  2204. {
  2205. Linked<const PTree> parent;
  2206. AttrValue *cur = nullptr;
  2207. public:
  2208. IMPLEMENT_IINTERFACE;
  2209. CAttributeIterator(const PTree *_parent) : parent(_parent)
  2210. {
  2211. }
  2212. // IAttributeIterator impl.
  2213. virtual bool first() override
  2214. {
  2215. cur = parent->getNextAttribute(nullptr);
  2216. return cur ? true : false;
  2217. }
  2218. virtual bool next() override
  2219. {
  2220. cur = parent->getNextAttribute(cur);
  2221. return cur ? true : false;
  2222. }
  2223. virtual bool isValid() override { return cur ? true : false; }
  2224. virtual const char *queryName() const override
  2225. {
  2226. return cur->key.get();
  2227. }
  2228. virtual const char *queryValue() const override
  2229. {
  2230. return cur->value.get();
  2231. }
  2232. virtual StringBuffer &getValue(StringBuffer &out) override
  2233. {
  2234. out.append(queryValue());
  2235. return out;
  2236. }
  2237. virtual unsigned count() override { return parent->getAttributeCount(); }
  2238. };
  2239. class CSortedAttributeIterator : implements IAttributeIterator, public CInterface
  2240. {
  2241. typedef ArrayIteratorOf<AttrArray, AttrValue &> AttrIterator;
  2242. AttrArray attrs;
  2243. AttrValue *cur;
  2244. AttrIterator *iter;
  2245. Linked<const PTree> parent;
  2246. public:
  2247. IMPLEMENT_IINTERFACE;
  2248. static int compareAttrs(AttrValue * const *ll, AttrValue * const *rr)
  2249. {
  2250. return stricmp((*ll)->key.get(), (*rr)->key.get());
  2251. };
  2252. CSortedAttributeIterator(const PTree *_parent) : cur(NULL), iter(NULL), parent(_parent)
  2253. {
  2254. AttrValue *cur = parent->getNextAttribute(nullptr);
  2255. if (cur)
  2256. {
  2257. do
  2258. {
  2259. attrs.append(*cur);
  2260. cur = parent->getNextAttribute(cur);
  2261. }
  2262. while (cur);
  2263. attrs.sort(compareAttrs);
  2264. iter = new AttrIterator(attrs);
  2265. }
  2266. }
  2267. ~CSortedAttributeIterator()
  2268. {
  2269. if (iter)
  2270. delete iter;
  2271. }
  2272. // IAttributeIterator impl.
  2273. virtual bool first() override
  2274. {
  2275. if (!iter) return false;
  2276. if (!iter->first()) { cur = NULL; return false; }
  2277. cur = &iter->query();
  2278. return true;
  2279. }
  2280. virtual bool next() override
  2281. {
  2282. if (!iter) return false;
  2283. if (!iter->next()) { cur = NULL; return false; }
  2284. cur = &iter->query();
  2285. return true;
  2286. }
  2287. virtual bool isValid() override { return cur!=NULL; }
  2288. virtual const char *queryName() const override
  2289. {
  2290. assertex(cur);
  2291. return cur->key.get();
  2292. }
  2293. virtual const char *queryValue() const override
  2294. {
  2295. assertex(cur);
  2296. return cur->value.get();
  2297. }
  2298. virtual StringBuffer &getValue(StringBuffer &out) override
  2299. {
  2300. assertex(cur);
  2301. return out.append(queryValue());
  2302. }
  2303. virtual unsigned count() override { return attrs.ordinality(); }
  2304. };
  2305. if (sorted)
  2306. return new CSortedAttributeIterator(this);
  2307. else
  2308. return new CAttributeIterator(this);
  2309. }
  2310. ///////////////////
  2311. class CIndexIterator : implements IPropertyTreeIterator, public CInterface
  2312. {
  2313. Owned<IPropertyTreeIterator> subIter;
  2314. IPropertyTree *celem;
  2315. unsigned index, current;
  2316. public:
  2317. IMPLEMENT_IINTERFACE;
  2318. CIndexIterator(IPropertyTreeIterator *_subIter, unsigned _index) : subIter(_subIter), index(_index)
  2319. {
  2320. }
  2321. // IPropertyTreeIterator
  2322. virtual bool first() override
  2323. {
  2324. if (!index)
  2325. return false;
  2326. if (!subIter->first())
  2327. return false;
  2328. current = 1;
  2329. celem = NULL;
  2330. do
  2331. {
  2332. if (current == index)
  2333. {
  2334. celem = &subIter->query();
  2335. return true;
  2336. }
  2337. if (!subIter->next())
  2338. return false;
  2339. } while (++current <= index);
  2340. return false;
  2341. }
  2342. virtual bool isValid() override
  2343. {
  2344. return celem && (index >= current);
  2345. }
  2346. virtual bool next() override
  2347. {
  2348. celem = NULL;
  2349. return false;
  2350. }
  2351. virtual IPropertyTree & query() override
  2352. {
  2353. return *celem;
  2354. }
  2355. };
  2356. IPropertyTreeIterator *PTree::getElements(const char *xpath, IPTIteratorCodes flags) const
  2357. {
  2358. // NULL iterator for local value (i.e. maybe be single value or array)
  2359. if (NULL == xpath || '\0' == *xpath)
  2360. return new SingleIdIterator(*this);
  2361. Owned<IPropertyTreeIterator> iter;
  2362. const char *_xpath = xpath;
  2363. bool root=true;
  2364. restart:
  2365. switch (*xpath)
  2366. {
  2367. case '.':
  2368. root=false;
  2369. ++xpath;
  2370. if ('\0' == *xpath)
  2371. return new SingleIdIterator(*this);
  2372. else if ('/' != *xpath)
  2373. throw MakeXPathException(xpath-1, PTreeExcpt_XPath_Unsupported, 0, "\"/\" expected");
  2374. goto restart;
  2375. case '/':
  2376. ++xpath;
  2377. if ('/' == *xpath)
  2378. {
  2379. iter.setown(getElements(xpath+1));
  2380. if (checkChildren())
  2381. {
  2382. IPropertyTreeIterator *iter2 = new PTIdMatchIterator(this, "*", isnocase(), flags & iptiter_sort);
  2383. iter2 = new PTStackIterator(iter2, xpath-1);
  2384. SeriesPTIterator *series = new SeriesPTIterator();
  2385. series->addIterator(iter.getClear());
  2386. series->addIterator(iter2);
  2387. return series;
  2388. }
  2389. else
  2390. return iter.getClear();
  2391. }
  2392. else if (root)
  2393. throw MakeXPathException(xpath, PTreeExcpt_XPath_Unsupported, 0, "Root specifier \"/\" specifier is not supported");
  2394. else if ('\0' == *xpath)
  2395. return new SingleIdIterator(*this);
  2396. goto restart;
  2397. case '[':
  2398. {
  2399. ++xpath;
  2400. if (isdigit(*xpath)) {
  2401. StringAttr index;
  2402. xpath = readIndex(xpath, index);
  2403. unsigned i = atoi(index.get());
  2404. if (i)
  2405. {
  2406. if (value && value->isArray())
  2407. {
  2408. IPropertyTree *element = value->queryElement(--i);
  2409. if (element)
  2410. {
  2411. iter.setown(element->getElements(NULL));
  2412. }
  2413. }
  2414. else if (i == 1)
  2415. iter.setown(new SingleIdIterator(*this));
  2416. }
  2417. }
  2418. else
  2419. {
  2420. if (checkPattern(xpath))
  2421. iter.setown(new SingleIdIterator(*this));
  2422. }
  2423. if (']' != *xpath)
  2424. throw MakeXPathException(_xpath, PTreeExcpt_XPath_ParseError, xpath-_xpath, "Qualifier brace unclosed");
  2425. ++xpath;
  2426. break;
  2427. }
  2428. default:
  2429. {
  2430. bool wild;
  2431. const char *start = xpath;
  2432. readWildId(xpath, wild);
  2433. size32_t s = xpath-start;
  2434. if (s)
  2435. {
  2436. MAKE_LSTRING(id, start, s);
  2437. if (checkChildren())
  2438. {
  2439. IPropertyTree *child = NULL;
  2440. if (!wild)
  2441. child = children->query(id);
  2442. if ((wild || child) && '[' == *xpath) // check for local index not iterative qualifier.
  2443. {
  2444. const char *xxpath = xpath+1;
  2445. if (isdigit(*xxpath)) {
  2446. StringAttr idxstr;
  2447. xxpath = readIndex(xxpath, idxstr);
  2448. if (']' != *xxpath)
  2449. throw MakeXPathException(_xpath, PTreeExcpt_XPath_ParseError, xpath-_xpath, "Qualifier brace unclosed");
  2450. ++xxpath;
  2451. unsigned index = atoi(idxstr.get());
  2452. if (index)
  2453. {
  2454. Owned<IPropertyTreeIterator> _iter = getElements(id);
  2455. if (_iter->first())
  2456. {
  2457. do
  2458. {
  2459. if (0 == --index)
  2460. {
  2461. iter.setown(new SingleIdIterator((PTree &)_iter->query()));
  2462. break;
  2463. }
  2464. }
  2465. while (_iter->next());
  2466. }
  2467. }
  2468. xpath = xxpath;
  2469. }
  2470. else
  2471. {
  2472. const char *start = xxpath-1;
  2473. for (;;)
  2474. {
  2475. char quote = 0;
  2476. while (']' != *(++xxpath) || quote)
  2477. {
  2478. switch (*xxpath) {
  2479. case '\"':
  2480. case '\'':
  2481. {
  2482. if (quote)
  2483. {
  2484. if (*xxpath == quote)
  2485. quote = 0;
  2486. }
  2487. else
  2488. quote = *xxpath;
  2489. break;
  2490. }
  2491. case '\0':
  2492. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xxpath-start, "Qualifier brace unclosed");
  2493. }
  2494. }
  2495. ++xxpath;
  2496. if ('[' == *xxpath)
  2497. {
  2498. ++xxpath;
  2499. if (isdigit(*xxpath))
  2500. {
  2501. const char *lhsStart = start+1;
  2502. Owned<IPropertyTreeIterator> siter = checkMapIterator(lhsStart, *child);
  2503. if (!siter)
  2504. {
  2505. if (wild)
  2506. iter.setown(new PTIdMatchIterator(this, id, isnocase(), flags & iptiter_sort));
  2507. else
  2508. iter.setown(child->getElements(NULL));
  2509. StringAttr qualifier(start, (xxpath-1)-start);
  2510. siter.setown(new PTStackIterator(iter.getClear(), qualifier.get()));
  2511. }
  2512. StringAttr index;
  2513. xxpath = readIndex(xxpath, index);
  2514. unsigned i = atoi(index.get());
  2515. iter.setown(new CIndexIterator(siter.getClear(), i));
  2516. ++xxpath;
  2517. break;
  2518. }
  2519. }
  2520. else
  2521. {
  2522. if (!wild)
  2523. {
  2524. const char *lhsStart = start+1;
  2525. Owned<IPropertyTreeIterator> mapIter = checkMapIterator(lhsStart, *child);
  2526. if (mapIter)
  2527. {
  2528. xxpath = lhsStart;
  2529. iter.swap(mapIter);
  2530. break;
  2531. }
  2532. }
  2533. if (wild)
  2534. iter.setown(new PTIdMatchIterator(this, id, isnocase(), flags & iptiter_sort));
  2535. else
  2536. iter.setown(child->getElements(NULL));
  2537. StringAttr qualifier(start, xxpath-start);
  2538. iter.setown(new PTStackIterator(iter.getClear(), qualifier.get()));
  2539. break;
  2540. }
  2541. }
  2542. xpath = xxpath;
  2543. }
  2544. }
  2545. else
  2546. {
  2547. if (wild)
  2548. iter.setown(new PTIdMatchIterator(this, id, isnocase(), flags & iptiter_sort));
  2549. else if (child)
  2550. iter.setown(child->getElements(NULL));
  2551. }
  2552. }
  2553. }
  2554. break;
  2555. }
  2556. }
  2557. if (!iter)
  2558. iter.setown(LINK(nullPTreeIterator));
  2559. if (*xpath == '\0' || (*xpath == '/' && '\0' == *(xpath+1)))
  2560. return iter.getClear();
  2561. else
  2562. return new PTStackIterator(iter.getClear(), xpath);
  2563. }
  2564. void PTree::localizeElements(const char *xpath, bool allTail)
  2565. {
  2566. // null action for local ptree
  2567. }
  2568. unsigned PTree::numChildren() const
  2569. {
  2570. if (!checkChildren()) return 0;
  2571. return children->numChildren();
  2572. }
  2573. unsigned PTree::getCount(const char *xpath) const
  2574. {
  2575. unsigned c=0;
  2576. Owned<IPropertyTreeIterator> iter = getElements(xpath);
  2577. ForEach(*iter)
  2578. ++c;
  2579. return c;
  2580. }
  2581. void getXPathMatchTree(IPropertyTree &parentContext, const char *xpath, IPropertyTree *&matchContainer)
  2582. {
  2583. if (!xpath || !*xpath)
  2584. {
  2585. matchContainer = createPTree(parentContext.queryName());
  2586. return;
  2587. }
  2588. StringBuffer head;
  2589. const char *str = xpath;
  2590. const char *end = str+strlen(xpath);
  2591. bool quote = false;
  2592. bool inQualifier = false;
  2593. bool done = false;
  2594. bool recurse = false;
  2595. while (end != str)
  2596. {
  2597. switch (*str) {
  2598. case '"':
  2599. if (quote) quote = false;
  2600. else quote = true;
  2601. break;
  2602. case '[':
  2603. if (inQualifier)
  2604. {
  2605. if (!quote)
  2606. throw MakeXPathException(xpath, PTreeExcpt_XPath_ParseError, str-xpath, "Unclosed qualifier detected");
  2607. }
  2608. else
  2609. inQualifier = true;
  2610. break;
  2611. case ']':
  2612. if (inQualifier)
  2613. {
  2614. if (!quote)
  2615. inQualifier = false;
  2616. }
  2617. else if (!quote)
  2618. throw MakeXPathException(xpath, PTreeExcpt_XPath_ParseError, str-xpath, "Unopened qualifier detected");
  2619. break;
  2620. case '/':
  2621. if (!quote && !inQualifier)
  2622. {
  2623. if ('/' == *(str+1))
  2624. recurse = true;
  2625. done = true;
  2626. }
  2627. break;
  2628. }
  2629. if (done) break;
  2630. ++str;
  2631. }
  2632. const char *tail;
  2633. if (str==end) // top-level matches
  2634. {
  2635. head.append(xpath);
  2636. if (0 == head.length())
  2637. {
  2638. matchContainer = createPTree(xpath);
  2639. return;
  2640. }
  2641. tail = NULL;
  2642. }
  2643. else
  2644. {
  2645. head.append(str-xpath, xpath);
  2646. if (recurse)
  2647. tail = str+2;
  2648. else
  2649. tail = str+1;
  2650. }
  2651. Owned<IPropertyTreeIterator> parentIter = parentContext.getElements(head.str());
  2652. Owned<IPropertyTree> matchParent;
  2653. ForEach (*parentIter)
  2654. {
  2655. IPropertyTree &parent = parentIter->query();
  2656. if (!matchParent)
  2657. matchParent.setown(createPTree(parentContext.queryName()));
  2658. if (tail && *tail)
  2659. {
  2660. IPropertyTree *childContainer = NULL;
  2661. getXPathMatchTree(parent, tail, childContainer);
  2662. if (childContainer)
  2663. {
  2664. if (!head.length())
  2665. matchParent.setown(childContainer);
  2666. else
  2667. {
  2668. unsigned pos = ((PTree &)parentContext).findChild(&parent);
  2669. matchParent->addPropTree(childContainer->queryName(), childContainer);
  2670. childContainer->setPropInt("@pos", pos+1);
  2671. }
  2672. if (!matchContainer)
  2673. matchContainer = LINK(matchParent);
  2674. }
  2675. if (recurse)
  2676. {
  2677. Owned<IPropertyTreeIterator> iter = parent.getElements("*");
  2678. ForEach (*iter)
  2679. {
  2680. IPropertyTree *childContainer = NULL;
  2681. IPropertyTree &child = iter->query();
  2682. getXPathMatchTree(child, xpath, childContainer);
  2683. if (childContainer)
  2684. {
  2685. unsigned pos = ((PTree &)parent).findChild(&child);
  2686. matchParent->addPropTree(childContainer->queryName(), childContainer);
  2687. childContainer->setPropInt("@pos", pos+1);
  2688. if (!matchContainer)
  2689. matchContainer = LINK(matchParent);
  2690. }
  2691. }
  2692. }
  2693. }
  2694. else
  2695. {
  2696. if (&parent != &parentContext)
  2697. {
  2698. IPropertyTree *childContainer = matchParent->addPropTree(parent.queryName(), createPTree());
  2699. unsigned pos = ((PTree &)parentContext).findChild(&parent);
  2700. childContainer->setPropInt("@pos", pos+1);
  2701. }
  2702. if (!matchContainer)
  2703. matchContainer = LINK(matchParent);
  2704. }
  2705. }
  2706. }
  2707. IPropertyTree *getXPathMatchTree(IPropertyTree &parent, const char *xpath)
  2708. {
  2709. IPropertyTree *matchTree = NULL;
  2710. getXPathMatchTree(parent, xpath, matchTree);
  2711. return matchTree;
  2712. }
  2713. void PTree::serializeAttributes(MemoryBuffer &tgt)
  2714. {
  2715. IAttributeIterator *aIter = getAttributes();
  2716. if (aIter->first())
  2717. {
  2718. do
  2719. {
  2720. tgt.append(aIter->queryName());
  2721. tgt.append(aIter->queryValue());
  2722. }
  2723. while (aIter->next());
  2724. }
  2725. tgt.append(""); // attribute terminator. i.e. blank attr name.
  2726. aIter->Release();
  2727. }
  2728. void PTree::serializeSelf(MemoryBuffer &tgt)
  2729. {
  2730. const char *_name = queryName();
  2731. tgt.append(_name ? _name : "");
  2732. tgt.append(flags);
  2733. serializeAttributes(tgt);
  2734. if (value)
  2735. value->serialize(tgt);
  2736. else
  2737. tgt.append((size32_t)0);
  2738. }
  2739. void PTree::serializeCutOff(MemoryBuffer &tgt, int cutoff, int depth)
  2740. {
  2741. serializeSelf(tgt);
  2742. if (-1 == cutoff || depth<cutoff)
  2743. {
  2744. Owned<IPropertyTreeIterator> iter = getElements("*");
  2745. if (iter->first())
  2746. {
  2747. do
  2748. {
  2749. IPropertyTree *_child = &iter->query();
  2750. PTree *child = QUERYINTERFACE(_child, PTree); assertex(child);
  2751. child->serializeCutOff(tgt, cutoff, depth+1);
  2752. }
  2753. while (iter->next());
  2754. }
  2755. }
  2756. tgt.append(""); // element terminator. i.e. blank child name.
  2757. }
  2758. // serializable impl.
  2759. void PTree::serialize(MemoryBuffer &tgt)
  2760. {
  2761. serializeCutOff(tgt, -1, 0);
  2762. }
  2763. void PTree::deserialize(MemoryBuffer &src)
  2764. {
  2765. deserializeSelf(src);
  2766. StringAttr eName;
  2767. for (;;)
  2768. {
  2769. size32_t pos = src.getPos();
  2770. src.read(eName);
  2771. if (eName.isEmpty())
  2772. break;
  2773. src.reset(pos); // reset to re-read tree name
  2774. IPropertyTree *child = create(src);
  2775. addPropTree(eName, child);
  2776. }
  2777. }
  2778. void PTree::deserializeSelf(MemoryBuffer &src)
  2779. {
  2780. setName(NULL); // needs to be cleared before flags changed
  2781. StringAttr _name;
  2782. src.read(_name);
  2783. src.read(flags);
  2784. if (_name[0]==0)
  2785. setName(NULL);
  2786. else
  2787. setName(_name);
  2788. StringAttr attrName, attrValue;
  2789. for (;;)
  2790. {
  2791. src.read(attrName);
  2792. if (attrName.isEmpty())
  2793. break;
  2794. src.read(attrValue);
  2795. setProp(attrName, attrValue);
  2796. }
  2797. size32_t size;
  2798. unsigned pos = src.getPos();
  2799. src.read(size);
  2800. if (value) delete value;
  2801. if (size)
  2802. {
  2803. src.reset(pos);
  2804. value = new CPTValue(src);
  2805. }
  2806. else value = NULL;
  2807. }
  2808. IPropertyTree *PTree::clone(IPropertyTree &srcTree, bool self, bool sub)
  2809. {
  2810. IPropertyTree *_dstTree = self ? this : create(srcTree.queryName());
  2811. PTree *dstTree = QUERYINTERFACE(_dstTree, PTree);
  2812. dbgassertex(dstTree);
  2813. if (self)
  2814. dstTree->setName(srcTree.queryName());
  2815. clone(srcTree, *dstTree, sub);
  2816. return _dstTree;
  2817. }
  2818. void PTree::clone(IPropertyTree &srcTree, IPropertyTree &dstTree, bool sub)
  2819. {
  2820. PTree *_dstTree = QUERYINTERFACE((&dstTree), PTree); assertex(_dstTree); //JCSMORE
  2821. flags = _dstTree->flags;
  2822. if (srcTree.isBinary(NULL))
  2823. {
  2824. MemoryBuffer mb;
  2825. verifyex(srcTree.getPropBin(NULL, mb));
  2826. dstTree.setPropBin(NULL, mb.length(), mb.toByteArray());
  2827. }
  2828. else if (srcTree.isCompressed(NULL))
  2829. {
  2830. StringBuffer s;
  2831. verifyex(srcTree.getProp(NULL, s));
  2832. dstTree.setProp(NULL, s.str());
  2833. }
  2834. else
  2835. dstTree.setProp(NULL, srcTree.queryProp(NULL));
  2836. IAttributeIterator *attrs = srcTree.getAttributes();
  2837. if (attrs->first())
  2838. {
  2839. do
  2840. {
  2841. dstTree.setProp(attrs->queryName(), attrs->queryValue());
  2842. }
  2843. while (attrs->next());
  2844. }
  2845. attrs->Release();
  2846. if (sub)
  2847. {
  2848. Owned<IPropertyTreeIterator> iter = srcTree.getElements("*");
  2849. if (iter->first())
  2850. {
  2851. do
  2852. {
  2853. IPropertyTree &child = iter->query();
  2854. IPropertyTree *newChild = clone(child, false, sub);
  2855. dstTree.addPropTree(newChild->queryName(), newChild);
  2856. }
  2857. while (iter->next());
  2858. }
  2859. }
  2860. }
  2861. IPropertyTree *PTree::ownPTree(IPropertyTree *tree)
  2862. {
  2863. if (!isEquivalent(tree) || tree->IsShared() || isCaseInsensitive() != tree->isCaseInsensitive())
  2864. {
  2865. IPropertyTree *newTree = clone(*tree);
  2866. tree->Release();
  2867. return newTree;
  2868. }
  2869. else
  2870. return tree;
  2871. }
  2872. IPropertyTree *PTree::queryCreateBranch(IPropertyTree *branch, const char *prop, bool *newBranch)
  2873. {
  2874. IPropertyTree *childBranch = branch->queryPropTree(prop);
  2875. if (!childBranch)
  2876. {
  2877. if (newBranch) *newBranch = true;
  2878. childBranch = create(prop);
  2879. branch->setPropTree(prop, childBranch);
  2880. }
  2881. else if (newBranch) *newBranch = false;
  2882. return childBranch;
  2883. }
  2884. IPropertyTree *PTree::splitBranchProp(const char *xpath, const char *&prop, bool error)
  2885. {
  2886. prop = splitXPathX(xpath);
  2887. MAKE_LSTRING(path, xpath, prop-xpath);
  2888. IPropertyTree *branch = queryPropTree(path);
  2889. if (!branch && error)
  2890. throw MakeIPTException(-1, "path %s not found, when setting prop %s", path, xpath);
  2891. return branch;
  2892. }
  2893. IPropertyTree *_createPropBranch(IPropertyTree *tree, const char *xpath, bool createIntermediates, IPropertyTree *&created, IPropertyTree *&createdParent)
  2894. {
  2895. const char *prop;
  2896. StringBuffer path;
  2897. prop = splitXPathUQ(xpath, path);
  2898. IPropertyTree *branch = tree->queryPropTree(path.str());
  2899. if (!branch)
  2900. {
  2901. if (path.length() == strlen(xpath))
  2902. throw MakeIPTException(-1, "createPropBranch: cannot create path : %s", xpath);
  2903. if (!createIntermediates)
  2904. throw MakeIPTException(-1, "createPropBranch: no path found for : %s", path.str());
  2905. if ('/' == path.charAt(path.length()-1))
  2906. path.remove(path.length()-1, 1);
  2907. branch = _createPropBranch(tree, path.str(), createIntermediates, created, createdParent);
  2908. assertex(branch);
  2909. }
  2910. if (prop && '\0' != *prop && '@' != *prop)
  2911. {
  2912. IPropertyTree *_branch = branch->queryPropTree(prop);
  2913. if (_branch)
  2914. branch = _branch;
  2915. else
  2916. {
  2917. IPropertyTree *p = branch;
  2918. branch = branch->addPropTree(prop, createPTree());
  2919. if (!created) { created = branch; createdParent = p; }
  2920. }
  2921. }
  2922. return branch;
  2923. }
  2924. IPropertyTree *createPropBranch(IPropertyTree *tree, const char *xpath, bool createIntermediates, IPropertyTree **created, IPropertyTree **createdParent)
  2925. {
  2926. IPropertyTree *_created = NULL, *_createdParent = NULL;
  2927. try
  2928. {
  2929. IPropertyTree *ret = _createPropBranch(tree, xpath, createIntermediates, _created, _createdParent);
  2930. if (created) *created = _created;
  2931. if (createdParent) *createdParent = _createdParent;
  2932. return ret;
  2933. }
  2934. catch (...)
  2935. {
  2936. if (_created) (_createdParent)->removeTree(_created);
  2937. throw;
  2938. }
  2939. }
  2940. void PTree::addLocal(size32_t l, const void *data, bool _binary, int pos)
  2941. {
  2942. if (!l) return; // right thing to do on addProp("x", NULL) ?
  2943. IPTArrayValue *newValue = new CPTValue(l, data, _binary);
  2944. Owned<IPropertyTree> tree = create(queryName(), newValue);
  2945. PTree *_tree = QUERYINTERFACE(tree.get(), PTree); assertex(_tree);
  2946. if (_binary)
  2947. IptFlagSet(_tree->flags, ipt_binary);
  2948. else
  2949. IptFlagClr(_tree->flags, ipt_binary);
  2950. addingNewElement(*tree, pos);
  2951. IPTArrayValue *array;
  2952. if (value && value->isArray())
  2953. {
  2954. array = value;
  2955. if (pos != -1 && ((unsigned)pos > array->elements()))
  2956. throw MakeIPTException(-1, "Error trying to insert element at %d of %d", pos, array->elements());
  2957. }
  2958. else
  2959. {
  2960. if (pos > 0)
  2961. throw MakeIPTException(-1, "Error trying to insert element at %d of 0", pos);
  2962. // detach children and attributes of this branch now owned by element of newly created array.
  2963. IPropertyTree *element1 = detach();
  2964. array = new CPTArray();
  2965. addingNewElement(*element1, ANE_APPEND);
  2966. static_cast<PTree *>(element1)->setOwner(array);
  2967. array->addElement(element1);
  2968. value = array;
  2969. }
  2970. _tree->setOwner(array);
  2971. tree->Link();
  2972. if (-1 == pos)
  2973. array->addElement(tree);
  2974. else
  2975. array->setElement(pos, tree);
  2976. }
  2977. enum exprType { t_none, t_equality, t_inequality, t_lteq, t_lt, t_gt, t_gteq } tType;
  2978. inline bool match(bool wild, bool numeric, const char *xpath, exprType t, const char *value, unsigned len, const char *pat, unsigned patLen, bool nocase)
  2979. {
  2980. int m;
  2981. if (numeric)
  2982. {
  2983. __int64 lhsN = atoi64_l(value, len);
  2984. __int64 rhsN = atoi64_l(pat, patLen);
  2985. m = lhsN<rhsN?-1:lhsN>rhsN?1:0;
  2986. }
  2987. else if (wild)
  2988. m = false==WildMatch(value, len, pat, patLen, nocase);
  2989. else
  2990. {
  2991. if (len == patLen)
  2992. m = nocase ? memicmp(value, pat, len) : memcmp(value, pat, len);
  2993. else if (len < patLen)
  2994. m = -1;
  2995. else
  2996. m = 1;
  2997. }
  2998. switch (t)
  2999. {
  3000. case t_inequality:
  3001. return m!=0;
  3002. case t_lt:
  3003. return m<0;
  3004. case t_lteq:
  3005. return m<=0;
  3006. case t_equality:
  3007. return m==0;
  3008. case t_gteq:
  3009. return m>=0;
  3010. case t_gt:
  3011. return m>0;
  3012. }
  3013. throw MakeXPathException(xpath, PTreeExcpt_XPath_ParseError, 0, "Invalid xpath qualifier expression in xpath: %s", xpath);
  3014. }
  3015. bool PTree::checkPattern(const char *&xxpath) const
  3016. {
  3017. // Pattern is an additional filter at the current node level
  3018. // It can be [condition], or it can be empty (we don't support anything else)
  3019. // supported conditions are:
  3020. // tag - must have child called tag
  3021. // @attr - must have attribute called attr
  3022. // tag="value" - must have child called tag with given value
  3023. // @attr="value" - must have attribute called attr with given value
  3024. const char *xpath = xxpath;
  3025. while (*xpath == ' ' || *xpath == '\t') xpath++;
  3026. const char *start = xpath;
  3027. bool wild = false, nocase = isnocase();
  3028. if (*xpath=='@')
  3029. xpath++;
  3030. char quote = 0;
  3031. const char *lhsEnd, *quoteBegin, *quoteEnd, *rhsBegin, *rhsEnd;
  3032. lhsEnd = quoteBegin = quoteEnd = rhsBegin = rhsEnd = NULL;
  3033. exprType tType = t_none;
  3034. bool numeric=false;
  3035. #ifdef WARNLEGACYCOMPARE
  3036. bool legacynumeric=false;
  3037. #endif
  3038. for (;;)
  3039. {
  3040. switch (*xpath) {
  3041. case '"':
  3042. case '\'':
  3043. if (quote)
  3044. {
  3045. if (*xpath == quote)
  3046. {
  3047. quote = 0;
  3048. quoteEnd = xpath;
  3049. }
  3050. }
  3051. else
  3052. {
  3053. if (quoteBegin)
  3054. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Quoted left hand side already seen");
  3055. quote = *xpath;
  3056. quoteBegin = xpath+1;
  3057. }
  3058. break;
  3059. case '[':
  3060. if (!quote)
  3061. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Unclosed qualifier detected");
  3062. break;
  3063. case ']':
  3064. if (!quote)
  3065. {
  3066. if (!lhsEnd)
  3067. lhsEnd = xpath;
  3068. rhsEnd = xpath;
  3069. }
  3070. break;
  3071. case ' ':
  3072. case '\t':
  3073. if (!lhsEnd)
  3074. lhsEnd = xpath;
  3075. break;
  3076. case '!':
  3077. if (!quote)
  3078. {
  3079. if (tType)
  3080. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Unexpected expression operator xpath");
  3081. if ('=' != *(xpath+1))
  3082. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Invalid xpath qualifier expression in xpath");
  3083. if (!lhsEnd)
  3084. lhsEnd = xpath;
  3085. ++xpath;
  3086. tType = t_inequality;
  3087. wild = true; // true by default now, introduced ~ syntax, to denote wild string
  3088. }
  3089. break;
  3090. case '=':
  3091. if (!quote)
  3092. {
  3093. if (wild)
  3094. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Wildcard match '~' makes no sense in this context");
  3095. if (tType)
  3096. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Unexpected expression operator xpath");
  3097. tType = t_equality;
  3098. wild = true; // true by default now, introduced ~ syntax, to denote wild string
  3099. if (!lhsEnd)
  3100. lhsEnd = xpath;
  3101. }
  3102. break;
  3103. case '>':
  3104. if (!quote)
  3105. {
  3106. if (wild)
  3107. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Wildcard match '~' makes no sense in this context");
  3108. if (tType)
  3109. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Unexpected expression operator in xpath");
  3110. if (!lhsEnd)
  3111. lhsEnd = xpath;
  3112. #ifdef WARNLEGACYCOMPARE
  3113. legacynumeric = true;
  3114. #endif
  3115. if ('=' == *(xpath+1))
  3116. {
  3117. ++xpath;
  3118. tType = t_gteq;
  3119. }
  3120. else
  3121. tType = t_gt;
  3122. }
  3123. break;
  3124. case '<':
  3125. if (!quote)
  3126. {
  3127. if (tType)
  3128. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Unexpected expression operator in xpath");
  3129. if (!lhsEnd)
  3130. lhsEnd = xpath;
  3131. #ifdef WARNLEGACYCOMPARE
  3132. legacynumeric = true;
  3133. #endif
  3134. if ('=' == *(xpath+1))
  3135. {
  3136. ++xpath;
  3137. tType = t_lteq;
  3138. }
  3139. else
  3140. tType = t_lt;
  3141. }
  3142. break;
  3143. case '~':
  3144. if (!quote)
  3145. {
  3146. if (!tType)
  3147. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Unexpected wild operator in xpath");
  3148. wild = true;
  3149. }
  3150. break;
  3151. case '?':
  3152. if (!quote)
  3153. {
  3154. if (!tType)
  3155. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Unexpected case-insensitive operator in xpath");
  3156. nocase = true;
  3157. }
  3158. break;
  3159. case '\0':
  3160. rhsEnd = xpath;
  3161. break;
  3162. }
  3163. if (rhsEnd)
  3164. break;
  3165. xpath++;
  3166. if (!rhsBegin && tType && !isspace(*xpath))
  3167. rhsBegin = xpath;
  3168. }
  3169. if (quote)
  3170. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Parse error, unclosed quoted content");
  3171. if (tType)
  3172. {
  3173. if (quoteBegin && !quoteEnd)
  3174. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Parse error, RHS missing closing quote");
  3175. if (rhsBegin && !rhsEnd)
  3176. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Parse error, RHS missing closing quote");
  3177. if (!quoteBegin && rhsEnd) // validate it's a numeric
  3178. {
  3179. const char *c = rhsBegin;
  3180. for (;;)
  3181. {
  3182. if (!isdigit(*c++))
  3183. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Parse error, RHS is an unquoted string");
  3184. if (c==rhsEnd) break;
  3185. }
  3186. }
  3187. }
  3188. MAKE_LSTRING(lhs, start, lhsEnd-start);
  3189. bool ret = false;
  3190. const char *tProp = splitXPathX(lhs);
  3191. MAKE_LSTRING(head, lhs, tProp-lhs);
  3192. Owned<IPropertyTreeIterator> iter = getElements(head);
  3193. ForEach (*iter)
  3194. {
  3195. IPropertyTree &found = iter->query();
  3196. if (t_none == tType)
  3197. {
  3198. if (found.hasProp(tProp))
  3199. {
  3200. ret = true;
  3201. break;
  3202. }
  3203. }
  3204. else
  3205. {
  3206. Owned<IPropertyTreeIterator> _iter2;
  3207. IPropertyTreeIterator *iter2;
  3208. IPropertyTree *matchElem;
  3209. if (isAttribute(tProp))
  3210. {
  3211. matchElem = &found;
  3212. iter2 = NULL;
  3213. }
  3214. else
  3215. {
  3216. _iter2.setown(found.getElements(tProp));
  3217. iter2 = _iter2;
  3218. if (iter2->first())
  3219. matchElem = &iter2->query();
  3220. else
  3221. continue;
  3222. tProp = NULL;
  3223. }
  3224. for (;;)
  3225. {
  3226. if (matchElem->isBinary(tProp))
  3227. UNIMPLEMENTED;
  3228. const char *rhs;
  3229. unsigned rhslength;
  3230. if (quoteEnd)
  3231. {
  3232. rhs = quoteBegin;
  3233. rhslength = quoteEnd-quoteBegin;
  3234. #ifdef WARNLEGACYCOMPARE
  3235. if (legacynumeric)
  3236. {
  3237. if (isdigit(*rhs))
  3238. IWARNLOG("Possible deprecated use of quoted numeric comparison operation: %s", xxpath);
  3239. }
  3240. #endif
  3241. }
  3242. else if (rhsEnd)
  3243. {
  3244. rhs = rhsBegin;
  3245. rhslength = rhsEnd-rhsBegin;
  3246. numeric = true;
  3247. }
  3248. else
  3249. {
  3250. rhs = NULL;
  3251. rhslength = 0;
  3252. }
  3253. if (matchElem->isCompressed(tProp))
  3254. {
  3255. StringBuffer s;
  3256. matchElem->getProp(tProp, s);
  3257. ret = match(wild, numeric, xxpath, tType, s.str(), s.length(), rhs, rhslength, nocase);
  3258. }
  3259. else
  3260. {
  3261. const char *value = matchElem->queryProp(tProp);
  3262. if (value)
  3263. ret = match(wild, numeric, xxpath, tType, value, value?(size32_t)strlen(value):0, rhs, rhslength, nocase);
  3264. else if (tType == t_equality)
  3265. ret = (NULL == rhs || '\0' == *rhs);
  3266. else if (tType == t_inequality)
  3267. ret = (NULL != rhs && '\0' != *rhs);
  3268. }
  3269. if (ret)
  3270. break;
  3271. if (!iter2 || !iter2->next())
  3272. break;
  3273. matchElem = &iter2->query();
  3274. }
  3275. if (ret)
  3276. break;
  3277. }
  3278. }
  3279. xxpath = xpath;
  3280. return ret;
  3281. }
  3282. AttrValue *PTree::findAttribute(const char *key) const
  3283. {
  3284. if (attrs)
  3285. {
  3286. AttrValue *a = attrs+numAttrs;
  3287. if (isnocase())
  3288. {
  3289. while (a-- != attrs)
  3290. {
  3291. if (strieq(a->key.get(), key))
  3292. return a;
  3293. }
  3294. }
  3295. else
  3296. {
  3297. while (a-- != attrs)
  3298. {
  3299. if (streq(a->key.get(), key))
  3300. return a;
  3301. }
  3302. }
  3303. }
  3304. return nullptr;
  3305. }
  3306. const char *PTree::getAttributeValue(const char *key) const
  3307. {
  3308. AttrValue *e = findAttribute(key);
  3309. if (e)
  3310. return e->value.get();
  3311. return nullptr;
  3312. }
  3313. unsigned PTree::getAttributeCount() const
  3314. {
  3315. return numAttrs;
  3316. }
  3317. AttrValue *PTree::getNextAttribute(AttrValue *cur) const
  3318. {
  3319. if (0 == numAttrs)
  3320. return nullptr;
  3321. else if (nullptr == cur)
  3322. return attrs;
  3323. else
  3324. {
  3325. if (cur == (attrs+(numAttrs-1)))
  3326. return nullptr;
  3327. return ++cur;
  3328. }
  3329. }
  3330. //////////////////////
  3331. // LocalPTree
  3332. static RelaxedAtomic<unsigned> numLocalTrees;
  3333. unsigned queryNumLocalTrees()
  3334. {
  3335. return numLocalTrees;
  3336. }
  3337. LocalPTree::LocalPTree(const char *_name, byte _flags, IPTArrayValue *_value, ChildMap *_children) : PTree(_flags|ipt_fast, _value, _children)
  3338. {
  3339. if (_name)
  3340. setName(_name);
  3341. numLocalTrees++;
  3342. }
  3343. LocalPTree::~LocalPTree()
  3344. {
  3345. numLocalTrees--;
  3346. name.destroy();
  3347. if (!attrs)
  3348. return;
  3349. AttrValue *a = attrs+numAttrs;
  3350. while (a--!=attrs)
  3351. {
  3352. a->key.destroy();
  3353. a->value.destroy();
  3354. }
  3355. free(attrs);
  3356. }
  3357. const char *LocalPTree::queryName() const
  3358. {
  3359. return name.get();
  3360. }
  3361. void LocalPTree::setName(const char *_name)
  3362. {
  3363. if (_name==name.get())
  3364. return;
  3365. AttrStr *oname = name.getPtr(); // Don't free until after we copy - they could overlap
  3366. if (!name.set(_name))
  3367. name.setPtr(AttrStr::create(_name));
  3368. if (oname)
  3369. AttrStr::destroy(oname);
  3370. }
  3371. bool LocalPTree::removeAttribute(const char *key)
  3372. {
  3373. AttrValue *del = findAttribute(key);
  3374. if (!del)
  3375. return false;
  3376. if (arrayOwner)
  3377. {
  3378. CQualifierMap *map = arrayOwner->queryMap();
  3379. if (map)
  3380. map->removeEntryIfMapped(key, del->value.get(), this);
  3381. }
  3382. numAttrs--;
  3383. unsigned pos = del-attrs;
  3384. del->key.destroy();
  3385. del->value.destroy();
  3386. memmove(attrs+pos, attrs+pos+1, (numAttrs-pos)*sizeof(AttrValue));
  3387. return true;
  3388. }
  3389. void LocalPTree::setAttribute(const char *key, const char *val)
  3390. {
  3391. if (!key)
  3392. return;
  3393. if (!validateXMLTag(key+1))
  3394. throw MakeIPTException(-1, "Invalid xml attribute: %s", key);
  3395. if (!val)
  3396. val = ""; // cannot have NULL value
  3397. AttrValue *v = findAttribute(key);
  3398. AttrStr *goer = nullptr;
  3399. if (v)
  3400. {
  3401. if (streq(v->value.get(), val))
  3402. return;
  3403. goer = v->value.getPtr();
  3404. }
  3405. else
  3406. {
  3407. attrs = (AttrValue *)realloc(attrs, (numAttrs+1)*sizeof(AttrValue));
  3408. v = new(&attrs[numAttrs++]) AttrValue; // Initialize new AttrValue
  3409. if (!v->key.set(key))
  3410. v->key.setPtr(isnocase() ? AttrStr::createNC(key) : AttrStr::create(key));
  3411. }
  3412. if (arrayOwner)
  3413. {
  3414. CQualifierMap *map = arrayOwner->queryMap();
  3415. if (map)
  3416. {
  3417. if (goer)
  3418. map->replaceEntryIfMapped(key, v->value.get(), val, this);
  3419. else
  3420. map->insertEntryIfMapped(key, val, this);
  3421. }
  3422. }
  3423. if (!v->value.set(val))
  3424. v->value.setPtr(AttrStr::create(val));
  3425. if (goer)
  3426. AttrStr::destroy(goer);
  3427. }
  3428. #ifdef TRACE_STRING_SIZE
  3429. std::atomic<__int64> AttrStr::totsize { 0 };
  3430. std::atomic<__int64> AttrStr::maxsize { 0 };
  3431. #endif
  3432. #ifdef TRACE_ATOM_SIZE
  3433. std::atomic<__int64> AttrStrAtom::totsize { 0 };
  3434. std::atomic<__int64> AttrStrAtom::maxsize { 0 };
  3435. #endif
  3436. ///////////////////
  3437. static RelaxedAtomic<unsigned> numAtomTrees;
  3438. unsigned queryNumAtomTrees()
  3439. {
  3440. return numAtomTrees;
  3441. }
  3442. CAtomPTree::CAtomPTree(const char *_name, byte _flags, IPTArrayValue *_value, ChildMap *_children) : PTree(_flags|ipt_lowmem, _value, _children)
  3443. {
  3444. numAtomTrees++;
  3445. if (_name)
  3446. setName(_name);
  3447. }
  3448. CAtomPTree::~CAtomPTree()
  3449. {
  3450. numAtomTrees--;
  3451. bool nc = isnocase();
  3452. HashKeyElement *name_ptr = name.getPtr();
  3453. if (name_ptr)
  3454. {
  3455. AtomRefTable *kT = nc?keyTableNC:keyTable;
  3456. #ifdef TRACE_ATOM_SIZE
  3457. size_t gosize = sizeof(HashKeyElement)+strlen(name_ptr->get())+1;
  3458. if (kT->releaseKey(name_ptr))
  3459. AttrStrAtom::totsize -= gosize;
  3460. #else
  3461. kT->releaseKey(name_ptr);
  3462. #endif
  3463. }
  3464. if (!attrs)
  3465. return;
  3466. AttrValue *a = attrs+numAttrs;
  3467. {
  3468. CriticalBlock block(hashcrit);
  3469. while (a--!=attrs)
  3470. {
  3471. if (a->key.isPtr())
  3472. attrHT->removekey(a->key.getPtr(), nc);
  3473. if (a->value.isPtr())
  3474. attrHT->removeval(a->value.getPtr());
  3475. }
  3476. freeAttrArray(attrs, numAttrs);
  3477. }
  3478. }
  3479. void CAtomPTree::setName(const char *_name)
  3480. {
  3481. AtomRefTable *kT = isnocase()?keyTableNC:keyTable;
  3482. HashKeyElement *oname = name.getPtr(); // NOTE - don't release yet as could overlap source name
  3483. if (!_name)
  3484. name.setPtr(nullptr);
  3485. else
  3486. {
  3487. if (!validateXMLTag(_name))
  3488. throw MakeIPTException(PTreeExcpt_InvalidTagName, ": %s", _name);
  3489. if (!name.set(_name))
  3490. {
  3491. #ifdef TRACE_ALL_ATOM
  3492. DBGLOG("TRACE_ALL_ATOM: %s", _name);
  3493. #endif
  3494. #ifdef TRACE_ATOM_SIZE
  3495. bool didCreate;
  3496. name.setPtr(kT->queryCreate(_name, didCreate));
  3497. if (didCreate)
  3498. {
  3499. AttrStrAtom::totsize += sizeof(HashKeyElement)+strlen(_name)+1;
  3500. if (AttrStrAtom::totsize > AttrStrAtom::maxsize)
  3501. {
  3502. AttrStrAtom::maxsize.store(AttrStrAtom::totsize);
  3503. DBGLOG("TRACE_ATOM_SIZE: total size now %" I64F "d", AttrStrAtom::maxsize.load());
  3504. }
  3505. }
  3506. #else
  3507. name.setPtr(kT->queryCreate(_name));
  3508. #endif
  3509. }
  3510. }
  3511. if (oname)
  3512. {
  3513. #ifdef TRACE_ATOM_SIZE
  3514. size_t gosize = sizeof(HashKeyElement)+strlen(oname->get())+1;
  3515. if (kT->releaseKey(oname))
  3516. AttrStrAtom::totsize -= gosize;
  3517. #else
  3518. kT->releaseKey(oname);
  3519. #endif
  3520. }
  3521. }
  3522. const char *CAtomPTree::queryName() const
  3523. {
  3524. return name.get();
  3525. }
  3526. unsigned CAtomPTree::queryHash() const
  3527. {
  3528. if (name.isPtr())
  3529. {
  3530. assert(name.getPtr());
  3531. return name.getPtr()->queryHash();
  3532. }
  3533. else
  3534. {
  3535. const char *_name = name.get();
  3536. size32_t nl = strlen(_name);
  3537. return isnocase() ? hashnc((const byte *) _name, nl, 0): hashc((const byte *) _name, nl, 0);
  3538. }
  3539. }
  3540. AttrValue *CAtomPTree::newAttrArray(unsigned n)
  3541. {
  3542. // NB crit must be locked
  3543. if (!n)
  3544. return nullptr;
  3545. if (freelistmax<=n)
  3546. {
  3547. freelist = (AttrValue **)realloc(freelist, sizeof(AttrValue *)*(n+1));
  3548. while (freelistmax<=n)
  3549. freelist[freelistmax++] = nullptr;
  3550. }
  3551. AttrValue *&p = freelist[n];
  3552. AttrValue *ret = p;
  3553. if (ret)
  3554. p = *(AttrValue **)ret;
  3555. else
  3556. ret = (AttrValue *)freeallocator.alloc(sizeof(AttrValue)*n);
  3557. return ret;
  3558. }
  3559. void CAtomPTree::freeAttrArray(AttrValue *a, unsigned n)
  3560. {
  3561. // NB crit must be locked
  3562. if (a)
  3563. {
  3564. AttrValue *&p = freelist[n];
  3565. *(AttrValue **)a = p;
  3566. p = a;
  3567. }
  3568. }
  3569. void CAtomPTree::setAttribute(const char *key, const char *val)
  3570. {
  3571. if (!key)
  3572. return;
  3573. if (!validateXMLTag(key+1))
  3574. throw MakeIPTException(-1, "Invalid xml attribute: %s", key);
  3575. if (!val)
  3576. val = ""; // cannot have NULL value
  3577. AttrValue *v = findAttribute(key);
  3578. if (v)
  3579. {
  3580. if (streq(v->value.get(), val))
  3581. return;
  3582. if (arrayOwner)
  3583. {
  3584. CQualifierMap *map = arrayOwner->queryMap();
  3585. if (map)
  3586. map->replaceEntryIfMapped(key, v->value.get(), val, this);
  3587. }
  3588. AttrStr * goer = v->value.getPtr();
  3589. if (!v->value.set(val))
  3590. {
  3591. CriticalBlock block(hashcrit);
  3592. if (goer)
  3593. attrHT->removeval(goer);
  3594. v->value.setPtr(attrHT->addval(val));
  3595. }
  3596. else if (goer)
  3597. {
  3598. CriticalBlock block(hashcrit);
  3599. attrHT->removeval(goer);
  3600. }
  3601. }
  3602. else
  3603. {
  3604. CriticalBlock block(hashcrit);
  3605. AttrValue *newattrs = newAttrArray(numAttrs+1);
  3606. if (attrs)
  3607. {
  3608. memcpy(newattrs, attrs, numAttrs*sizeof(AttrValue));
  3609. freeAttrArray(attrs, numAttrs);
  3610. }
  3611. if (arrayOwner)
  3612. {
  3613. CQualifierMap *map = arrayOwner->queryMap();
  3614. if (map)
  3615. map->insertEntryIfMapped(key, val, this);
  3616. }
  3617. v = &newattrs[numAttrs];
  3618. if (!v->key.set(key))
  3619. v->key.setPtr(attrHT->addkey(key, isnocase()));
  3620. if (!v->value.set(val))
  3621. v->value.setPtr(attrHT->addval(val));
  3622. numAttrs++;
  3623. attrs = newattrs;
  3624. }
  3625. }
  3626. bool CAtomPTree::removeAttribute(const char *key)
  3627. {
  3628. AttrValue *del = findAttribute(key);
  3629. if (!del)
  3630. return false;
  3631. numAttrs--;
  3632. if (arrayOwner)
  3633. {
  3634. CQualifierMap *map = arrayOwner->queryMap();
  3635. if (map)
  3636. map->removeEntryIfMapped(key, del->value.get(), this);
  3637. }
  3638. CriticalBlock block(hashcrit);
  3639. if (del->key.isPtr())
  3640. attrHT->removekey(del->key.getPtr(), isnocase());
  3641. if (del->value.isPtr())
  3642. attrHT->removeval(del->value.getPtr());
  3643. AttrValue *newattrs = newAttrArray(numAttrs);
  3644. if (newattrs)
  3645. {
  3646. unsigned pos = del-attrs;
  3647. memcpy(newattrs, attrs, pos*sizeof(AttrValue));
  3648. memcpy(newattrs+pos, attrs+pos+1, (numAttrs-pos)*sizeof(AttrValue));
  3649. }
  3650. freeAttrArray(attrs, numAttrs+1);
  3651. attrs = newattrs;
  3652. return true;
  3653. }
  3654. ///////////////////
  3655. bool isEmptyPTree(const IPropertyTree *t)
  3656. {
  3657. if (!t)
  3658. return true;
  3659. if (t->numUniq())
  3660. return false;
  3661. Owned<IAttributeIterator> ai = t->getAttributes();
  3662. if (ai->first())
  3663. return false;
  3664. const char *s = t->queryProp(NULL);
  3665. if (s&&*s)
  3666. return false;
  3667. return true;
  3668. }
  3669. ///////////////////
  3670. PTLocalIteratorBase::PTLocalIteratorBase(const PTree *_tree, const char *_id, bool _nocase, bool _sort) : nocase(_nocase), sort(_sort), id(_id), tree(_tree)
  3671. {
  3672. class CPTArrayIterator : public ArrayIIteratorOf<IArrayOf<IPropertyTree>, IPropertyTree, IPropertyTreeIterator>
  3673. {
  3674. public:
  3675. CPTArrayIterator(IPropertyTreeIterator &src) : ArrayIIteratorOf<IArrayOf<IPropertyTree>, IPropertyTree, IPropertyTreeIterator>(elems)
  3676. {
  3677. ForEach(src)
  3678. elems.append(src.get());
  3679. elems.sort(comparePropTrees);
  3680. }
  3681. IArrayOf<IPropertyTree> elems;
  3682. };
  3683. tree->Link();
  3684. baseIter = tree->checkChildren()->getIterator(sort);
  3685. iter = NULL;
  3686. current = NULL;
  3687. }
  3688. PTLocalIteratorBase::~PTLocalIteratorBase()
  3689. {
  3690. baseIter->Release();
  3691. ::Release(iter);
  3692. tree->Release();
  3693. }
  3694. // IPropertyTreeIterator
  3695. bool PTLocalIteratorBase::first()
  3696. {
  3697. ::Release(iter); iter=NULL;
  3698. if (!baseIter || !baseIter->first()) return false;
  3699. return _next();
  3700. }
  3701. bool PTLocalIteratorBase::_next()
  3702. {
  3703. if (iter && iter->isValid() && iter->next())
  3704. return true;
  3705. for (;;)
  3706. {
  3707. for (;;)
  3708. {
  3709. if (!baseIter->isValid())
  3710. {
  3711. current = NULL;
  3712. return false;
  3713. }
  3714. else if (match())
  3715. break;
  3716. baseIter->next();
  3717. }
  3718. IPropertyTree *element = &baseIter->query();
  3719. baseIter->next();
  3720. if (iter)
  3721. iter->Release();
  3722. iter = element->getElements(NULL);
  3723. if (iter->first())
  3724. {
  3725. current = &iter->query();
  3726. return true;
  3727. }
  3728. }
  3729. }
  3730. bool PTLocalIteratorBase::next()
  3731. {
  3732. return _next();
  3733. }
  3734. bool PTLocalIteratorBase::isValid()
  3735. {
  3736. return (current != NULL);
  3737. }
  3738. /////////////////////////////
  3739. bool PTIdMatchIterator::match()
  3740. {
  3741. IPropertyTree &tree = baseIter->query();
  3742. const char *key = tree.queryName();
  3743. return (0 != WildMatch(key, id, nocase));
  3744. }
  3745. ////////////////////////////
  3746. SingleIdIterator::SingleIdIterator(const PTree &_tree, unsigned pos, unsigned _many) : many(_many), count(0), whichNext(pos-1), start(pos-1), current(NULL), tree(_tree)
  3747. {
  3748. tree.Link();
  3749. }
  3750. SingleIdIterator::~SingleIdIterator()
  3751. {
  3752. tree.Release();
  3753. }
  3754. void SingleIdIterator::setCurrent(unsigned pos)
  3755. {
  3756. current = tree.value->queryElement(pos);
  3757. }
  3758. // IInterface impl.
  3759. bool SingleIdIterator::first()
  3760. {
  3761. whichNext = start;
  3762. if (!tree.value || !tree.value->isArray())
  3763. {
  3764. if (0 == whichNext)
  3765. {
  3766. current = const_cast<PTree*>(&tree);
  3767. count = 1;
  3768. }
  3769. }
  3770. else
  3771. {
  3772. count = tree.value->elements();
  3773. if (whichNext < count)
  3774. setCurrent(whichNext);
  3775. else
  3776. return false;
  3777. }
  3778. ++whichNext;
  3779. return true;
  3780. }
  3781. bool SingleIdIterator::next()
  3782. {
  3783. if ((whichNext>=count) || ((unsigned) -1 != many && whichNext>start+many))
  3784. {
  3785. current = NULL;
  3786. return false;
  3787. }
  3788. setCurrent(whichNext++);
  3789. return true;
  3790. }
  3791. bool SingleIdIterator::isValid()
  3792. {
  3793. return (NULL != current);
  3794. }
  3795. //////////////
  3796. class StackElement
  3797. {
  3798. public:
  3799. void init(IPropertyTreeIterator *_iter, const char *_xpath)
  3800. {
  3801. xpath = (char *)strdup(_xpath);
  3802. iter=LINK(_iter);
  3803. }
  3804. void clear()
  3805. {
  3806. ::Release(iter);
  3807. if (xpath)
  3808. free(xpath);
  3809. }
  3810. IPropertyTreeIterator *get(StringAttr &str)
  3811. {
  3812. str.setown(xpath); return iter; // NB used in place of pop, as element invalid after call
  3813. }
  3814. IPropertyTreeIterator *iter;
  3815. char * xpath;
  3816. };
  3817. ///////////////////
  3818. PTStackIterator::PTStackIterator(IPropertyTreeIterator *_iter, const char *_xpath) : rootIter(_iter), xpath(_xpath)
  3819. {
  3820. iter = NULL;
  3821. xxpath = "";
  3822. current = NULL;
  3823. stacklen = 0;
  3824. stackmax = 4;
  3825. stack = (StackElement *)malloc(sizeof(StackElement)*stackmax);
  3826. }
  3827. PTStackIterator::~PTStackIterator()
  3828. {
  3829. while (stacklen)
  3830. stack[--stacklen].clear();
  3831. ::Release(iter);
  3832. ::Release(rootIter);
  3833. free(stack);
  3834. }
  3835. void PTStackIterator::setIterator(IPropertyTreeIterator *_iter)
  3836. {
  3837. assertex(_iter);
  3838. if (iter)
  3839. iter->Release();
  3840. iter = _iter;
  3841. iter->first();
  3842. }
  3843. // IIterator impl.
  3844. bool PTStackIterator::first()
  3845. {
  3846. while (stacklen)
  3847. stack[--stacklen].clear();
  3848. current = NULL;
  3849. xxpath = xpath;
  3850. rootIter->Link();
  3851. setIterator(rootIter);
  3852. return next();
  3853. }
  3854. bool PTStackIterator::isValid()
  3855. {
  3856. return (current != NULL);
  3857. }
  3858. IPropertyTree &PTStackIterator::query()
  3859. {
  3860. assertex(current);
  3861. return *current;
  3862. }
  3863. bool PTStackIterator::next()
  3864. {
  3865. bool separator = false;
  3866. if (iter)
  3867. {
  3868. IPropertyTree *element = NULL;
  3869. StringBuffer qualifierText;
  3870. for (;;)
  3871. {
  3872. while (!iter->isValid())
  3873. {
  3874. if (iter) iter->Release();
  3875. iter = popFromStack(stackPath); // leaves linked
  3876. if (!iter)
  3877. {
  3878. current = NULL;
  3879. return false;
  3880. }
  3881. xxpath = stackPath;
  3882. element = NULL;
  3883. }
  3884. if (!element)
  3885. {
  3886. element = &iter->query();
  3887. iter->next();
  3888. }
  3889. while (element)
  3890. {
  3891. switch (*xxpath)
  3892. {
  3893. case '\0':
  3894. current = element;
  3895. return true;
  3896. case '.':
  3897. if (separator) throw MakeXPathException(xpath, PTreeExcpt_XPath_ParseError, 0, "Syntax error");
  3898. separator=false;
  3899. ++xxpath;
  3900. if (*xpath && '/' != *xpath)
  3901. throw MakeXPathException(xpath-1, PTreeExcpt_XPath_Unsupported, 0, "\"/\" expected");
  3902. break;
  3903. case '/':
  3904. ++xxpath;
  3905. if ('/' == *xxpath)
  3906. {
  3907. --xxpath;
  3908. if (iter->isValid())
  3909. pushToStack(iter, xxpath);
  3910. setIterator(element->getElements(xxpath));
  3911. xxpath = "";
  3912. element = NULL;
  3913. }
  3914. separator=true;
  3915. break;
  3916. default:
  3917. separator=false;
  3918. if (iter->isValid())
  3919. pushToStack(iter, xxpath);
  3920. bool wild, numeric;
  3921. const char *start = xxpath;
  3922. readWildIdIndex(xxpath, wild, numeric);
  3923. size32_t s = xxpath-start;
  3924. if (s)
  3925. {
  3926. // NB: actually an id not qualifier, just sharing var.
  3927. qualifierText.clear().append(s, start);
  3928. bool mapped = false;
  3929. if (!wild && !numeric)
  3930. {
  3931. ChildMap *children = ((PTree *)element)->checkChildren();
  3932. if (children)
  3933. {
  3934. IPropertyTree *child = children->query(qualifierText);
  3935. if (child)
  3936. {
  3937. if ('[' == *xxpath)
  3938. {
  3939. const char *newXXPath = xxpath+1;
  3940. Owned<IPropertyTreeIterator> mapIter = checkMapIterator(newXXPath, *child);
  3941. if (mapIter)
  3942. {
  3943. setIterator(mapIter.getClear());
  3944. mapped = true;
  3945. xxpath = newXXPath;
  3946. }
  3947. }
  3948. }
  3949. }
  3950. }
  3951. if (!mapped)
  3952. setIterator(element->getElements(qualifierText));
  3953. }
  3954. else // must be qualifier.
  3955. {
  3956. if ('[' != *xxpath)
  3957. throw MakeXPathException(xxpath, PTreeExcpt_XPath_ParseError, 0, "Qualifier expected e.g. [..]");
  3958. const char *start = xxpath;
  3959. char quote = 0;
  3960. while (']' != *(++xxpath) || quote)
  3961. {
  3962. switch (*xxpath) {
  3963. case '\"':
  3964. case '\'':
  3965. {
  3966. if (quote)
  3967. {
  3968. if (*xxpath == quote)
  3969. quote = 0;
  3970. }
  3971. else
  3972. quote = *xxpath;
  3973. break;
  3974. }
  3975. case '\0':
  3976. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xxpath-start, "Qualifier brace unclosed");
  3977. }
  3978. }
  3979. ++xxpath;
  3980. qualifierText.clear().append(xxpath-start, start);
  3981. setIterator(element->getElements(qualifierText.str()));
  3982. }
  3983. element = NULL;
  3984. break;
  3985. }
  3986. }
  3987. }
  3988. }
  3989. return false;
  3990. }
  3991. void PTStackIterator::pushToStack(IPropertyTreeIterator *iter, const char *xpath)
  3992. {
  3993. if (stacklen==stackmax) {
  3994. stackmax *= 2;
  3995. stack = (StackElement *)realloc(stack, sizeof(StackElement)*stackmax);
  3996. }
  3997. stack[stacklen++].init(iter, xpath);
  3998. }
  3999. IPropertyTreeIterator *PTStackIterator::popFromStack(StringAttr &path)
  4000. {
  4001. if (!stacklen)
  4002. return NULL;
  4003. return stack[--stacklen].get(path);
  4004. }
  4005. #define DEFAULT_PTREE_TYPE LocalPTree
  4006. // factory methods
  4007. IPropertyTree *createPTree(MemoryBuffer &src, byte flags)
  4008. {
  4009. IPropertyTree *tree = createPTree(nullptr, flags);
  4010. tree->deserialize(src);
  4011. return tree;
  4012. }
  4013. IPropertyTree *createPTreeFromIPT(const IPropertyTree *srcTree, ipt_flags flags)
  4014. {
  4015. Owned<PTree> tree = (PTree *)createPTree(NULL, flags);
  4016. return tree->clone(*srcTree->queryBranch(NULL));
  4017. }
  4018. void mergePTree(IPropertyTree *target, IPropertyTree *toMerge)
  4019. {
  4020. Owned<IAttributeIterator> aiter = toMerge->getAttributes();
  4021. ForEach (*aiter)
  4022. target->addProp(aiter->queryName(), aiter->queryValue());
  4023. Owned<IPropertyTreeIterator> iter = toMerge->getElements("*");
  4024. ForEach (*iter)
  4025. {
  4026. IPropertyTree &e = iter->query();
  4027. target->addPropTree(e.queryName(), LINK(&e));
  4028. }
  4029. }
  4030. void _synchronizePTree(IPropertyTree *target, const IPropertyTree *source, bool removeTargetsNotInSource)
  4031. {
  4032. Owned<IAttributeIterator> aiter = target->getAttributes();
  4033. StringArray targetAttrs;
  4034. if (removeTargetsNotInSource)
  4035. {
  4036. ForEach (*aiter)
  4037. targetAttrs.append(aiter->queryName());
  4038. }
  4039. aiter.setown(source->getAttributes());
  4040. ForEach (*aiter)
  4041. {
  4042. const char *attr = aiter->queryName();
  4043. if (!target->hasProp(attr))
  4044. target->setProp(attr, aiter->queryValue());
  4045. else
  4046. {
  4047. const char *sValue = aiter->queryValue();
  4048. const char *tValue = target->queryProp(attr);
  4049. if (NULL == sValue)
  4050. {
  4051. if (NULL != tValue)
  4052. target->setProp(attr, sValue);
  4053. }
  4054. else if (NULL == tValue ||0 != strcmp(sValue, tValue))
  4055. target->setProp(attr, sValue);
  4056. if (removeTargetsNotInSource)
  4057. targetAttrs.zap(attr);
  4058. }
  4059. }
  4060. if (removeTargetsNotInSource)
  4061. {
  4062. // remaining
  4063. ForEachItemIn (a, targetAttrs)
  4064. target->removeProp(targetAttrs.item(a));
  4065. }
  4066. bool equal = true;
  4067. MemoryBuffer srcMb;
  4068. const char *src = NULL;
  4069. if (target->isBinary())
  4070. {
  4071. MemoryBuffer tgtMb;
  4072. target->getPropBin(NULL, tgtMb);
  4073. source->getPropBin(NULL, srcMb);
  4074. if (tgtMb.length() != srcMb.length())
  4075. equal = false;
  4076. else if (0 != memcmp(tgtMb.toByteArray(), srcMb.toByteArray(), tgtMb.length()))
  4077. equal = false;
  4078. }
  4079. else
  4080. {
  4081. const char *tgt = target->queryProp(NULL);
  4082. src = source->queryProp(NULL);
  4083. unsigned lTgt = tgt?(size32_t)strlen(tgt):0;
  4084. unsigned lSrc = src?(size32_t)strlen(src):0;
  4085. if (lTgt != lSrc)
  4086. equal = false;
  4087. else if (0 != lTgt && (0 != strcmp(tgt, src)))
  4088. equal = false;
  4089. }
  4090. if (!equal)
  4091. {
  4092. if (target->isBinary())
  4093. target->setPropBin(NULL, srcMb.length(), srcMb.toByteArray());
  4094. else
  4095. target->setProp(NULL, src);
  4096. }
  4097. ICopyArrayOf<IPropertyTree> toProcess;
  4098. Owned<IPropertyTreeIterator> iter = source->getElements("*");
  4099. ForEach (*iter)
  4100. toProcess.append(iter->query());
  4101. iter.setown(target->getElements("*"));
  4102. ICopyArrayOf<IPropertyTree> removeTreeList;
  4103. Owned<IPropertyTreeIterator> srcTypeIter;
  4104. StringAttr firstOfType;
  4105. ForEach (*iter)
  4106. {
  4107. IPropertyTree &e = iter->query();
  4108. const char *name = e.queryName();
  4109. IPropertyTree *sourceCompare;
  4110. if (!source->hasProp(name))
  4111. {
  4112. removeTreeList.append(e);
  4113. firstOfType.clear();
  4114. srcTypeIter.clear();
  4115. }
  4116. else
  4117. {
  4118. if (firstOfType.isEmpty() || 0 != strcmp(firstOfType, e.queryName()))
  4119. {
  4120. if (firstOfType.length() && srcTypeIter)
  4121. {
  4122. // add remaining
  4123. while (srcTypeIter->next())
  4124. {
  4125. sourceCompare = &srcTypeIter->query();
  4126. target->addPropTree(sourceCompare->queryName(), LINK(sourceCompare));
  4127. toProcess.zap(*sourceCompare);
  4128. }
  4129. }
  4130. srcTypeIter.setown(source->getElements(e.queryName()));
  4131. firstOfType.set(e.queryName());
  4132. assertex(srcTypeIter->first());
  4133. sourceCompare = &srcTypeIter->query();
  4134. }
  4135. else // 2nd of type etc..
  4136. sourceCompare = srcTypeIter->next() ? &srcTypeIter->query() : NULL;
  4137. if (sourceCompare)
  4138. {
  4139. toProcess.zap(*sourceCompare);
  4140. _synchronizePTree(&e, sourceCompare, removeTargetsNotInSource);
  4141. }
  4142. else
  4143. removeTreeList.append(e);
  4144. }
  4145. }
  4146. if (removeTargetsNotInSource)
  4147. {
  4148. ForEachItemIn (rt, removeTreeList)
  4149. target->removeTree(&removeTreeList.item(rt));
  4150. }
  4151. // add unprocessed source elements, not reference by name in target
  4152. ForEachItemIn (s, toProcess)
  4153. {
  4154. IPropertyTree &e = toProcess.item(s);
  4155. target->addPropTree(e.queryName(), LINK(&e));
  4156. }
  4157. }
  4158. /* ensure target is equivalent to source whilst retaining elements already present in target.
  4159. * presevers ordering of matching elements.
  4160. * If removeTargetsNotInSource = true (default) elements in the target not present in the source will be removed
  4161. */
  4162. void synchronizePTree(IPropertyTree *target, const IPropertyTree *source, bool removeTargetsNotInSource, bool rootsMustMatch)
  4163. {
  4164. if (rootsMustMatch)
  4165. {
  4166. const char *srcName = source->queryName();
  4167. const char *tgtName = target->queryName();
  4168. if (0 != strcmp(srcName, tgtName))
  4169. throw MakeIPTException(PTreeExcpt_Unsupported, "Cannot synchronize if root nodes mismatch");
  4170. }
  4171. _synchronizePTree(target, source, removeTargetsNotInSource);
  4172. }
  4173. IPropertyTree *ensurePTree(IPropertyTree *root, const char *xpath)
  4174. {
  4175. return createPropBranch(root, xpath, true);
  4176. }
  4177. IPTreeReadException *createPTreeReadException(int code, const char *msg, const char *context, unsigned line, offset_t offset)
  4178. {
  4179. //Do not use jlib_thrown_decl because it causes problems with VS2017 - I think because of beforeDispose() in CInterfaceOf.
  4180. //The type of the object actually thrown is IPTreeReadException which does have a jlib_thrown_decl - so it will still be caught.
  4181. class CPTreeReadException : implements CInterfaceOf<IPTreeReadException>
  4182. {
  4183. int code;
  4184. StringAttr msg;
  4185. StringAttr context;
  4186. unsigned line;
  4187. offset_t offset;
  4188. StringBuffer &getErrorMessage(StringBuffer &out) const
  4189. {
  4190. switch (code)
  4191. {
  4192. case PTreeRead_EOS:
  4193. return out.append("Error - end of stream");
  4194. case PTreeRead_syntax:
  4195. return out.append("Error - syntax error");
  4196. }
  4197. return out;
  4198. }
  4199. public:
  4200. CPTreeReadException(int _code, const char *_msg, const char *_context, unsigned _line, offset_t _offset) : code(_code), msg(_msg), context(_context), line(_line), offset(_offset) { }
  4201. // IException
  4202. int errorCode() const { return code; }
  4203. StringBuffer &errorMessage(StringBuffer &str) const
  4204. {
  4205. getErrorMessage(str);
  4206. if (msg.length())
  4207. str.append(" \"").append(msg).append("\"");
  4208. str.append(" [");
  4209. if (line>1) // don't bother with line 1, there may be no line breaks.
  4210. str.append("line ").append(line).append(", ");
  4211. str.append("file offset ").append(offset).append("]");
  4212. if (context.length())
  4213. str.newline().append(context);
  4214. return str;
  4215. }
  4216. MessageAudience errorAudience() const { return MSGAUD_user; }
  4217. const char *queryDescription() { return msg; }
  4218. unsigned queryLine() { return line; }
  4219. offset_t queryOffset() { return offset; }
  4220. const char *queryContext() { return context.get(); }
  4221. };
  4222. return new CPTreeReadException(code, msg, context, line, offset);
  4223. }
  4224. template <typename T>
  4225. class CommonReaderBase : public CInterface
  4226. {
  4227. Linked<ISimpleReadStream> lstream;
  4228. ISimpleReadStream *stream;
  4229. bool bufOwned, nullTerm;
  4230. byte *buf, *bufPtr;
  4231. size32_t bufSize, bufRemaining;
  4232. protected:
  4233. PTreeReaderOptions readerOptions;
  4234. bool ignoreWhiteSpace, noRoot;
  4235. Linked<IPTreeNotifyEvent> iEvent;
  4236. offset_t curOffset;
  4237. unsigned line;
  4238. char nextChar;
  4239. private:
  4240. void init()
  4241. {
  4242. ignoreWhiteSpace = 0 != ((unsigned)readerOptions & (unsigned)ptr_ignoreWhiteSpace);
  4243. noRoot = 0 != ((unsigned)readerOptions & (unsigned)ptr_noRoot);
  4244. }
  4245. void resetState()
  4246. {
  4247. bufPtr = buf;
  4248. nextChar = 0;
  4249. if (nullTerm || stream)
  4250. bufRemaining = 0;
  4251. curOffset = 0;
  4252. line = 0;
  4253. }
  4254. public:
  4255. CommonReaderBase(ISimpleReadStream &_stream, IPTreeNotifyEvent &_iEvent, PTreeReaderOptions _readerOptions, size32_t _bufSize=0) :
  4256. bufSize(_bufSize), readerOptions(_readerOptions), iEvent(&_iEvent)
  4257. {
  4258. if (!bufSize) bufSize = 0x20000;
  4259. buf = new byte[bufSize];
  4260. bufRemaining = 0;
  4261. curOffset = 0;
  4262. bufOwned = true;
  4263. nullTerm = false;
  4264. lstream.set(&_stream);
  4265. stream = &_stream; // for efficiency
  4266. init();
  4267. resetState();
  4268. }
  4269. CommonReaderBase(const void *_buf, size32_t bufLength, IPTreeNotifyEvent &_iEvent, PTreeReaderOptions _readerOptions) :
  4270. readerOptions(_readerOptions), iEvent(&_iEvent)
  4271. {
  4272. bufSize = 0; // not used for direct reads
  4273. stream = NULL; // not used for direct reads
  4274. bufRemaining = bufLength;
  4275. nullTerm = false;
  4276. buf = (byte *)_buf;
  4277. bufOwned = false;
  4278. init();
  4279. resetState();
  4280. }
  4281. CommonReaderBase(const void *_buf, IPTreeNotifyEvent &_iEvent, PTreeReaderOptions _readerOptions) :
  4282. readerOptions(_readerOptions), iEvent(&_iEvent)
  4283. {
  4284. bufSize = 0; // not used for direct reads
  4285. stream = NULL; // not used for direct reads
  4286. curOffset = 0;
  4287. bufRemaining = 0;
  4288. nullTerm = true;
  4289. buf = (byte *)_buf;
  4290. bufOwned = false;
  4291. init();
  4292. resetState();
  4293. }
  4294. ~CommonReaderBase()
  4295. {
  4296. if (bufOwned)
  4297. delete [] buf;
  4298. }
  4299. protected:
  4300. virtual void reset()
  4301. {
  4302. resetState();
  4303. }
  4304. void rewind(size32_t n)
  4305. {
  4306. assertex(curOffset >= n);
  4307. if (!n) return;
  4308. curOffset -= n;
  4309. size32_t d = (size32_t)(bufPtr-buf);
  4310. if (n > d) n = d;
  4311. if (!nullTerm)
  4312. bufRemaining += n;
  4313. for (;;)
  4314. {
  4315. --bufPtr;
  4316. if (!--n) break;
  4317. if (10 == *bufPtr) --line;
  4318. }
  4319. }
  4320. bool checkBOM()
  4321. {
  4322. bool utf16 = false;
  4323. bool utf8 = false;
  4324. // Note - technically the utf16 LE case could also be utf32 LE (utf32 BE would be 0x00 0x00 0xfe 0xff)
  4325. // But utf32 is so rare that we ignore it for now
  4326. switch ((unsigned char)nextChar)
  4327. {
  4328. case 0xff:
  4329. readNext();
  4330. if (0xfe == (unsigned char)nextChar)
  4331. utf16 = true;
  4332. break;
  4333. case 0xfe:
  4334. readNext();
  4335. if (0xff == (unsigned char)nextChar)
  4336. utf16 = true;
  4337. break;
  4338. case 0xef:
  4339. readNext();
  4340. if (0xbb == (unsigned char)nextChar)
  4341. {
  4342. readNext();
  4343. if (0xbf == (unsigned char)nextChar)
  4344. utf8 = true;
  4345. }
  4346. break;
  4347. default:
  4348. break;
  4349. }
  4350. if (utf8)
  4351. return true;
  4352. else if (utf16)
  4353. error("Unsupported utf16 format detected in BOM header", false);
  4354. return false;
  4355. }
  4356. inline void expecting(const char *str)
  4357. {
  4358. StringBuffer errorMsg("Expecting \"");
  4359. error(errorMsg.append(str).append("\"").str());
  4360. }
  4361. inline void eos()
  4362. {
  4363. error("String terminator hit");
  4364. }
  4365. void match(const char *txt, const char *msg=NULL)
  4366. {
  4367. const char *c = txt;
  4368. for (;;)
  4369. {
  4370. if (*c == '\0') break;
  4371. readNext();
  4372. if (toupper(nextChar) != toupper(*c))
  4373. {
  4374. if (msg)
  4375. error(msg);
  4376. throw c;
  4377. }
  4378. c++;
  4379. }
  4380. }
  4381. void error(const char *msg=NULL, bool giveContext=true, PTreeReadExcptCode code=PTreeRead_syntax) __attribute__((noreturn))
  4382. {
  4383. StringBuffer context;
  4384. if (giveContext)
  4385. {
  4386. size32_t bufPos = (size32_t)(bufPtr-buf);
  4387. unsigned preLen = std::min(40U, bufPos);
  4388. size32_t bR = bufRemaining;
  4389. if (nullTerm)
  4390. {
  4391. byte *tPtr = bufPtr;
  4392. while (bR<40)
  4393. {
  4394. if ('\0' == *tPtr++) break;
  4395. bR++;
  4396. }
  4397. }
  4398. unsigned postLen = std::min(80-preLen, bR);
  4399. const char *bufferContext = (const char *)(bufPtr - preLen);
  4400. context.append(preLen, bufferContext);
  4401. context.append("*ERROR*");
  4402. context.append(postLen, bufferContext+preLen);
  4403. }
  4404. throw createPTreeReadException(code, msg, context.str(), line+1, curOffset);
  4405. }
  4406. inline void readNext()
  4407. {
  4408. if (!readNextToken())
  4409. error("End of stream encountered whilst parsing", true, PTreeRead_EOS);
  4410. curOffset++;
  4411. }
  4412. inline bool checkReadNext()
  4413. {
  4414. if (!readNextToken())
  4415. return false;
  4416. curOffset++;
  4417. return true;
  4418. }
  4419. inline bool checkStartReadNext()
  4420. {
  4421. if (curOffset || nextChar) //not at starting state
  4422. return true;
  4423. return readNextToken();
  4424. }
  4425. inline bool readNextToken();
  4426. inline bool checkSkipWS()
  4427. {
  4428. while (isspace(nextChar)) if (!checkReadNext()) return false;
  4429. return true;
  4430. }
  4431. inline void skipWS()
  4432. {
  4433. while (isspace(nextChar)) readNext();
  4434. }
  4435. };
  4436. class CInstStreamReader { public: }; // only used to ensure different template definitions.
  4437. class CInstBufferReader { public: };
  4438. class CInstStringReader { public: };
  4439. template <> inline bool CommonReaderBase<CInstStreamReader>::readNextToken()
  4440. {
  4441. // do own buffering, to have reasonable error context.
  4442. if (0 == bufRemaining)
  4443. {
  4444. size32_t _bufRemaining = stream->read(bufSize, buf);
  4445. if (!_bufRemaining)
  4446. return false;
  4447. bufRemaining = _bufRemaining;
  4448. bufPtr = buf;
  4449. }
  4450. --bufRemaining;
  4451. nextChar = *bufPtr++;
  4452. if (10 == nextChar)
  4453. line++;
  4454. return true;
  4455. }
  4456. template <> inline bool CommonReaderBase<CInstBufferReader>::readNextToken()
  4457. {
  4458. if (0 == bufRemaining)
  4459. return false;
  4460. --bufRemaining;
  4461. nextChar = *bufPtr++;
  4462. if (10 == nextChar)
  4463. line++;
  4464. return true;
  4465. }
  4466. template <> inline bool CommonReaderBase<CInstStringReader>::readNextToken()
  4467. {
  4468. nextChar = *bufPtr++;
  4469. if ('\0' == nextChar)
  4470. {
  4471. --bufPtr;
  4472. return false;
  4473. }
  4474. if (10 == nextChar)
  4475. line++;
  4476. return true;
  4477. }
  4478. template <typename X>
  4479. class CXMLReaderBase : public CommonReaderBase<X>, implements IEntityHelper
  4480. {
  4481. StringAttrMapping entityTable;
  4482. protected:
  4483. bool ignoreNameSpaces;
  4484. bool hadXMLDecl;
  4485. private:
  4486. void init()
  4487. {
  4488. ignoreNameSpaces = 0 != ((unsigned) readerOptions & (unsigned)ptr_ignoreNameSpaces);
  4489. }
  4490. void resetState()
  4491. {
  4492. hadXMLDecl = false;
  4493. }
  4494. public:
  4495. typedef CommonReaderBase<X> PARENT;
  4496. using PARENT::nextChar;
  4497. using PARENT::readNext;
  4498. using PARENT::expecting;
  4499. using PARENT::match;
  4500. using PARENT::error;
  4501. using PARENT::skipWS;
  4502. using PARENT::rewind;
  4503. using PARENT::readerOptions;
  4504. CXMLReaderBase(ISimpleReadStream &_stream, IPTreeNotifyEvent &_iEvent, PTreeReaderOptions _xmlReaderOptions, size32_t _bufSize=0)
  4505. : CommonReaderBase<X>(_stream, _iEvent, _xmlReaderOptions, _bufSize)
  4506. {
  4507. init();
  4508. resetState();
  4509. }
  4510. CXMLReaderBase(const void *_buf, size32_t bufLength, IPTreeNotifyEvent &_iEvent, PTreeReaderOptions _xmlReaderOptions)
  4511. : CommonReaderBase<X>(_buf, bufLength, _iEvent, _xmlReaderOptions)
  4512. {
  4513. init();
  4514. resetState();
  4515. }
  4516. CXMLReaderBase(const void *_buf, IPTreeNotifyEvent &_iEvent, PTreeReaderOptions _xmlReaderOptions)
  4517. : CommonReaderBase<X>(_buf, _iEvent, _xmlReaderOptions)
  4518. {
  4519. init();
  4520. resetState();
  4521. }
  4522. protected:
  4523. virtual void reset() override
  4524. {
  4525. resetState();
  4526. PARENT::reset();
  4527. }
  4528. void readID(StringBuffer &id)
  4529. {
  4530. if (isValidXPathStartChr(nextChar))
  4531. {
  4532. for (;;)
  4533. {
  4534. id.append(nextChar);
  4535. readNext();
  4536. if (!isValidXPathChr(nextChar)) break;
  4537. }
  4538. }
  4539. }
  4540. void skipString()
  4541. {
  4542. if ('"' == nextChar)
  4543. {
  4544. do { readNext(); } while ('"' != nextChar);
  4545. }
  4546. else if ('\'' == nextChar)
  4547. {
  4548. do { readNext(); } while ('\'' != nextChar);
  4549. }
  4550. else expecting("\" or '");
  4551. }
  4552. bool lookupRefValue(const char *name, StringBuffer &value)
  4553. {
  4554. StringAttr *val = entityTable.getValue(name);
  4555. if (!val) return false;
  4556. value.append(*val);
  4557. return true;
  4558. }
  4559. void storeEntity(const char *name, const char *value)
  4560. {
  4561. entityTable.setValue(name, value);
  4562. }
  4563. void parseEntity()
  4564. {
  4565. try { match("NTITY"); }
  4566. catch (const char *) { error("Bad syntax"); }
  4567. readNext();
  4568. skipWS();
  4569. StringBuffer entityName;
  4570. if ('%' != nextChar)
  4571. {
  4572. readID(entityName);
  4573. skipWS();
  4574. if ('"' == nextChar)
  4575. {
  4576. StringBuffer refValue;
  4577. for (;;)
  4578. {
  4579. readNext();
  4580. if (!nextChar || '"' == nextChar)
  4581. break;
  4582. if ('&' == nextChar)
  4583. {
  4584. readNext();
  4585. StringBuffer ref;
  4586. if ('#' == nextChar)
  4587. {
  4588. ref.append("&#");
  4589. for (;;)
  4590. {
  4591. readNext();
  4592. if (!nextChar)
  4593. expecting(";");
  4594. if (';' == nextChar) break;
  4595. ref.append(nextChar);
  4596. }
  4597. ref.append(";");
  4598. decodeXML(ref, refValue);
  4599. }
  4600. else
  4601. {
  4602. readID(ref);
  4603. if (';' != nextChar)
  4604. expecting(";");
  4605. if (!lookupRefValue(ref, refValue))
  4606. {
  4607. StringBuffer _ref("&");
  4608. _ref.append(ref).append(';');
  4609. decodeXML(ref, refValue); // try inbuilts
  4610. }
  4611. }
  4612. }
  4613. else
  4614. refValue.append(nextChar);
  4615. }
  4616. storeEntity(entityName, refValue);
  4617. }
  4618. }
  4619. do { readNext(); }
  4620. while (nextChar && nextChar != '>');
  4621. }
  4622. void parseIntSubset()
  4623. {
  4624. for (;;)
  4625. {
  4626. readNext();
  4627. skipWS();
  4628. if (']'== nextChar) break;
  4629. if ('<' == nextChar)
  4630. {
  4631. readNext();
  4632. switch (nextChar)
  4633. {
  4634. case '!':
  4635. {
  4636. readNext();
  4637. switch (nextChar)
  4638. {
  4639. case '-':
  4640. parseComment();
  4641. break;
  4642. case 'E':
  4643. parseEntity();
  4644. break;
  4645. default: // ignore anything else
  4646. do { readNext(); }
  4647. while (nextChar && nextChar != '>');
  4648. break;
  4649. }
  4650. break;
  4651. }
  4652. case '?':
  4653. {
  4654. StringBuffer pi;
  4655. parsePI(pi);
  4656. break;
  4657. }
  4658. }
  4659. }
  4660. }
  4661. }
  4662. void parseOther()
  4663. {
  4664. switch (nextChar)
  4665. {
  4666. case '-':
  4667. parseComment2();
  4668. break;
  4669. case 'D':
  4670. {
  4671. try { match("OCTYPE"); }
  4672. catch (const char *) { error("Bad syntax"); }
  4673. readNext();
  4674. skipWS();
  4675. StringBuffer doctypeid;
  4676. readID(doctypeid);
  4677. for (;;)
  4678. {
  4679. skipWS();
  4680. if ('>' == nextChar) break;
  4681. if ('[' == nextChar)
  4682. {
  4683. parseIntSubset();
  4684. if (']' != nextChar)
  4685. expecting("]");
  4686. }
  4687. else if ('S' == nextChar)
  4688. {
  4689. match("YSTEM");
  4690. readNext();
  4691. skipWS();
  4692. skipString();
  4693. }
  4694. else if ('P' == nextChar)
  4695. {
  4696. match("UBLIC");
  4697. readNext();
  4698. skipWS();
  4699. skipString();
  4700. readNext();
  4701. skipWS();
  4702. skipString();
  4703. }
  4704. readNext();
  4705. }
  4706. break;
  4707. }
  4708. default:
  4709. error("Invalid information tag");
  4710. }
  4711. }
  4712. void parsePIOrDecl()
  4713. {
  4714. StringBuffer target;
  4715. parsePI(target);
  4716. if (0 == strcmp("xml", target.str()))
  4717. {
  4718. if (hadXMLDecl)
  4719. error("Only one XML declartion permitted");
  4720. hadXMLDecl = true;
  4721. }
  4722. }
  4723. void parseCData(StringBuffer &text)
  4724. {
  4725. try { match("CDATA["); }
  4726. catch (const char *) { error("Bad CDATA syntax"); }
  4727. for (;;)
  4728. {
  4729. readNext();
  4730. while (']' == nextChar)
  4731. {
  4732. readNext();
  4733. while (']' == nextChar)
  4734. {
  4735. readNext();
  4736. if ('>' == nextChar)
  4737. return;
  4738. else
  4739. text.append(']');
  4740. }
  4741. text.append(']');
  4742. }
  4743. text.append(nextChar);
  4744. }
  4745. }
  4746. void parsePI(StringBuffer &target)
  4747. {
  4748. readNext();
  4749. if (!isValidXPathStartChr(nextChar))
  4750. error("Invalid PI target");
  4751. for (;;)
  4752. {
  4753. target.append(nextChar);
  4754. readNext();
  4755. if (!isValidXPathChr(nextChar))
  4756. break;
  4757. }
  4758. skipWS();
  4759. unsigned closeTag=0;
  4760. for (;;)
  4761. {
  4762. if (!nextChar)
  4763. error("Missing closing PI tag ?>");
  4764. if (1 == closeTag)
  4765. {
  4766. if ('>' == nextChar)
  4767. break;
  4768. closeTag = 0;
  4769. }
  4770. else if ('?' == nextChar)
  4771. closeTag = 1;
  4772. readNext();
  4773. }
  4774. }
  4775. void parseDirective(StringBuffer &res)
  4776. {
  4777. readNext();
  4778. switch (nextChar) {
  4779. case '-':
  4780. parseComment2();
  4781. break;
  4782. case '[':
  4783. parseCData(res);
  4784. break;
  4785. default:
  4786. error("Unrecognised syntax");
  4787. }
  4788. }
  4789. void parseComment()
  4790. {
  4791. readNext();
  4792. if (nextChar != '-') error("Bad comment syntax");
  4793. parseComment2();
  4794. }
  4795. void parseComment2()
  4796. {
  4797. readNext();
  4798. if (nextChar != '-') error("Bad comment syntax");
  4799. readNext();
  4800. unsigned seen = 0;
  4801. while (nextChar)
  4802. {
  4803. if (seen==2)
  4804. {
  4805. if (nextChar=='>')
  4806. return;
  4807. else if (nextChar != '-') // should be syntax error really.
  4808. seen = 0;
  4809. }
  4810. else if (nextChar=='-')
  4811. seen++;
  4812. else
  4813. seen = 0;
  4814. readNext();
  4815. }
  4816. error("Bad comment syntax");
  4817. }
  4818. const char *_decodeXML(unsigned read, const char *startMark, StringBuffer &ret)
  4819. {
  4820. const char *errMark = NULL;
  4821. try { return decodeXML(startMark, ret, &errMark, this); }
  4822. catch (IException *e)
  4823. {
  4824. if (errMark)
  4825. rewind((unsigned)(errMark-startMark));
  4826. StringBuffer errMsg;
  4827. e->errorMessage(errMsg);
  4828. e->Release();
  4829. error(errMsg.str());
  4830. }
  4831. return NULL; // will never get here.
  4832. }
  4833. // IEntityHelper impl.
  4834. virtual bool find(const char *entity, StringBuffer &value) override
  4835. {
  4836. return lookupRefValue(entity, value);
  4837. }
  4838. };
  4839. template <class X>
  4840. class CXMLReader : public CXMLReaderBase<X>, implements IPTreeReader
  4841. {
  4842. bool rootTerminated;
  4843. StringBuffer attrName, attrval;
  4844. StringBuffer tmpStr;
  4845. void init()
  4846. {
  4847. attrName.append('@');
  4848. }
  4849. void resetState()
  4850. {
  4851. rootTerminated = false;
  4852. }
  4853. public:
  4854. typedef CXMLReaderBase<X> PARENT;
  4855. using PARENT::nextChar;
  4856. using PARENT::readNext;
  4857. using PARENT::expecting;
  4858. using PARENT::match;
  4859. using PARENT::error;
  4860. using PARENT::skipWS;
  4861. using PARENT::checkBOM;
  4862. using PARENT::checkReadNext;
  4863. using PARENT::checkSkipWS;
  4864. using PARENT::eos;
  4865. using PARENT::curOffset;
  4866. using PARENT::noRoot;
  4867. using PARENT::ignoreWhiteSpace;
  4868. using PARENT::iEvent;
  4869. using PARENT::parseDirective;
  4870. using PARENT::parseOther;
  4871. using PARENT::parsePI;
  4872. using PARENT::parsePIOrDecl;
  4873. using PARENT::parseComment;
  4874. using PARENT::_decodeXML;
  4875. using PARENT::ignoreNameSpaces;
  4876. using PARENT::hadXMLDecl;
  4877. IMPLEMENT_IINTERFACE;
  4878. CXMLReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions, size32_t bufSize=0)
  4879. : PARENT(stream, iEvent, xmlReaderOptions, bufSize)
  4880. {
  4881. init();
  4882. resetState();
  4883. }
  4884. CXMLReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions)
  4885. : PARENT(buf, bufLength, iEvent, xmlReaderOptions)
  4886. {
  4887. init();
  4888. resetState();
  4889. }
  4890. CXMLReader(const void *buf, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions)
  4891. : PARENT(buf, iEvent, xmlReaderOptions)
  4892. {
  4893. init();
  4894. resetState();
  4895. }
  4896. virtual void reset() override
  4897. {
  4898. resetState();
  4899. PARENT::reset();
  4900. }
  4901. // IPTreeReader
  4902. virtual void load() override { loadXML(); }
  4903. virtual offset_t queryOffset() override { return curOffset; }
  4904. void loadXML()
  4905. {
  4906. bool head=true;
  4907. restart:
  4908. if (!checkReadNext()) return;
  4909. if (head)
  4910. {
  4911. head = false;
  4912. if (checkBOM())
  4913. if (!checkReadNext()) return;
  4914. }
  4915. if (!checkSkipWS()) return;
  4916. if ('<' != nextChar)
  4917. expecting("<");
  4918. readNext();
  4919. if ('!' == nextChar)
  4920. {
  4921. readNext();
  4922. parseOther();
  4923. goto restart;
  4924. }
  4925. else if ('?' == nextChar)
  4926. {
  4927. parsePIOrDecl();
  4928. goto restart;
  4929. }
  4930. if (!noRoot && rootTerminated)
  4931. {
  4932. if (ignoreWhiteSpace)
  4933. if (!checkSkipWS()) return;
  4934. error("Trailing xml after close of root tag");
  4935. }
  4936. _loadXML();
  4937. if (noRoot)
  4938. {
  4939. head = true;
  4940. hadXMLDecl = false;
  4941. }
  4942. else
  4943. rootTerminated = true;
  4944. goto restart;
  4945. }
  4946. void _loadXML()
  4947. {
  4948. restart:
  4949. offset_t startOffset = curOffset-2;
  4950. if ('!' == nextChar) // not sure this branch can ever be hit.
  4951. {
  4952. parseComment();
  4953. readNext();
  4954. if ('<' != nextChar)
  4955. expecting("<");
  4956. goto restart;
  4957. }
  4958. StringBuffer tagName;
  4959. if (ignoreWhiteSpace)
  4960. skipWS();
  4961. while (!isspace(nextChar) && nextChar != '>' && nextChar != '/')
  4962. {
  4963. tagName.append(nextChar);
  4964. readNext();
  4965. if ('<' == nextChar)
  4966. error("Unmatched close tag encountered");
  4967. }
  4968. StringBuffer completeTagname(tagName);
  4969. if (ignoreNameSpaces)
  4970. {
  4971. const char *colon;
  4972. if ((colon = strchr(tagName.str(), ':')) != NULL)
  4973. tagName.remove(0, (size32_t)(colon - tagName.str() + 1));
  4974. }
  4975. iEvent->beginNode(tagName.str(), false, startOffset);
  4976. skipWS();
  4977. bool endTag = false;
  4978. bool base64 = false;
  4979. while(nextChar != '>')
  4980. {
  4981. skipWS();
  4982. if (nextChar=='/')
  4983. {
  4984. readNext();
  4985. if (nextChar != '>')
  4986. expecting(">");
  4987. endTag = true;
  4988. break;
  4989. }
  4990. attrName.setLength(1);
  4991. attrval.clear();
  4992. while (nextChar && !isspace(nextChar) && nextChar != '=' && nextChar != '>' && nextChar != '/')
  4993. {
  4994. attrName.append(nextChar);
  4995. readNext();
  4996. }
  4997. skipWS();
  4998. if (nextChar == '=') readNext(); else expecting("=");
  4999. skipWS();
  5000. if (nextChar == '"')
  5001. {
  5002. readNext();
  5003. while (nextChar != '"')
  5004. {
  5005. if (!nextChar)
  5006. eos();
  5007. attrval.append(nextChar);
  5008. readNext();
  5009. }
  5010. }
  5011. else if (nextChar == '\'')
  5012. {
  5013. readNext();
  5014. while (nextChar != '\'')
  5015. {
  5016. attrval.append(nextChar);
  5017. readNext();
  5018. }
  5019. }
  5020. else
  5021. error();
  5022. _decodeXML(0, attrval.str(), tmpStr.clear());
  5023. if (0 == strcmp(attrName.str(), "@xsi:type") &&
  5024. (0 == stricmp(tmpStr.str(),"SOAP-ENC:base64")))
  5025. base64 = true;
  5026. else
  5027. iEvent->newAttribute(attrName.str(), tmpStr.str());
  5028. readNext();
  5029. skipWS();
  5030. }
  5031. iEvent->beginNodeContent(tagName.str());
  5032. StringBuffer tagText;
  5033. bool binary = base64;
  5034. if (!endTag)
  5035. {
  5036. if (nextChar == '>')
  5037. {
  5038. for (;;)
  5039. {
  5040. for (;;)
  5041. {
  5042. readNext();
  5043. if (ignoreWhiteSpace)
  5044. skipWS();
  5045. if ('\0' == nextChar)
  5046. eos();
  5047. StringBuffer mark;
  5048. while (nextChar && nextChar !='<') { mark.append(nextChar); readNext(); }
  5049. size32_t l = mark.length();
  5050. size32_t r = l+1;
  5051. if (l)
  5052. {
  5053. if (ignoreWhiteSpace)
  5054. {
  5055. while (l-- && isspace(mark.charAt(l)));
  5056. mark.setLength(l+1);
  5057. }
  5058. tagText.ensureCapacity(mark.length());
  5059. _decodeXML(r, mark.str(), tagText);
  5060. }
  5061. readNext();
  5062. if ('!' == nextChar)
  5063. parseDirective(tagText);
  5064. else if ('?' == nextChar)
  5065. {
  5066. parsePI(tmpStr.clear());
  5067. #ifdef STRICT_PI
  5068. if (0 == stricmp(tmpStr.str(), "xml"))
  5069. error("Reserved PI target used");
  5070. #endif
  5071. }
  5072. else
  5073. break;
  5074. }
  5075. if (nextChar=='/')
  5076. {
  5077. if (base64)
  5078. {
  5079. JBASE64_Decode(tagText.str(), tmpStr.clear());
  5080. tagText.swapWith(tmpStr);
  5081. }
  5082. else
  5083. {
  5084. if (strlen(tagText.str()) != tagText.length())
  5085. binary = true;
  5086. }
  5087. break; // exit
  5088. }
  5089. else
  5090. _loadXML();
  5091. }
  5092. readNext();
  5093. unsigned i = 0;
  5094. while (!isspace(nextChar) && nextChar != '>')
  5095. {
  5096. if ((i >= completeTagname.length()) ||
  5097. (nextChar != completeTagname.charAt(i++)))
  5098. error("Mismatched opening and closing tags");
  5099. readNext();
  5100. }
  5101. if (i != completeTagname.length())
  5102. error("Mismatched opening and closing tags");
  5103. skipWS();
  5104. if (nextChar != '>')
  5105. expecting(">");
  5106. }
  5107. }
  5108. iEvent->endNode(tagName.str(), tagText.length(), tagText.str(), binary, curOffset);
  5109. }
  5110. };
  5111. template <class X>
  5112. class CPullXMLReader : public CXMLReaderBase<X>, implements IPullPTreeReader
  5113. {
  5114. typedef CXMLReaderBase<X> PARENT;
  5115. using PARENT::nextChar;
  5116. using PARENT::readNext;
  5117. using PARENT::expecting;
  5118. using PARENT::match;
  5119. using PARENT::error;
  5120. using PARENT::skipWS;
  5121. using PARENT::checkBOM;
  5122. using PARENT::checkReadNext;
  5123. using PARENT::checkSkipWS;
  5124. using PARENT::eos;
  5125. using PARENT::curOffset;
  5126. using PARENT::noRoot;
  5127. using PARENT::ignoreWhiteSpace;
  5128. using PARENT::iEvent;
  5129. using PARENT::parseDirective;
  5130. using PARENT::parseOther;
  5131. using PARENT::parsePI;
  5132. using PARENT::parsePIOrDecl;
  5133. using PARENT::parseComment;
  5134. using PARENT::_decodeXML;
  5135. using PARENT::ignoreNameSpaces;
  5136. using PARENT::hadXMLDecl;
  5137. class CStateInfo : public CInterface
  5138. {
  5139. public:
  5140. CStateInfo()
  5141. {
  5142. tag.ensureCapacity(15);
  5143. binary = base64 = false;
  5144. }
  5145. inline void reset()
  5146. {
  5147. binary = base64 = false;
  5148. tag.clear();
  5149. tagText.clear();
  5150. }
  5151. const char *wnsTag;
  5152. StringBuffer tag;
  5153. StringBuffer tagText;
  5154. bool binary, base64;
  5155. };
  5156. CICopyArrayOf<CStateInfo> stack, freeStateInfo;
  5157. CStateInfo *stateInfo;
  5158. enum ParseStates { headerStart, tagStart, tagAttributes, tagContent, tagContent2, tagClose, tagEnd, tagMarker } state;
  5159. bool endOfRoot;
  5160. StringBuffer attrName, attrval, mark, tmpStr;
  5161. void resetState()
  5162. {
  5163. stack.kill();
  5164. state = headerStart;
  5165. stateInfo = NULL;
  5166. endOfRoot = false;
  5167. attrName.append('@');
  5168. }
  5169. public:
  5170. IMPLEMENT_IINTERFACE;
  5171. CPullXMLReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions, size32_t bufSize=0)
  5172. : CXMLReaderBase<X>(stream, iEvent, xmlReaderOptions, bufSize)
  5173. {
  5174. resetState();
  5175. }
  5176. CPullXMLReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions)
  5177. : CXMLReaderBase<X>(buf, bufLength, iEvent, xmlReaderOptions)
  5178. {
  5179. resetState();
  5180. }
  5181. CPullXMLReader(const void *buf, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions)
  5182. : CXMLReaderBase<X>(buf, iEvent, xmlReaderOptions)
  5183. {
  5184. resetState();
  5185. }
  5186. ~CPullXMLReader()
  5187. {
  5188. ForEachItemIn(i, stack)
  5189. delete &stack.item(i);
  5190. ForEachItemIn(i2, freeStateInfo)
  5191. delete &freeStateInfo.item(i2);
  5192. }
  5193. // IPullPTreeReader
  5194. virtual void load() override
  5195. {
  5196. while (next()) {}
  5197. }
  5198. virtual void reset() override
  5199. {
  5200. PARENT::reset();
  5201. resetState();
  5202. }
  5203. virtual offset_t queryOffset() override { return curOffset; }
  5204. virtual bool next() override
  5205. {
  5206. switch (state)
  5207. {
  5208. case headerStart:
  5209. {
  5210. if (!checkReadNext()) return false;
  5211. if (checkBOM())
  5212. if (!checkReadNext()) return false;
  5213. for (;;)
  5214. {
  5215. if (!checkSkipWS()) return false;
  5216. if ('<' != nextChar)
  5217. expecting("<");
  5218. readNext();
  5219. if ('!' == nextChar)
  5220. {
  5221. readNext();
  5222. parseOther();
  5223. }
  5224. else if ('?' == nextChar)
  5225. parsePIOrDecl();
  5226. else
  5227. break;
  5228. if (!checkReadNext()) return false;
  5229. }
  5230. state = tagStart;
  5231. break;
  5232. }
  5233. case tagStart:
  5234. {
  5235. offset_t startOffset;
  5236. for (;;)
  5237. {
  5238. if ('!' != nextChar)
  5239. break;
  5240. parseComment();
  5241. readNext();
  5242. if ('<' != nextChar)
  5243. expecting("<");
  5244. }
  5245. startOffset = curOffset-2;
  5246. if (freeStateInfo.ordinality())
  5247. {
  5248. stateInfo = &freeStateInfo.popGet();
  5249. stateInfo->reset();
  5250. }
  5251. else
  5252. stateInfo = new CStateInfo;
  5253. stack.append(*stateInfo);
  5254. if ('/' == nextChar)
  5255. error("Unmatched close tag encountered");
  5256. while (!isspace(nextChar) && nextChar != '>')
  5257. {
  5258. stateInfo->tag.append(nextChar);
  5259. readNext();
  5260. if ('/' == nextChar) break;
  5261. if ('<' == nextChar)
  5262. error("Unmatched close tag encountered");
  5263. }
  5264. stateInfo->wnsTag = stateInfo->tag.str();
  5265. if (ignoreNameSpaces)
  5266. {
  5267. const char *colon;
  5268. if ((colon = strchr(stateInfo->wnsTag, ':')) != NULL)
  5269. stateInfo->wnsTag = colon+1;
  5270. }
  5271. endOfRoot = false;
  5272. try
  5273. {
  5274. iEvent->beginNode(stateInfo->wnsTag, false, startOffset);
  5275. }
  5276. catch (IPTreeException *pe)
  5277. {
  5278. if (PTreeExcpt_InvalidTagName == pe->errorCode())
  5279. {
  5280. pe->Release();
  5281. StringBuffer msg("Expecting valid start tag, but got \"");
  5282. error(msg.append(stateInfo->wnsTag).append("\"").str());
  5283. }
  5284. throw;
  5285. }
  5286. state = tagAttributes;
  5287. break;
  5288. }
  5289. case tagAttributes:
  5290. {
  5291. skipWS();
  5292. if (nextChar == '>')
  5293. state = tagContent;
  5294. else
  5295. {
  5296. skipWS();
  5297. if (nextChar=='/')
  5298. {
  5299. readNext();
  5300. if (nextChar != '>')
  5301. expecting(">");
  5302. // no actual content
  5303. iEvent->beginNodeContent(stateInfo->wnsTag);
  5304. state = tagEnd;
  5305. break;
  5306. }
  5307. attrName.setLength(1);
  5308. attrval.clear();
  5309. while (nextChar && !isspace(nextChar) && nextChar != '=' && nextChar != '>' && nextChar != '/')
  5310. {
  5311. attrName.append(nextChar);
  5312. readNext();
  5313. }
  5314. skipWS();
  5315. if (nextChar == '=') readNext(); else expecting("=");
  5316. skipWS();
  5317. if (nextChar == '"')
  5318. {
  5319. readNext();
  5320. while (nextChar != '"')
  5321. {
  5322. if (!nextChar)
  5323. eos();
  5324. attrval.append(nextChar);
  5325. readNext();
  5326. }
  5327. }
  5328. else if (nextChar == '\'')
  5329. {
  5330. readNext();
  5331. while (nextChar != '\'')
  5332. {
  5333. attrval.append(nextChar);
  5334. readNext();
  5335. }
  5336. }
  5337. else
  5338. error();
  5339. _decodeXML(0, attrval.str(), tmpStr.clear());
  5340. if (0 == strcmp(attrName.str(), "@xsi:type") &&
  5341. (0 == stricmp(tmpStr.str(),"SOAP-ENC:base64")))
  5342. stateInfo->base64 = true;
  5343. else
  5344. iEvent->newAttribute(attrName.str(), tmpStr.str());
  5345. readNext();
  5346. skipWS();
  5347. }
  5348. break;
  5349. }
  5350. case tagContent:
  5351. {
  5352. iEvent->beginNodeContent(stateInfo->wnsTag);
  5353. if ('>' != nextChar)
  5354. state = tagEnd;
  5355. else
  5356. state = tagContent2;
  5357. break;
  5358. }
  5359. case tagContent2:
  5360. {
  5361. try
  5362. {
  5363. for (;;)
  5364. {
  5365. if (endOfRoot)
  5366. {
  5367. if (!checkReadNext()) return false;
  5368. if (!checkSkipWS()) return false;
  5369. }
  5370. else
  5371. {
  5372. readNext();
  5373. if (ignoreWhiteSpace)
  5374. skipWS();
  5375. }
  5376. if ('\0' == nextChar)
  5377. eos();
  5378. mark.clear();
  5379. state = tagMarker;
  5380. while (nextChar && nextChar !='<') { mark.append(nextChar); readNext(); }
  5381. if (!nextChar)
  5382. break;
  5383. size32_t l = mark.length();
  5384. size32_t r = l+1;
  5385. if (l && stateInfo)
  5386. {
  5387. if (ignoreWhiteSpace)
  5388. {
  5389. const char *tb = mark.str();
  5390. const char *t = tb+l-1;
  5391. if (isspace(*t))
  5392. {
  5393. while (t != tb && isspace(*(--t)));
  5394. mark.setLength((size32_t)(t-tb+1));
  5395. }
  5396. }
  5397. stateInfo->tagText.ensureCapacity(mark.length());
  5398. _decodeXML(r, mark.str(), stateInfo->tagText);
  5399. }
  5400. if (endOfRoot && mark.length())
  5401. {
  5402. const char *m = mark.str();
  5403. const char *e = m+mark.length();
  5404. do { if (!isspace(*m++)) error("Trailing content after close of root tag"); }
  5405. while (m!=e);
  5406. }
  5407. readNext();
  5408. if ('!' == nextChar)
  5409. {
  5410. parseDirective(stateInfo->tagText);
  5411. state = tagContent2;
  5412. }
  5413. else if ('?' == nextChar)
  5414. {
  5415. parsePI(tmpStr.clear());
  5416. #ifdef STRICT_PI
  5417. if (0 == stricmp(tmpStr.str(), "xml"))
  5418. error("Reserved PI target used");
  5419. #endif
  5420. state = tagContent2;
  5421. }
  5422. else
  5423. break;
  5424. }
  5425. }
  5426. catch (IPTreeReadException *e)
  5427. {
  5428. if (endOfRoot && PTreeRead_EOS == e->errorCode() && (state != tagContent2 && mark.length())) // only to provide more meaningful error
  5429. {
  5430. const char *m = mark.str();
  5431. const char *es = m+mark.length();
  5432. do
  5433. {
  5434. if (!isspace(*m++))
  5435. {
  5436. e->Release();
  5437. error("Trailing content after close of root tag");
  5438. }
  5439. }
  5440. while (m!=es);
  5441. }
  5442. throw;
  5443. }
  5444. if ('/' == nextChar)
  5445. {
  5446. if (endOfRoot && !noRoot)
  5447. error("Trailing tag close after close of root tag");
  5448. if (stateInfo->base64)
  5449. {
  5450. JBASE64_Decode(stateInfo->tagText.str(), tmpStr.clear());
  5451. stateInfo->tagText.swapWith(tmpStr);
  5452. stateInfo->binary = true;
  5453. // next state tagContent2 still
  5454. }
  5455. else
  5456. {
  5457. if (strlen(stateInfo->tagText.str()) != stateInfo->tagText.length())
  5458. stateInfo->binary = true;
  5459. }
  5460. state = tagClose;
  5461. break; // exit
  5462. }
  5463. else
  5464. {
  5465. if (endOfRoot && !noRoot)
  5466. error("Trailing tag open after close of root tag");
  5467. state = tagStart;
  5468. }
  5469. break;
  5470. }
  5471. case tagClose:
  5472. {
  5473. readNext();
  5474. const char *t = stateInfo->tag.str();
  5475. const char *te = t+stateInfo->tag.length();
  5476. for (;;)
  5477. {
  5478. if (nextChar == '>' || isspace(nextChar))
  5479. {
  5480. if (t != te)
  5481. error("Mismatched opening and closing tags");
  5482. break;
  5483. }
  5484. else if (nextChar != *t++)
  5485. error("Mismatched opening and closing tags");
  5486. readNext();
  5487. }
  5488. skipWS();
  5489. if (nextChar != '>')
  5490. expecting(">");
  5491. state = tagEnd;
  5492. break;
  5493. }
  5494. case tagEnd:
  5495. {
  5496. iEvent->endNode(stateInfo->wnsTag, stateInfo->tagText.length(), stateInfo->tagText.str(), stateInfo->binary, curOffset);
  5497. freeStateInfo.append(*stateInfo);
  5498. stack.pop();
  5499. endOfRoot = 0==stack.ordinality();
  5500. stateInfo = stack.ordinality()?&stack.tos():NULL;
  5501. if (endOfRoot && noRoot)
  5502. {
  5503. state = headerStart;
  5504. hadXMLDecl = false;
  5505. endOfRoot = false;
  5506. }
  5507. else
  5508. state = tagContent2;
  5509. break;
  5510. }
  5511. }
  5512. return true;
  5513. }
  5514. };
  5515. IPTreeReader *createXMLStreamReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions, size32_t bufSize)
  5516. {
  5517. class CXMLStreamReader : public CXMLReader<CInstStreamReader>
  5518. {
  5519. public:
  5520. CXMLStreamReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions, size32_t bufSize=0) : CXMLReader<CInstStreamReader>(stream, iEvent, xmlReaderOptions, bufSize) { }
  5521. };
  5522. return new CXMLStreamReader(stream, iEvent, xmlReaderOptions, bufSize);
  5523. }
  5524. IPTreeReader *createXMLStringReader(const char *xml, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions)
  5525. {
  5526. class CXMLStringReader : public CXMLReader<CInstStringReader>
  5527. {
  5528. public:
  5529. CXMLStringReader(const void *xml, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions) : CXMLReader<CInstStringReader>(xml, iEvent, xmlReaderOptions) { }
  5530. };
  5531. if (NULL == xml)
  5532. throw createPTreeReadException(PTreeRead_syntax, "Null string passed to createXMLStringReader", NULL, 0, 0);
  5533. return new CXMLStringReader(xml, iEvent, xmlReaderOptions);
  5534. }
  5535. IPTreeReader *createXMLBufferReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions)
  5536. {
  5537. class CXMLBufferReader : public CXMLReader<CInstBufferReader>
  5538. {
  5539. public:
  5540. CXMLBufferReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions) : CXMLReader<CInstBufferReader>(buf, bufLength, iEvent, xmlReaderOptions) { }
  5541. };
  5542. return new CXMLBufferReader(buf, bufLength, iEvent, xmlReaderOptions);
  5543. }
  5544. IPullPTreeReader *createPullXMLStreamReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions, size32_t bufSize)
  5545. {
  5546. class CXMLStreamReader : public CPullXMLReader<CInstStreamReader>
  5547. {
  5548. public:
  5549. CXMLStreamReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions, size32_t bufSize=0) : CPullXMLReader<CInstStreamReader>(stream, iEvent, xmlReaderOptions, bufSize) { }
  5550. };
  5551. return new CXMLStreamReader(stream, iEvent, xmlReaderOptions, bufSize);
  5552. }
  5553. IPullPTreeReader *createPullXMLStringReader(const char *xml, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions)
  5554. {
  5555. class CXMLStringReader : public CPullXMLReader<CInstStringReader>
  5556. {
  5557. public:
  5558. CXMLStringReader(const void *xml, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions) : CPullXMLReader<CInstStringReader>(xml, iEvent, xmlReaderOptions) { }
  5559. };
  5560. return new CXMLStringReader(xml, iEvent, xmlReaderOptions);
  5561. }
  5562. IPullPTreeReader *createPullXMLBufferReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions)
  5563. {
  5564. class CXMLBufferReader : public CPullXMLReader<CInstBufferReader>
  5565. {
  5566. public:
  5567. CXMLBufferReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions) : CPullXMLReader<CInstBufferReader>(buf, bufLength, iEvent, xmlReaderOptions) { }
  5568. };
  5569. return new CXMLBufferReader(buf, bufLength, iEvent, xmlReaderOptions);
  5570. }
  5571. IPTreeMaker *createPTreeMaker(byte flags, IPropertyTree *root, IPTreeNodeCreator *nodeCreator)
  5572. {
  5573. return new CPTreeMaker(flags, nodeCreator, root);
  5574. }
  5575. IPTreeMaker *createRootLessPTreeMaker(byte flags, IPropertyTree *root, IPTreeNodeCreator *nodeCreator)
  5576. {
  5577. return new CPTreeMaker(flags, nodeCreator, root, true);
  5578. }
  5579. ////////////////////////////
  5580. ///////////////////////////
  5581. static IPTreeMaker *createDefaultPTreeMaker(byte flags, PTreeReaderOptions readFlags)
  5582. {
  5583. bool noRoot = 0 != ((unsigned)readFlags & (unsigned)ptr_noRoot);
  5584. return new CPTreeMaker(flags, NULL, NULL, noRoot);
  5585. }
  5586. IPropertyTree *createPTree(ISimpleReadStream &stream, byte flags, PTreeReaderOptions readFlags, IPTreeMaker *iMaker)
  5587. {
  5588. Owned<IPTreeMaker> _iMaker;
  5589. if (!iMaker)
  5590. {
  5591. iMaker = createDefaultPTreeMaker(flags, readFlags);
  5592. _iMaker.setown(iMaker);
  5593. }
  5594. Owned<IPTreeReader> reader = createXMLStreamReader(stream, *iMaker, readFlags);
  5595. reader->load();
  5596. if (iMaker->queryRoot())
  5597. return LINK(iMaker->queryRoot());
  5598. else
  5599. return iMaker->create(NULL);
  5600. }
  5601. IPropertyTree *createPTree(IFileIO &ifileio, byte flags, PTreeReaderOptions readFlags, IPTreeMaker *iMaker)
  5602. {
  5603. OwnedIFileIOStream stream = createIOStream(&ifileio);
  5604. return createPTree(*stream, flags, readFlags, iMaker);
  5605. }
  5606. IPropertyTree *createPTree(IFile &ifile, byte flags, PTreeReaderOptions readFlags, IPTreeMaker *iMaker)
  5607. {
  5608. OwnedIFileIO ifileio = ifile.open(IFOread);
  5609. if (!ifileio)
  5610. throw MakeStringException(0, "Could not locate filename: %s", ifile.queryFilename());
  5611. return createPTree(*ifileio, flags, readFlags, iMaker);
  5612. }
  5613. IPropertyTree *createPTreeFromXMLFile(const char *filename, byte flags, PTreeReaderOptions readFlags, IPTreeMaker *iMaker)
  5614. {
  5615. OwnedIFile ifile = createIFile(filename);
  5616. return createPTree(*ifile, flags, readFlags, iMaker);
  5617. }
  5618. IPropertyTree *createPTreeFromXMLString(const char *xml, byte flags, PTreeReaderOptions readFlags, IPTreeMaker *iMaker)
  5619. {
  5620. Owned<IPTreeMaker> _iMaker;
  5621. if (!iMaker)
  5622. {
  5623. iMaker = createDefaultPTreeMaker(flags, readFlags);
  5624. _iMaker.setown(iMaker);
  5625. }
  5626. Owned<IPTreeReader> reader = createXMLStringReader(xml, *iMaker, readFlags);
  5627. reader->load();
  5628. return LINK(iMaker->queryRoot());
  5629. }
  5630. IPropertyTree *createPTreeFromXMLString(unsigned len, const char *xml, byte flags, PTreeReaderOptions readFlags, IPTreeMaker *iMaker)
  5631. {
  5632. Owned<IPTreeMaker> _iMaker;
  5633. if (!iMaker)
  5634. {
  5635. iMaker = createDefaultPTreeMaker(flags, readFlags);
  5636. _iMaker.setown(iMaker);
  5637. }
  5638. Owned<IPTreeReader> reader = createXMLBufferReader(xml, len, *iMaker, readFlags);
  5639. reader->load();
  5640. return LINK(iMaker->queryRoot());
  5641. }
  5642. //////////////////////////
  5643. /////////////////////////
  5644. inline bool isHiddenWhenSanitized(const char *val)
  5645. {
  5646. if (!val || !*val)
  5647. return false;
  5648. return !(streq(val, "0") || streq(val, "1") || strieq(val, "true") || strieq(val, "false") || strieq(val, "yes") || strieq(val, "no"));
  5649. }
  5650. inline bool isSanitizedAndHidden(const char *val, byte flags, bool attribute)
  5651. {
  5652. bool sanitize = (attribute) ? ((flags & YAML_SanitizeAttributeValues)!=0) : ((flags & YAML_Sanitize)!=0);
  5653. if (sanitize)
  5654. return isHiddenWhenSanitized(val);
  5655. return false;
  5656. }
  5657. static void _toXML(const IPropertyTree *tree, IIOStream &out, unsigned indent, unsigned flags)
  5658. {
  5659. const char *name = tree->queryName();
  5660. if (!name) name = "__unnamed__";
  5661. bool isBinary = tree->isBinary(NULL);
  5662. bool inlinebody = true;
  5663. if (flags & XML_Embed) writeCharsNToStream(out, ' ', indent);
  5664. writeCharToStream(out, '<');
  5665. writeStringToStream(out, name);
  5666. Owned<IAttributeIterator> it = tree->getAttributes(true);
  5667. if (it->first())
  5668. {
  5669. unsigned attributeindent = indent+2+(size32_t)strlen(name);
  5670. bool first = true;
  5671. do
  5672. {
  5673. const char *key = it->queryName();
  5674. if (!isBinary || stricmp(key, "@xsi:type")!=0)
  5675. {
  5676. if (first)
  5677. {
  5678. if (flags & XML_LineBreak) inlinebody = false;
  5679. first = false;
  5680. writeCharToStream(out, ' ');
  5681. }
  5682. else if ((flags & XML_LineBreakAttributes) && it->count() > 3)
  5683. {
  5684. writeStringToStream(out, "\n");
  5685. writeCharsNToStream(out, ' ', attributeindent);
  5686. }
  5687. else
  5688. writeCharToStream(out, ' ');
  5689. writeStringToStream(out, key+1);
  5690. if (flags & XML_SingleQuoteAttributeValues)
  5691. writeStringToStream(out, "='");
  5692. else
  5693. writeStringToStream(out, "=\"");
  5694. const char *val = it->queryValue();
  5695. if (val)
  5696. {
  5697. if (isSanitizedAndHidden(val, flags, true))
  5698. writeCharsNToStream(out, '*', strlen(val));
  5699. else
  5700. encodeXML(val, out, ENCODE_NEWLINES, (unsigned)-1, true);
  5701. }
  5702. if (flags & XML_SingleQuoteAttributeValues)
  5703. writeCharToStream(out, '\'');
  5704. else
  5705. writeCharToStream(out, '"');
  5706. }
  5707. }
  5708. while (it->next());
  5709. }
  5710. Owned<IPropertyTreeIterator> sub = tree->getElements("*", 0 != (flags & XML_SortTags) ? iptiter_sort : iptiter_null);
  5711. MemoryBuffer thislevelbin;
  5712. StringBuffer _thislevel;
  5713. const char *thislevel = NULL; // to avoid uninitialized warning
  5714. bool empty;
  5715. if (isBinary)
  5716. {
  5717. if (flags & XML_LineBreak) inlinebody = false;
  5718. writeStringToStream(out, " xsi:type=\"SOAP-ENC:base64\"");
  5719. empty = (!tree->getPropBin(NULL, thislevelbin))||(thislevelbin.length()==0);
  5720. }
  5721. else
  5722. {
  5723. if (tree->isCompressed(NULL))
  5724. {
  5725. empty = false; // can't be empty if compressed;
  5726. verifyex(tree->getProp(NULL, _thislevel));
  5727. thislevel = _thislevel.str();
  5728. }
  5729. else
  5730. empty = (NULL == (thislevel = tree->queryProp(NULL)));
  5731. }
  5732. if (sub->first())
  5733. {
  5734. if (flags & XML_LineBreak) inlinebody = false;
  5735. }
  5736. else if (empty && !(flags & XML_Sanitize))
  5737. {
  5738. if (flags & XML_LineBreak)
  5739. writeStringToStream(out, "/>\n");
  5740. else
  5741. writeStringToStream(out, "/>");
  5742. return;
  5743. }
  5744. writeCharToStream(out, '>');
  5745. if (!inlinebody)
  5746. writeStringToStream(out, "\n");
  5747. for(; sub->isValid(); sub->next())
  5748. _toXML(&sub->query(), out, indent+1, flags);
  5749. if (!empty)
  5750. {
  5751. if (!inlinebody)
  5752. writeCharsNToStream(out, ' ', indent+1);
  5753. if (flags & XML_Sanitize)
  5754. {
  5755. // NOTE - we don't output anything for binary.... is that ok?
  5756. if (thislevel)
  5757. {
  5758. if (isHiddenWhenSanitized(thislevel))
  5759. writeCharsNToStream(out, '*', strlen(thislevel));
  5760. else
  5761. writeStringToStream(out, thislevel);
  5762. }
  5763. }
  5764. else if (isBinary)
  5765. {
  5766. if (flags & XML_NoBinaryEncode64)
  5767. {
  5768. if (flags & XML_NoEncode)
  5769. {
  5770. out.write(thislevelbin.length(), thislevelbin.toByteArray());
  5771. }
  5772. else
  5773. {
  5774. const char * buff = static_cast<const char *>(thislevelbin.toByteArray());
  5775. const unsigned len = thislevelbin.length();
  5776. unsigned prefix = 0;
  5777. while ((prefix < len) && isspace(buff[prefix]))
  5778. prefix++;
  5779. encodeXML(buff, out, ENCODE_WHITESPACE, prefix, true);
  5780. if (prefix != len) { // check not all spaces
  5781. unsigned suffix = len;
  5782. while (isspace(buff[suffix-1]))
  5783. suffix--;
  5784. encodeXML(buff+prefix, out, 0, suffix-prefix, true);
  5785. encodeXML(buff+suffix, out, ENCODE_WHITESPACE, len-suffix, true);
  5786. }
  5787. }
  5788. }
  5789. else
  5790. JBASE64_Encode(thislevelbin.toByteArray(), thislevelbin.length(), out, true);
  5791. }
  5792. else
  5793. {
  5794. if (flags & XML_NoEncode)
  5795. {
  5796. writeStringToStream(out, thislevel);
  5797. }
  5798. else
  5799. {
  5800. const char *m = thislevel;
  5801. const char *p = m;
  5802. while (isspace(*p))
  5803. p++;
  5804. encodeXML(m, out, ENCODE_WHITESPACE, p-m, true);
  5805. if (*p) { // check not all spaces
  5806. const char *s = p+strlen(p);
  5807. while (isspace(*(s-1)))
  5808. s--;
  5809. assertex(s>p);
  5810. encodeXML(p, out, 0, s-p, true);
  5811. encodeXML(s, out, ENCODE_WHITESPACE, (unsigned)-1, true);
  5812. }
  5813. }
  5814. if (!inlinebody)
  5815. writeStringToStream(out, "\n");
  5816. }
  5817. }
  5818. if (!inlinebody)
  5819. writeCharsNToStream(out, ' ', indent);
  5820. writeStringToStream(out, "</");
  5821. writeStringToStream(out, name);
  5822. if (flags & XML_LineBreak)
  5823. writeStringToStream(out, ">\n");
  5824. else
  5825. writeCharToStream(out, '>');
  5826. }
  5827. class CStringBufferMarkupIOAdapter : public CInterfaceOf<IIOStream>
  5828. {
  5829. StringBuffer &out;
  5830. public:
  5831. CStringBufferMarkupIOAdapter(StringBuffer &_out) : out(_out) { }
  5832. virtual void flush() override { }
  5833. virtual size32_t read(size32_t len, void * data) override { UNIMPLEMENTED; return 0; }
  5834. virtual size32_t write(size32_t len, const void * data) override { out.append(len, (const char *)data); return len; }
  5835. };
  5836. jlib_decl StringBuffer &toXML(const IPropertyTree *tree, StringBuffer &ret, unsigned indent, unsigned flags)
  5837. {
  5838. CStringBufferMarkupIOAdapter adapter(ret);
  5839. _toXML(tree->queryBranch(NULL), adapter, indent, flags);
  5840. return ret;
  5841. }
  5842. void toXML(const IPropertyTree *tree, IIOStream &out, unsigned indent, unsigned flags)
  5843. {
  5844. _toXML(tree, out, indent, flags);
  5845. }
  5846. void printXML(const IPropertyTree *tree, unsigned indent, unsigned flags)
  5847. {
  5848. StringBuffer xml;
  5849. toXML(tree, xml, indent, flags);
  5850. printf("%s", xml.str());
  5851. }
  5852. void dbglogXML(const IPropertyTree *tree, unsigned indent, unsigned flags)
  5853. {
  5854. StringBuffer xml;
  5855. toXML(tree, xml, indent, flags);
  5856. DBGLOG("%s", xml.str());
  5857. }
  5858. void saveXML(const char *filename, const IPropertyTree *tree, unsigned indent, unsigned flags)
  5859. {
  5860. OwnedIFile ifile = createIFile(filename);
  5861. saveXML(*ifile, tree, indent, flags);
  5862. }
  5863. void saveXML(IFile &ifile, const IPropertyTree *tree, unsigned indent, unsigned flags)
  5864. {
  5865. OwnedIFileIO ifileio = ifile.open(IFOcreate);
  5866. if (!ifileio)
  5867. throw MakeStringException(0, "saveXML: could not find %s to open", ifile.queryFilename());
  5868. saveXML(*ifileio, tree, indent, flags);
  5869. }
  5870. void saveXML(IFileIO &ifileio, const IPropertyTree *tree, unsigned indent, unsigned flags)
  5871. {
  5872. Owned<IIOStream> stream = createIOStream(&ifileio);
  5873. stream.setown(createBufferedIOStream(stream));
  5874. saveXML(*stream, tree, indent, flags);
  5875. }
  5876. void saveXML(IIOStream &stream, const IPropertyTree *tree, unsigned indent, unsigned flags)
  5877. {
  5878. toXML(tree, stream, indent, flags);
  5879. }
  5880. /////////////////////////
  5881. void checkWriteJSONDelimiter(IIOStream &out, bool &delimit)
  5882. {
  5883. if (delimit)
  5884. writeCharToStream(out, ',');
  5885. delimit = false;
  5886. }
  5887. static void writeJSONNameToStream(IIOStream &out, const char *name, unsigned indent, bool &delimit)
  5888. {
  5889. if (!name || !*name)
  5890. return;
  5891. checkWriteJSONDelimiter(out, delimit);
  5892. if (indent)
  5893. {
  5894. writeCharToStream(out, '\n');
  5895. writeCharsNToStream(out, ' ', indent);
  5896. }
  5897. else
  5898. writeCharToStream(out, ' ');
  5899. writeCharToStream(out, '"');
  5900. writeStringToStream(out, name);
  5901. writeStringToStream(out, "\": ");
  5902. delimit = false;
  5903. }
  5904. static void writeJSONValueToStream(IIOStream &out, const char *val, bool &delimit, bool hidden=false)
  5905. {
  5906. checkWriteJSONDelimiter(out, delimit);
  5907. delimit = true;
  5908. if (!val)
  5909. {
  5910. writeStringToStream(out, "null");
  5911. return;
  5912. }
  5913. writeCharToStream(out, '"');
  5914. if (hidden)
  5915. writeCharsNToStream(out, '*', strlen(val));
  5916. else
  5917. {
  5918. StringBuffer s;
  5919. writeStringToStream(out, encodeJSON(s, val));
  5920. }
  5921. writeCharToStream(out, '"');
  5922. }
  5923. static void writeJSONBase64ValueToStream(IIOStream &out, const char *val, size32_t len, bool &delimit, bool hidden)
  5924. {
  5925. checkWriteJSONDelimiter(out, delimit);
  5926. delimit = true;
  5927. if (!val)
  5928. {
  5929. writeStringToStream(out, "null");
  5930. return;
  5931. }
  5932. writeCharToStream(out, '"');
  5933. if (hidden)
  5934. JBASE64_Encode("****", strlen("****"), out, false);
  5935. else
  5936. JBASE64_Encode(val, len, out, false);
  5937. writeCharToStream(out, '"');
  5938. }
  5939. bool isRootArrayObjectHidden(bool root, const char *name, byte flags)
  5940. {
  5941. return ((flags & JSON_HideRootArrayObject) && root && name && streq(name,"__array__"));
  5942. }
  5943. static void _toJSON(const IPropertyTree *tree, IIOStream &out, unsigned indent, byte flags, bool &delimit, bool root=false, bool isArrayItem=false)
  5944. {
  5945. Owned<IAttributeIterator> it = tree->getAttributes(true);
  5946. bool hasAttributes = it->first();
  5947. bool complex = (hasAttributes || tree->hasChildren() || tree->isBinary());
  5948. bool isBinary = tree->isBinary(NULL);
  5949. const char *name = tree->queryName();
  5950. if (!root && !isArrayItem)
  5951. {
  5952. if (!name || !*name)
  5953. name = "__unnamed__";
  5954. writeJSONNameToStream(out, name, (flags & JSON_Format) ? indent : 0, delimit);
  5955. }
  5956. checkWriteJSONDelimiter(out, delimit);
  5957. if (isArrayItem && (flags & JSON_Format))
  5958. {
  5959. writeCharToStream(out, '\n');
  5960. writeCharsNToStream(out, ' ', indent);
  5961. }
  5962. bool hiddenRootArrayObject = isRootArrayObjectHidden(root, name, flags);
  5963. if (!hiddenRootArrayObject)
  5964. {
  5965. if (root || complex)
  5966. {
  5967. writeCharToStream(out, '{');
  5968. delimit = false;
  5969. }
  5970. if (hasAttributes)
  5971. {
  5972. ForEach(*it)
  5973. {
  5974. const char *key = it->queryName();
  5975. if (!isBinary || stricmp(key, "@xsi:type")!=0)
  5976. {
  5977. const char *val = it->queryValue();
  5978. if (val)
  5979. {
  5980. writeJSONNameToStream(out, key, (flags & JSON_Format) ? indent+1 : 0, delimit);
  5981. if (flags & JSON_SanitizeAttributeValues)
  5982. writeJSONValueToStream(out, val, delimit, isHiddenWhenSanitized(val));
  5983. else
  5984. {
  5985. StringBuffer encoded;
  5986. encodeJSON(encoded, val);
  5987. writeJSONValueToStream(out, encoded.str(), delimit);
  5988. }
  5989. }
  5990. }
  5991. }
  5992. }
  5993. }
  5994. MemoryBuffer thislevelbin;
  5995. StringBuffer _thislevel;
  5996. const char *thislevel = NULL; // to avoid uninitialized warning
  5997. bool isNull = true;
  5998. if (!hiddenRootArrayObject)
  5999. {
  6000. if (isBinary)
  6001. {
  6002. isNull = (!tree->getPropBin(NULL, thislevelbin))||(thislevelbin.length()==0);
  6003. }
  6004. else
  6005. {
  6006. if (tree->isCompressed(NULL))
  6007. {
  6008. isNull = false; // can't be empty if compressed;
  6009. verifyex(tree->getProp(NULL, _thislevel));
  6010. thislevel = _thislevel.str();
  6011. }
  6012. else
  6013. isNull = (NULL == (thislevel = tree->queryProp(NULL)));
  6014. }
  6015. if (isNull && !root && !complex)
  6016. {
  6017. writeJSONValueToStream(out, NULL, delimit);
  6018. return;
  6019. }
  6020. }
  6021. Owned<IPropertyTreeIterator> sub = tree->getElements(hiddenRootArrayObject ? "__item__" : "*", 0 != (flags & JSON_SortTags) ? iptiter_sort : iptiter_null);
  6022. //note that detection of repeating elements relies on the fact that ptree elements
  6023. //of the same name will be grouped together
  6024. bool repeatingElement = false;
  6025. sub->first();
  6026. while(sub->isValid())
  6027. {
  6028. Linked<IPropertyTree> element = &sub->query();
  6029. const char *name = element->queryName();
  6030. sub->next();
  6031. if (!repeatingElement)
  6032. {
  6033. if (hiddenRootArrayObject)
  6034. {
  6035. writeCharToStream(out, '[');
  6036. repeatingElement = true;
  6037. delimit = false;
  6038. }
  6039. else if (sub->isValid() && streq(name, sub->query().queryName()))
  6040. {
  6041. if (flags & JSON_Format)
  6042. indent++;
  6043. writeJSONNameToStream(out, name, (flags & JSON_Format) ? indent : 0, delimit);
  6044. writeCharToStream(out, '[');
  6045. repeatingElement = true;
  6046. delimit = false;
  6047. }
  6048. }
  6049. _toJSON(element, out, indent+1, flags, delimit, false, repeatingElement);
  6050. if (repeatingElement && (!sub->isValid() || !streq(name, sub->query().queryName())))
  6051. {
  6052. if (flags & JSON_Format)
  6053. {
  6054. writeCharToStream(out, '\n');
  6055. writeCharsNToStream(out, ' ', indent);
  6056. indent--;
  6057. }
  6058. writeCharToStream(out, ']');
  6059. repeatingElement = false;
  6060. delimit = true;
  6061. }
  6062. }
  6063. if (!hiddenRootArrayObject && !isNull)
  6064. {
  6065. if (complex)
  6066. writeJSONNameToStream(out, isBinary ? "#valuebin" : "#value", (flags & JSON_Format) ? indent+1 : 0, delimit);
  6067. if (isBinary)
  6068. writeJSONBase64ValueToStream(out, thislevelbin.toByteArray(), thislevelbin.length(), delimit, flags & XML_Sanitize);
  6069. else
  6070. {
  6071. writeJSONValueToStream(out, thislevel, delimit, isSanitizedAndHidden(thislevel, flags, false));
  6072. }
  6073. }
  6074. if (!hiddenRootArrayObject)
  6075. {
  6076. if (root || complex)
  6077. {
  6078. if (flags & JSON_Format)
  6079. {
  6080. writeCharToStream(out, '\n');
  6081. writeCharsNToStream(out, ' ', indent);
  6082. }
  6083. writeCharToStream(out, '}');
  6084. delimit = true;
  6085. }
  6086. }
  6087. }
  6088. jlib_decl StringBuffer &toJSON(const IPropertyTree *tree, StringBuffer &ret, unsigned indent, byte flags)
  6089. {
  6090. CStringBufferMarkupIOAdapter adapter(ret);
  6091. bool delimit = false;
  6092. _toJSON(tree->queryBranch(NULL), adapter, indent, flags, delimit, true);
  6093. return ret;
  6094. }
  6095. void toJSON(const IPropertyTree *tree, IIOStream &out, unsigned indent, byte flags)
  6096. {
  6097. bool delimit = false;
  6098. _toJSON(tree, out, indent, flags, delimit, true);
  6099. }
  6100. void printJSON(const IPropertyTree *tree, unsigned indent, byte flags)
  6101. {
  6102. StringBuffer json;
  6103. toJSON(tree, json, indent, flags);
  6104. printf("%s", json.str());
  6105. }
  6106. void dbglogJSON(const IPropertyTree *tree, unsigned indent, unsigned flags)
  6107. {
  6108. StringBuffer json;
  6109. toJSON(tree, json, indent, flags);
  6110. DBGLOG("%s", json.str());
  6111. }
  6112. static inline void skipWS(const char *&xpath)
  6113. {
  6114. while (isspace(*xpath)) xpath++;
  6115. }
  6116. static void _validateXPathSyntax(const char *xpath);
  6117. static void validateQualifier(const char *&xxpath)
  6118. {
  6119. const char *xpath = xxpath;
  6120. const char *start = xpath;
  6121. skipWS(xpath);
  6122. const char *lhsStart = xpath;
  6123. for (;;)
  6124. {
  6125. switch (*xpath) {
  6126. case ']':
  6127. case '!':
  6128. case '=':
  6129. case '\0':
  6130. break;
  6131. default:
  6132. if (!isspace(*xpath))
  6133. {
  6134. xpath++;
  6135. continue;
  6136. }
  6137. }
  6138. break;
  6139. }
  6140. StringAttr lhs(lhsStart, xpath-lhsStart);
  6141. _validateXPathSyntax(lhs);
  6142. skipWS(xpath);
  6143. exprType tType = t_none;
  6144. if ('=' == *xpath)
  6145. {
  6146. ++xpath;
  6147. tType = t_equality;
  6148. }
  6149. else if ('!' == *xpath)
  6150. {
  6151. ++xpath;
  6152. if (*xpath && '=' == *xpath)
  6153. {
  6154. tType = t_inequality;
  6155. ++xpath;
  6156. }
  6157. else
  6158. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, 0, "Invalid qualifier expression");
  6159. }
  6160. if (t_none != tType)
  6161. {
  6162. skipWS(xpath);
  6163. if ('~' == *xpath)
  6164. {
  6165. ++xpath; // Signifies wild (now always true but still accepted...)
  6166. }
  6167. skipWS(xpath);
  6168. char qu = *xpath;
  6169. if (qu != '\'' && qu != '\"')
  6170. throw MakeXPathException(xpath, PTreeExcpt_XPath_ParseError, 0, "Syntax error - no opening \" or \'");
  6171. ++xpath;
  6172. while (*xpath && *xpath != qu)
  6173. xpath++;
  6174. if (!*xpath)
  6175. throw MakeXPathException(xpath, PTreeExcpt_XPath_ParseError, 0, "Syntax error - no closing \" or \'");
  6176. xpath++;
  6177. }
  6178. skipWS(xpath);
  6179. if (']' != *xpath)
  6180. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, 0, "No closing brace to qualifier");
  6181. xxpath = xpath;
  6182. }
  6183. static void _validateXPathSyntax(const char *xpath)
  6184. {
  6185. if (NULL == xpath || '\0' == *xpath)
  6186. return;
  6187. else
  6188. {
  6189. const char *_xpath = xpath;
  6190. restart:
  6191. if (NULL == xpath || '\0' == *xpath)
  6192. return;
  6193. switch (*xpath)
  6194. {
  6195. case '.':
  6196. ++xpath;
  6197. goto restart;
  6198. case '/':
  6199. ++xpath;
  6200. if ('/' == *xpath)
  6201. {
  6202. _validateXPathSyntax(xpath+1);
  6203. return;
  6204. }
  6205. goto restart;
  6206. case '[':
  6207. {
  6208. ++xpath;
  6209. if (isdigit(*xpath))
  6210. {
  6211. StringAttr index;
  6212. xpath = readIndex(xpath, index);
  6213. unsigned i = atoi(index.get());
  6214. if (i)
  6215. {
  6216. }
  6217. else
  6218. {
  6219. // should be syntax error
  6220. }
  6221. if (']' != *xpath)
  6222. throw MakeXPathException(_xpath, PTreeExcpt_XPath_ParseError, xpath-_xpath, "Qualifier brace unclosed");
  6223. }
  6224. else
  6225. validateQualifier(xpath);
  6226. ++xpath;
  6227. break;
  6228. }
  6229. default:
  6230. {
  6231. bool wild;
  6232. const char *start = xpath;
  6233. readWildId(xpath, wild); // validates also
  6234. size32_t s = xpath-start;
  6235. if (s)
  6236. {
  6237. StringAttr id(start, s);
  6238. if ('[' == *xpath) // check for local index not iterative qualifier.
  6239. {
  6240. const char *xxpath = xpath+1;
  6241. if (isdigit(*xxpath))
  6242. {
  6243. StringAttr idxstr;
  6244. xxpath = readIndex(xxpath, idxstr);
  6245. if (']' != *xxpath)
  6246. throw MakeXPathException(_xpath, PTreeExcpt_XPath_ParseError, xpath-_xpath, "Qualifier brace unclosed");
  6247. ++xxpath;
  6248. unsigned index = atoi(idxstr.get());
  6249. if (index)
  6250. {
  6251. }
  6252. xpath = xxpath;
  6253. }
  6254. }
  6255. }
  6256. else if ('@' == *xpath)
  6257. {
  6258. ++xpath;
  6259. const char *start = xpath;
  6260. readID(xpath, false);
  6261. size32_t s = xpath-start;
  6262. if (!s)
  6263. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Missing attribute?");
  6264. StringAttr id(start, s);
  6265. if (!validateXMLTag(id))
  6266. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Invalid xml tag: %s", id.get());
  6267. while (isspace(*xpath)) xpath++;
  6268. if ('\0' != *xpath)
  6269. throw MakeXPathException(start, PTreeExcpt_XPath_ParseError, xpath-start, "Cannot have embedded attribute within path (must be tail component)");
  6270. }
  6271. else
  6272. {
  6273. if ('[' != *xpath)
  6274. throw MakeXPathException(xpath, PTreeExcpt_XPath_ParseError, 0, "Qualifier expected e.g. [..]");
  6275. validateQualifier(xpath);
  6276. }
  6277. break;
  6278. }
  6279. }
  6280. }
  6281. if (*xpath == '\0' || (*xpath == '/' && '\0' == *(xpath+1)))
  6282. return;
  6283. else
  6284. _validateXPathSyntax(xpath);
  6285. }
  6286. bool validateXPathSyntax(const char *xpath, StringBuffer *error)
  6287. {
  6288. try
  6289. {
  6290. if (xpath && '/' == *xpath && *(xpath+1) != '/')
  6291. throw MakeXPathException(xpath, PTreeExcpt_XPath_Unsupported, 0, "Root specifier \"/\" specifier is not supported");
  6292. _validateXPathSyntax(xpath);
  6293. return true;
  6294. }
  6295. catch (IException *e)
  6296. {
  6297. if (error)
  6298. e->errorMessage(*error);
  6299. e->Release();
  6300. return false;
  6301. }
  6302. }
  6303. static bool isContentXPath(const char *xpath, StringBuffer &head)
  6304. {
  6305. unsigned l = xpath?strlen(xpath):0;
  6306. const char *x = xpath+l-2;
  6307. if (l>=2 && 0==strcmp(XMLTAG_CONTENT, x))
  6308. {
  6309. if (x != xpath)
  6310. head.append(x-xpath, xpath);
  6311. return true;
  6312. }
  6313. return false;
  6314. }
  6315. bool validateXMLParseXPath(const char *xpath, StringBuffer *error)
  6316. {
  6317. if (!xpath || !*xpath)
  6318. return true;
  6319. StringBuffer head;
  6320. if (isContentXPath(xpath, head))
  6321. {
  6322. if (head.length())
  6323. {
  6324. if ('/' == *xpath && '/' != *(xpath+1))
  6325. {
  6326. if (error)
  6327. {
  6328. Owned<IException> e = MakeStringException(0, "Invalid extract xml text '<>' usage, xpath cannot from be absolute: %s", xpath);
  6329. e->errorMessage(*error);
  6330. }
  6331. return false;
  6332. }
  6333. return validateXPathSyntax(head.str(), error);
  6334. }
  6335. return true;
  6336. }
  6337. else
  6338. return validateXPathSyntax('/' == *xpath && '/' != *(xpath+1) ? xpath+1 : xpath, error);
  6339. return true;
  6340. }
  6341. bool areMatchingPTrees(const IPropertyTree * left, const IPropertyTree * right)
  6342. {
  6343. if (left == right)
  6344. return true;
  6345. if (!left || !right)
  6346. return false;
  6347. bool isCaseInsensitive = left->isCaseInsensitive();
  6348. const char * lname = left->queryName();
  6349. const char * rname = right->queryName();
  6350. if (!lname || !rname)
  6351. {
  6352. if (lname || rname)
  6353. return false;
  6354. }
  6355. else if ((isCaseInsensitive ? stricmp(lname, rname) : strcmp(lname, rname)) != 0)
  6356. return false;
  6357. Owned<IAttributeIterator> leftAttrIter = left->getAttributes(true);
  6358. Owned<IAttributeIterator> rightAttrIter = right->getAttributes(true);
  6359. rightAttrIter->first();
  6360. ForEach(*leftAttrIter)
  6361. {
  6362. if (!rightAttrIter->isValid()) return false;
  6363. const char * lname = leftAttrIter->queryName();
  6364. const char * rname = rightAttrIter->queryName();
  6365. if ((isCaseInsensitive ? stricmp(lname, rname) : strcmp(lname, rname)) != 0)
  6366. return false;
  6367. if (strcmp(leftAttrIter->queryValue(), rightAttrIter->queryValue()) != 0)
  6368. return false;
  6369. rightAttrIter->next();
  6370. }
  6371. if (rightAttrIter->isValid()) return false;
  6372. Owned<IPropertyTreeIterator> leftElemIter = left->getElements("*", iptiter_sort);
  6373. Owned<IPropertyTreeIterator> rightElemIter = right->getElements("*", iptiter_sort);
  6374. rightElemIter->first();
  6375. ForEach(*leftElemIter)
  6376. {
  6377. if (!rightElemIter->isValid()) return false;
  6378. if (!areMatchingPTrees(&leftElemIter->query(), &rightElemIter->query()))
  6379. return false;
  6380. rightElemIter->next();
  6381. }
  6382. if (rightElemIter->isValid()) return false;
  6383. return true;
  6384. }
  6385. /////////////////////
  6386. static const char * skipWhitespace(const char * text)
  6387. {
  6388. while ((*text==' ') || (*text=='\t'))
  6389. text++;
  6390. return text;
  6391. }
  6392. static const char * skipAsterisk(const char * text)
  6393. {
  6394. if (*text=='*')
  6395. return skipWhitespace(text+1);
  6396. return text;
  6397. }
  6398. static const char * skipToNewline(const char * text)
  6399. {
  6400. while (*text && (*text != '\r') && (*text != '\n'))
  6401. text++;
  6402. return text;
  6403. }
  6404. static const char * skipNewline(const char * text)
  6405. {
  6406. if (*text == '\r')
  6407. text++;
  6408. if (*text == '\n')
  6409. text++;
  6410. return text;
  6411. }
  6412. void extractJavadoc(IPropertyTree * result, const char * text)
  6413. {
  6414. //Skip a leading blank line
  6415. text = skipWhitespace(text);
  6416. text = skipNewline(text);
  6417. //Now process each of the parameters...
  6418. StringBuffer tagname;
  6419. StringBuffer tagtext;
  6420. for (;;)
  6421. {
  6422. text = skipWhitespace(text);
  6423. text = skipAsterisk(text);
  6424. if ((*text == 0) || (*text == '@'))
  6425. {
  6426. if (tagtext.length())
  6427. {
  6428. if (tagname.length())
  6429. result->addProp(tagname.str(), tagtext.str());
  6430. else
  6431. result->setProp("", tagtext.str());
  6432. tagtext.clear();
  6433. }
  6434. if (*text == 0)
  6435. return;
  6436. text++;
  6437. const char * start = text;
  6438. while (isalnum(*text))
  6439. text++;
  6440. if (start != text)
  6441. tagname.clear().append(text-start, start);
  6442. text = skipWhitespace(text);
  6443. }
  6444. const char * start = text;
  6445. text = skipToNewline(text);
  6446. if (start != text)
  6447. {
  6448. if (tagtext.length())
  6449. tagtext.append(" ");
  6450. tagtext.append(text-start, start);
  6451. }
  6452. text = skipNewline(text);
  6453. }
  6454. }
  6455. /////////////////////
  6456. #ifdef _DEBUG
  6457. jlib_decl void validatePTree()
  6458. {
  6459. Owned<IPropertyTree> testTree = createPTreeFromXMLString(
  6460. "<ROOT>" \
  6461. " <E a=\"av1\" b=\"bv1\" c=\"cv1\"></E>" \
  6462. " <E a=\"av1\" b=\"bv1\" c=\"cv2\"></E>" \
  6463. " <E a=\"av2\" b=\"bv1\"></E>" \
  6464. " <SE c=\"cv1\"></SE>" \
  6465. " <E a=\"av1\" b=\"bv2\"></E>" \
  6466. " <E a=\"av2\" b=\"bv2\" c=\"cv3\">ev1</E>" \
  6467. "</ROOT>"
  6468. );
  6469. Owned<IPropertyTreeIterator> iter = testTree->getElements("E[@a=\"av1\"][@b=\"bv2\"]");
  6470. unsigned c = 0;
  6471. ForEach (*iter)
  6472. ++c;
  6473. assertex(1 == c);
  6474. int v = strcmp("bv1", testTree->queryProp("E[@a=\"av1\"][2]/@b"));
  6475. assertex(0 == v);
  6476. v = strcmp("cv2", testTree->queryProp("E[@a=\"av1\"][@b=\"bv1\"][2]/@c"));
  6477. assertex(0 == v);
  6478. v = strcmp("cv2", testTree->queryProp("E[@a=\"av1\"][2]/@c"));
  6479. assertex(0 == v);
  6480. v = strcmp("ev1", testTree->queryProp("E[@a=\"av2\"][@c]"));
  6481. assertex(0 == v);
  6482. }
  6483. jlib_decl void testValidateXPathSyntax()
  6484. {
  6485. verifyex(validateXPathSyntax("@abc"));
  6486. verifyex(validateXPathSyntax("prop"));
  6487. verifyex(validateXPathSyntax("a/b"));
  6488. verifyex(validateXPathSyntax("a/@b"));
  6489. const char *s = "a[@a=\"blah\"]/b";
  6490. verifyex(validateXPathSyntax(s));
  6491. s = "a/b[@b=\"blah\"]";
  6492. verifyex(validateXPathSyntax(s));
  6493. verifyex(validateXPathSyntax(s));
  6494. s = "a/b[b=\"blah\"]";
  6495. verifyex(validateXPathSyntax(s));
  6496. s = "a/b[a/b=\"blah\"]";
  6497. verifyex(validateXPathSyntax(s));
  6498. verifyex(validateXPathSyntax("a[1]/b[2]"));
  6499. s = "a[b]/b[c=\"a\"]/c";
  6500. verifyex(validateXPathSyntax(s));
  6501. verifyex(validateXPathSyntax("//a/b/c"));
  6502. verifyex(!validateXPathSyntax("a[b"));
  6503. verifyex(!validateXPathSyntax("a["));
  6504. verifyex(!validateXPathSyntax("a]"));
  6505. verifyex(!validateXPathSyntax("a[b=blah]"));
  6506. verifyex(!validateXPathSyntax("@a/b"));
  6507. verifyex(!validateXPathSyntax("a[b[c]]"));
  6508. verifyex(validateXMLParseXPath("<>"));
  6509. verifyex(validateXMLParseXPath("a/b/c<>"));
  6510. verifyex(validateXMLParseXPath("a/b/<>"));
  6511. verifyex(validateXMLParseXPath("/a/b"));
  6512. verifyex(!validateXMLParseXPath("a/b[\"]/<>"));
  6513. verifyex(!validateXMLParseXPath("/<>"));
  6514. }
  6515. jlib_decl void testJdocCompare()
  6516. {
  6517. Owned<IPropertyTree> t1 = createPTree();
  6518. Owned<IPropertyTree> t2 = createPTree();
  6519. Owned<IPropertyTree> t3 = createPTree();
  6520. Owned<IPropertyTree> t4 = createPTree();
  6521. Owned<IPropertyTree> t5 = createPTree();
  6522. extractJavadoc(t1, "Defines a record that contains information about a person");
  6523. extractJavadoc(t2, "Allows the name table to be filtered.\n\n@param ages\tThe ages that are allowed to be processed.\n\t\tbadForename Forname to avoid.\n\n@return\tthe filtered dataset.");
  6524. extractJavadoc(t3, "Allows the name table to be filtered.\n\n@param ages\tThe ages that are allowed to be processed.\n\t\tbadForename Forname to avoid.\n\n@return\tthe filtered dataset.");
  6525. extractJavadoc(t4, "Allows the name table to be filtered.\n\n@param ages\tThe ages that are allowed to be processed.\n\t\tbadForename Forname to avoid.\n\n@return\tthe filtered dataset.");
  6526. extractJavadoc(t5, "Allows the name table to be filtered.\n\n@param ages\tThe ages that are allowed to be processed.\n\t\tbadForename Forname to avoid.\n\n@return\tthe filtered dataset.");
  6527. IPropertyTree * t2c = t2->addPropTree("Child1", createPTree());
  6528. extractJavadoc(t2c, "This is some child data\n\n@param ages\tThe ages that are allowed to be processed.");
  6529. IPropertyTree * t3c = t3->addPropTree("Child1", createPTree());
  6530. extractJavadoc(t3c, "This is some child data\n\n@param ages\tThe ages that are allowed to be processed.");
  6531. IPropertyTree * t4c = t4->addPropTree("Child1", createPTree());
  6532. extractJavadoc(t4c, "This is some child data\n\n@param ages\tThe ages that are allowed to be processed, but differs.");
  6533. IPropertyTree * t5c = t5->addPropTree("Child1", createPTree());
  6534. extractJavadoc(t5c, "This is some child data\n\n@param ages\tThe ages that are allowed to be processed.");
  6535. t2->setProp("@childAttr", "1");
  6536. t3->setProp("@childAttr", "1");
  6537. t4->setProp("@childAttr", "1");
  6538. t5->setProp("@childAttr", "2");
  6539. verifyex(areMatchingPTrees(NULL, NULL));
  6540. verifyex(!areMatchingPTrees(NULL, t1));
  6541. verifyex(!areMatchingPTrees(t1, NULL));
  6542. verifyex(areMatchingPTrees(t1, t1));
  6543. verifyex(areMatchingPTrees(t2, t3));
  6544. verifyex(!areMatchingPTrees(t2, t4));
  6545. verifyex(!areMatchingPTrees(t2, t5));
  6546. }
  6547. #endif
  6548. template <class BASE_PTREE>
  6549. class COrderedPTree : public BASE_PTREE
  6550. {
  6551. template <class BASECHILDMAP>
  6552. class jlib_decl COrderedChildMap : public BASECHILDMAP
  6553. {
  6554. typedef COrderedChildMap<BASECHILDMAP> SELF;
  6555. ICopyArrayOf<IPropertyTree> order;
  6556. public:
  6557. IMPLEMENT_SUPERHASHTABLEOF_REF_FIND(IPropertyTree, constcharptr);
  6558. COrderedChildMap<BASECHILDMAP>() : BASECHILDMAP() { }
  6559. ~COrderedChildMap<BASECHILDMAP>() { SELF::kill(); }
  6560. virtual unsigned numChildren() const override { return order.ordinality(); }
  6561. virtual IPropertyTreeIterator *getIterator(bool sort) override
  6562. {
  6563. class CPTArrayIterator : public ArrayIIteratorOf<IArrayOf<IPropertyTree>, IPropertyTree, IPropertyTreeIterator>
  6564. {
  6565. IArrayOf<IPropertyTree> elems;
  6566. public:
  6567. CPTArrayIterator(ICopyArrayOf<IPropertyTree> &order, bool sort) : ArrayIIteratorOf<IArrayOf<IPropertyTree>, IPropertyTree, IPropertyTreeIterator>(elems)
  6568. {
  6569. ForEachItemIn(e, order)
  6570. elems.append(*LINK(&order.item(e)));
  6571. if (sort)
  6572. elems.sort(comparePropTrees);
  6573. }
  6574. };
  6575. return new CPTArrayIterator(order, sort);
  6576. }
  6577. virtual bool set(const char *key, IPropertyTree *tree) override
  6578. {
  6579. IPropertyTree *existing = find(*key);
  6580. if (existing)
  6581. {
  6582. unsigned pos = order.find(*existing);
  6583. BASECHILDMAP::set(key, tree);
  6584. order.replace(*tree, pos);
  6585. }
  6586. else
  6587. {
  6588. BASECHILDMAP::set(key, tree);
  6589. order.append(*tree);
  6590. }
  6591. return true;
  6592. }
  6593. virtual bool replace(const char *key, IPropertyTree *tree) override // provides different semantics, used if element being replaced is not to be treated as deleted.
  6594. {
  6595. return set(key, tree);
  6596. }
  6597. virtual bool remove(const char *key) override
  6598. {
  6599. IPropertyTree *child = BASECHILDMAP::find(*key);
  6600. if (!child)
  6601. return false;
  6602. order.zap(*child);
  6603. return BASECHILDMAP::removeExact(child);
  6604. }
  6605. virtual bool removeExact(IPropertyTree *child) override
  6606. {
  6607. order.zap(*child);
  6608. return BASECHILDMAP::removeExact(child);
  6609. }
  6610. };
  6611. public:
  6612. typedef COrderedPTree<BASE_PTREE> SELF;
  6613. COrderedPTree<BASE_PTREE>(const char *name=NULL, byte flags=ipt_none, IPTArrayValue *value=NULL, ChildMap *children=NULL)
  6614. : BASE_PTREE(name, flags|ipt_ordered, value, children) { }
  6615. virtual bool isEquivalent(IPropertyTree *tree) const override { return (NULL != QUERYINTERFACE(tree, COrderedPTree<BASE_PTREE>)); }
  6616. virtual IPropertyTree *create(const char *name=NULL, IPTArrayValue *value=NULL, ChildMap *children=NULL, bool existing=false) override
  6617. {
  6618. return new COrderedPTree<BASE_PTREE>(name, SELF::flags, value, children);
  6619. }
  6620. virtual IPropertyTree *create(MemoryBuffer &mb) override
  6621. {
  6622. IPropertyTree *tree = new COrderedPTree<BASE_PTREE>();
  6623. tree->deserialize(mb);
  6624. return tree;
  6625. }
  6626. virtual void createChildMap() override
  6627. {
  6628. if (SELF::isnocase())
  6629. SELF::children = new COrderedChildMap<ChildMapNC>();
  6630. else
  6631. SELF::children = new COrderedChildMap<ChildMap>();
  6632. }
  6633. };
  6634. IPropertyTree *createPTree(byte flags)
  6635. {
  6636. return createPTree(NULL, flags);
  6637. }
  6638. IPropertyTree *createPTree(const char *name, byte flags)
  6639. {
  6640. switch (flags & (ipt_ordered|ipt_fast|ipt_lowmem))
  6641. {
  6642. case ipt_ordered|ipt_fast:
  6643. return new COrderedPTree<LocalPTree>(name, flags);
  6644. case ipt_ordered|ipt_lowmem:
  6645. return new COrderedPTree<CAtomPTree>(name, flags);
  6646. case ipt_ordered:
  6647. return new COrderedPTree<DEFAULT_PTREE_TYPE>(name, flags);
  6648. case ipt_fast:
  6649. return new LocalPTree(name, flags);
  6650. case ipt_lowmem:
  6651. return new CAtomPTree(name, flags);
  6652. case 0:
  6653. return new DEFAULT_PTREE_TYPE(name, flags);
  6654. default:
  6655. throwUnexpectedX("Invalid flags - ipt_fast and ipt_lowmem should not be specified together");
  6656. }
  6657. }
  6658. typedef enum _ptElementType
  6659. {
  6660. elementTypeUnknown,
  6661. elementTypeNull,
  6662. elementTypeString,
  6663. elementTypeBool,
  6664. elementTypeInteger,
  6665. elementTypeReal,
  6666. elementTypeObject,
  6667. elementTypeArray
  6668. } ptElementType;
  6669. template <typename X>
  6670. class CJSONReaderBase : public CommonReaderBase<X>
  6671. {
  6672. public:
  6673. typedef CommonReaderBase<X> PARENT;
  6674. using PARENT::reset;
  6675. using PARENT::nextChar;
  6676. using PARENT::readNextToken;
  6677. using PARENT::checkReadNext;
  6678. using PARENT::checkStartReadNext;
  6679. using PARENT::readNext;
  6680. using PARENT::expecting;
  6681. using PARENT::match;
  6682. using PARENT::error;
  6683. using PARENT::skipWS;
  6684. using PARENT::rewind;
  6685. using PARENT::ignoreWhiteSpace;
  6686. CJSONReaderBase(ISimpleReadStream &_stream, IPTreeNotifyEvent &_iEvent, PTreeReaderOptions _readerOptions, size32_t _bufSize=0) :
  6687. CommonReaderBase<X>(_stream, _iEvent, _readerOptions, _bufSize)
  6688. {
  6689. }
  6690. CJSONReaderBase(const void *_buf, size32_t bufLength, IPTreeNotifyEvent &_iEvent, PTreeReaderOptions _readerOptions) :
  6691. CommonReaderBase<X>(_buf, bufLength, _iEvent, _readerOptions)
  6692. {
  6693. }
  6694. CJSONReaderBase(const void *_buf, IPTreeNotifyEvent &_iEvent, PTreeReaderOptions _readerOptions) :
  6695. CommonReaderBase<X>(_buf, _iEvent, _readerOptions)
  6696. {
  6697. }
  6698. ~CJSONReaderBase()
  6699. {
  6700. }
  6701. protected:
  6702. inline StringBuffer &appendChar(StringBuffer &id, char c)
  6703. {
  6704. int charlen = validJSONUtf8ChrLen(c);
  6705. if (!charlen)
  6706. error("invalid JSON character", true);
  6707. id.append(nextChar);
  6708. while (--charlen)
  6709. {
  6710. readNext();
  6711. id.append(nextChar);
  6712. }
  6713. return id;
  6714. }
  6715. void readString(StringBuffer &value)
  6716. {
  6717. readNext();
  6718. StringBuffer s;
  6719. bool decode=false;
  6720. while ('\"'!=nextChar)
  6721. {
  6722. if (nextChar=='\\')
  6723. decode=true;
  6724. appendChar(s, nextChar);
  6725. readNext();
  6726. }
  6727. size32_t r = s.length();
  6728. if (ignoreWhiteSpace)
  6729. s.trimRight();
  6730. if (decode)
  6731. _decodeJSON(r, s.str(), value, s.length()+1);
  6732. else
  6733. value.swapWith(s);
  6734. }
  6735. void readName(StringBuffer &name)
  6736. {
  6737. if ('\"'!=nextChar)
  6738. expecting("\"");
  6739. readString(name);
  6740. if (!name.length())
  6741. error("empty JSON id");
  6742. readNext();
  6743. skipWS();
  6744. if (':'!=nextChar)
  6745. expecting(":");
  6746. readNext();
  6747. }
  6748. ptElementType readValue(StringBuffer &value)
  6749. {
  6750. ptElementType type = elementTypeUnknown;
  6751. switch (nextChar)
  6752. {
  6753. case '\"':
  6754. {
  6755. readString(value);
  6756. type = elementTypeString;
  6757. break;
  6758. }
  6759. case 't':
  6760. match("rue", "Bad value");
  6761. value.append("true");
  6762. type = elementTypeBool;
  6763. break;
  6764. case 'f':
  6765. match("alse", "Bad value");
  6766. value.append("false");
  6767. type = elementTypeBool;
  6768. break;
  6769. case 'n':
  6770. match("ull", "Bad value");
  6771. type = elementTypeNull;
  6772. break;
  6773. case '-':
  6774. value.append(nextChar);
  6775. readNext();
  6776. //fall through
  6777. default:
  6778. if (!isdigit(nextChar))
  6779. error("Bad value");
  6780. type = elementTypeInteger;
  6781. bool exponent = false;
  6782. while (isdigit(nextChar) || '.'==nextChar || 'e'==nextChar || 'E'==nextChar)
  6783. {
  6784. if ('e'==nextChar || 'E'==nextChar)
  6785. {
  6786. if (exponent)
  6787. error("Bad value");
  6788. exponent=true;
  6789. value.append(nextChar);
  6790. readNext();
  6791. if ('-'==nextChar)
  6792. type=elementTypeReal;
  6793. else if (!isdigit(nextChar) && '+'!=nextChar)
  6794. error("Bad value");
  6795. }
  6796. if ('.'==nextChar)
  6797. {
  6798. if (exponent || type==elementTypeReal) //already found decimal
  6799. error("Bad value");
  6800. type = elementTypeReal;
  6801. }
  6802. value.append(nextChar);
  6803. readNext();
  6804. }
  6805. rewind(1);
  6806. break;
  6807. }
  6808. return type;
  6809. }
  6810. const char *_decodeJSON(unsigned read, const char *startMark, StringBuffer &ret, unsigned len)
  6811. {
  6812. const char *errMark = NULL;
  6813. try { return decodeJSON(startMark, ret, len, &errMark); }
  6814. catch (IException *e)
  6815. {
  6816. if (errMark)
  6817. {
  6818. if (read>(unsigned)(errMark-startMark))
  6819. rewind((unsigned)(read - (errMark-startMark)));
  6820. else
  6821. rewind((unsigned)(errMark-startMark));
  6822. }
  6823. StringBuffer errMsg;
  6824. e->errorMessage(errMsg);
  6825. e->Release();
  6826. error(errMsg.str());
  6827. }
  6828. return NULL; // will never get here.
  6829. }
  6830. };
  6831. template <class X>
  6832. class CJSONReader : public CJSONReaderBase<X>, implements IPTreeReader
  6833. {
  6834. typedef CJSONReaderBase<X> PARENT;
  6835. using PARENT::checkBOM;
  6836. using PARENT::rewind;
  6837. using PARENT::readNext;
  6838. using PARENT::readValue;
  6839. using PARENT::readName;
  6840. using PARENT::checkReadNext;
  6841. using PARENT::checkSkipWS;
  6842. using PARENT::checkStartReadNext;
  6843. using PARENT::expecting;
  6844. using PARENT::error;
  6845. using PARENT::eos;
  6846. using PARENT::_decodeJSON;
  6847. using PARENT::skipWS;
  6848. using PARENT::nextChar;
  6849. using PARENT::curOffset;
  6850. using PARENT::noRoot;
  6851. using PARENT::ignoreWhiteSpace;
  6852. using PARENT::iEvent;
  6853. // StringBuffer tmpStr;
  6854. public:
  6855. IMPLEMENT_IINTERFACE;
  6856. CJSONReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions, size32_t bufSize=0)
  6857. : PARENT(stream, iEvent, readerOptions, bufSize)
  6858. {
  6859. }
  6860. CJSONReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions)
  6861. : PARENT(buf, bufLength, iEvent, readerOptions)
  6862. {
  6863. }
  6864. CJSONReader(const void *buf, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions)
  6865. : PARENT(buf, iEvent, readerOptions)
  6866. {
  6867. }
  6868. void readValueNotify(const char *name, bool skipAttributes, StringBuffer *retValue, bool *isValueBinary)
  6869. {
  6870. offset_t startOffset = curOffset;
  6871. StringBuffer value;
  6872. if (readValue(value)!=elementTypeNull)
  6873. {
  6874. if ('@'==*name)
  6875. {
  6876. if (!skipAttributes)
  6877. iEvent->newAttribute(name, value.str());
  6878. return;
  6879. }
  6880. else if ('#'==*name)
  6881. {
  6882. dbgassertex(retValue && isValueBinary);
  6883. *isValueBinary = false;
  6884. if (0 == strncmp(name+1, "value", 5)) // this is a special IPT JSON prop name, representing a 'complex' value
  6885. {
  6886. if ('\0' == *(name+6)) // #value
  6887. {
  6888. retValue->swapWith(value);
  6889. return;
  6890. }
  6891. else if (streq(name+6, "bin")) // #valuebin
  6892. {
  6893. *isValueBinary = true;
  6894. JBASE64_Decode(value.str(), *retValue);
  6895. return;
  6896. }
  6897. }
  6898. }
  6899. }
  6900. iEvent->beginNode(name, false, startOffset);
  6901. iEvent->beginNodeContent(name);
  6902. iEvent->endNode(name, value.length(), value.str(), false, curOffset);
  6903. }
  6904. void readArray(const char *name)
  6905. {
  6906. if ('@'==*name)
  6907. name++;
  6908. readNext();
  6909. skipWS();
  6910. while (']' != nextChar)
  6911. {
  6912. switch (nextChar)
  6913. {
  6914. case '[':
  6915. iEvent->beginNode(name, false, curOffset);
  6916. iEvent->beginNodeContent(name);
  6917. readArray(name);
  6918. iEvent->endNode(name, 0, "", false, curOffset);
  6919. break;
  6920. case '{':
  6921. readObject(name);
  6922. break;
  6923. default:
  6924. readValueNotify(name, true, nullptr, nullptr);
  6925. break;
  6926. }
  6927. readNext();
  6928. skipWS();
  6929. if (','==nextChar)
  6930. readNext();
  6931. else if (']'!=nextChar)
  6932. error("expected ',' or ']'");
  6933. skipWS();
  6934. }
  6935. }
  6936. void readChild(const char *name, bool skipAttributes, StringBuffer *value, bool *isValueBinary)
  6937. {
  6938. skipWS();
  6939. switch (nextChar)
  6940. {
  6941. case '}':
  6942. {
  6943. VStringBuffer msg("named item with no value defined %s [%d]", name, (int) curOffset);
  6944. error(msg.str());
  6945. }
  6946. break;
  6947. case '{':
  6948. readObject(name);
  6949. break;
  6950. case '[':
  6951. readArray(name);
  6952. break;
  6953. default:
  6954. readValueNotify(name, skipAttributes, value, isValueBinary);
  6955. break;
  6956. }
  6957. }
  6958. void readObject(const char *name)
  6959. {
  6960. if ('@'==*name)
  6961. name++;
  6962. iEvent->beginNode(name, false, curOffset);
  6963. readNext();
  6964. skipWS();
  6965. bool attributesFinalized=false;
  6966. StringBuffer childValue; // for #value
  6967. bool isChildValueBinary = false; // for #value
  6968. while ('}' != nextChar)
  6969. {
  6970. StringBuffer tagName;
  6971. readName(tagName);
  6972. //internal convention so we can convert to and from xml
  6973. //values at top of object with names starting with '@' become ptree attributes
  6974. if (*tagName.str()!='@')
  6975. attributesFinalized=true;
  6976. readChild(tagName.str(), attributesFinalized, &childValue.clear(), &isChildValueBinary);
  6977. readNext();
  6978. skipWS();
  6979. if (','==nextChar)
  6980. readNext();
  6981. else if ('}'!=nextChar)
  6982. error("expected ',' or '}'");
  6983. skipWS();
  6984. }
  6985. iEvent->endNode(name, childValue.length(), childValue.str(), isChildValueBinary, curOffset);
  6986. }
  6987. void loadJSON()
  6988. {
  6989. if (!checkStartReadNext())
  6990. return;
  6991. if (checkBOM() && !checkReadNext())
  6992. return;
  6993. if (!checkSkipWS())
  6994. return;
  6995. if (noRoot)
  6996. {
  6997. StringBuffer tagName;
  6998. for (;;)
  6999. {
  7000. switch (nextChar)
  7001. {
  7002. case '\"': //treat named objects like we're in a noroot object
  7003. readName(tagName.clear());
  7004. readChild(tagName.str(), true, nullptr, nullptr);
  7005. break;
  7006. case '{': //treat unnamed objects like we're in a noroot array
  7007. readObject("__object__");
  7008. break;
  7009. case '[': //treat unnamed arrays like we're in a noroot array
  7010. iEvent->beginNode("__array__", false, curOffset);
  7011. readArray("__item__");
  7012. iEvent->endNode("__array__", 0, "", false, curOffset);
  7013. break;
  7014. default:
  7015. expecting("{[ or \"");
  7016. }
  7017. if (!checkReadNext() || !checkSkipWS())
  7018. break;
  7019. switch (nextChar)
  7020. {
  7021. case '{': //support file formats with whitespace (usually \n) seperated objects at the root
  7022. case '[':
  7023. break;
  7024. case ',':
  7025. readNext();
  7026. skipWS();
  7027. break;
  7028. default:
  7029. expecting(",");
  7030. break;
  7031. }
  7032. }
  7033. }
  7034. else
  7035. {
  7036. if ('{' == nextChar)
  7037. readObject("__object__");
  7038. else if ('[' == nextChar)
  7039. {
  7040. iEvent->beginNode("__array__", false, curOffset);
  7041. readArray("__item__");
  7042. iEvent->endNode("__array__", 0, "", false, curOffset);
  7043. }
  7044. else
  7045. error("expected '{' or '['");
  7046. if (checkReadNext() && checkSkipWS())
  7047. error("trailing content after JSON closed");
  7048. }
  7049. }
  7050. // IPTreeReader
  7051. virtual void load() { loadJSON(); }
  7052. virtual offset_t queryOffset() { return curOffset; }
  7053. };
  7054. template <class X>
  7055. class CPullJSONReader : public CJSONReaderBase<X>, implements IPullPTreeReader
  7056. {
  7057. typedef CJSONReaderBase<X> PARENT;
  7058. using PARENT::checkBOM;
  7059. using PARENT::rewind;
  7060. using PARENT::readNext;
  7061. using PARENT::readValue;
  7062. using PARENT::readName;
  7063. using PARENT::checkReadNext;
  7064. using PARENT::checkSkipWS;
  7065. using PARENT::checkStartReadNext;
  7066. using PARENT::expecting;
  7067. using PARENT::error;
  7068. using PARENT::eos;
  7069. using PARENT::_decodeJSON;
  7070. using PARENT::skipWS;
  7071. using PARENT::nextChar;
  7072. using PARENT::curOffset;
  7073. using PARENT::noRoot;
  7074. using PARENT::ignoreWhiteSpace;
  7075. using PARENT::iEvent;
  7076. class CStateInfo : public CInterface
  7077. {
  7078. public:
  7079. CStateInfo()
  7080. {
  7081. tag.ensureCapacity(15);
  7082. type = elementTypeUnknown;
  7083. childCount = 0;
  7084. wnsTag = NULL;
  7085. }
  7086. inline void reset()
  7087. {
  7088. wnsTag = NULL;
  7089. tag.clear();
  7090. tagText.clear();
  7091. type = elementTypeUnknown;
  7092. childCount = 0;
  7093. }
  7094. StringBuffer tag;
  7095. StringBuffer tagText;
  7096. ptElementType type;
  7097. const char *wnsTag;
  7098. unsigned childCount;
  7099. };
  7100. CICopyArrayOf<CStateInfo> stack, freeStateInfo;
  7101. CStateInfo *stateInfo;
  7102. enum ParseStates { headerStart, nameStart, valueStart, itemStart, objAttributes, itemContent, itemEnd } state;
  7103. bool endOfRoot;
  7104. bool preReadItemName;
  7105. bool more;
  7106. StringBuffer tag, value;
  7107. void init()
  7108. {
  7109. state = headerStart;
  7110. stateInfo = NULL;
  7111. endOfRoot = false;
  7112. preReadItemName = false;
  7113. more = true;
  7114. }
  7115. virtual void resetState()
  7116. {
  7117. stack.kill();
  7118. more = true;
  7119. }
  7120. public:
  7121. IMPLEMENT_IINTERFACE;
  7122. CPullJSONReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions, size32_t bufSize=0)
  7123. : CJSONReaderBase<X>(stream, iEvent, readerOptions, bufSize)
  7124. {
  7125. init();
  7126. }
  7127. CPullJSONReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions)
  7128. : CJSONReaderBase<X>(buf, bufLength, iEvent, readerOptions)
  7129. {
  7130. init();
  7131. }
  7132. CPullJSONReader(const void *buf, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions)
  7133. : CJSONReaderBase<X>(buf, iEvent, readerOptions)
  7134. {
  7135. init();
  7136. }
  7137. ~CPullJSONReader()
  7138. {
  7139. ForEachItemIn(i, stack)
  7140. delete &stack.item(i);
  7141. ForEachItemIn(i2, freeStateInfo)
  7142. delete &freeStateInfo.item(i2);
  7143. }
  7144. inline void checkDelimiter(const char *msg=",")
  7145. {
  7146. if (stateInfo && stateInfo->childCount > 0)
  7147. {
  7148. if (','!=nextChar)
  7149. expecting(msg);
  7150. readNext();
  7151. skipWS();
  7152. }
  7153. }
  7154. inline ptElementType getParentType()
  7155. {
  7156. if (stack.ordinality()<2)
  7157. return stateInfo->type;
  7158. return ((CStateInfo *)&stack.tos(1))->type;
  7159. }
  7160. void beginNode(const char *name, offset_t offset, ptElementType jsonType, bool notify=true)
  7161. {
  7162. if (stateInfo)
  7163. stateInfo->childCount++;
  7164. if (freeStateInfo.ordinality())
  7165. {
  7166. stateInfo = &freeStateInfo.popGet();
  7167. stateInfo->reset();
  7168. }
  7169. else
  7170. stateInfo = new CStateInfo;
  7171. stack.append(*stateInfo);
  7172. stateInfo->type=jsonType;
  7173. if (name)
  7174. stateInfo->tag.set(name);
  7175. else
  7176. stateInfo->tag.swapWith(tag);
  7177. stateInfo->wnsTag = stateInfo->tag.str();
  7178. if (!notify)
  7179. return;
  7180. try
  7181. {
  7182. iEvent->beginNode(stateInfo->wnsTag, false, offset);
  7183. }
  7184. catch (IPTreeException *pe)
  7185. {
  7186. if (PTreeExcpt_InvalidTagName == pe->errorCode())
  7187. {
  7188. pe->Release();
  7189. StringBuffer msg("Expecting valid start tag, but got \"");
  7190. error(msg.append(name).append("\"").str());
  7191. }
  7192. throw;
  7193. }
  7194. }
  7195. inline const char *arrayItemName(const char *defaultName)
  7196. {
  7197. if (stack.ordinality()>1)
  7198. return stateInfo->wnsTag;
  7199. return defaultName;
  7200. }
  7201. bool arrayItem(offset_t offset)
  7202. {
  7203. skipWS();
  7204. switch (nextChar)
  7205. {
  7206. case ']':
  7207. state=itemContent;
  7208. if (stack.ordinality()>1)
  7209. readNext();
  7210. if (!endNode(curOffset, getParentType()==elementTypeArray))
  7211. return false;
  7212. break;
  7213. case '{':
  7214. state=objAttributes;
  7215. readNext();
  7216. beginNode(arrayItemName("__object__"), offset, elementTypeObject);
  7217. break;
  7218. case '[':
  7219. state=valueStart;
  7220. readNext();
  7221. beginNode(arrayItemName("__array__"), offset, elementTypeArray, true);
  7222. break;
  7223. default:
  7224. state=valueStart;
  7225. ptElementType type = readValue(value.clear());
  7226. readNext();
  7227. beginNode(arrayItemName("__item__"), offset, type, true);
  7228. stateInfo->tagText.swapWith(value);
  7229. break;
  7230. }
  7231. return true;
  7232. }
  7233. void namedItem()
  7234. {
  7235. if (!preReadItemName)
  7236. readName(tag.clear());
  7237. else
  7238. preReadItemName = false;
  7239. skipWS();
  7240. switch (nextChar)
  7241. {
  7242. case '}':
  7243. error("unexpected object close marker");
  7244. case ']':
  7245. error("unexpected array close marker");
  7246. case '{':
  7247. state=objAttributes;
  7248. readNext();
  7249. beginNode(NULL, curOffset, elementTypeObject);
  7250. break;
  7251. case '[':
  7252. readNext();
  7253. beginNode(NULL, curOffset, elementTypeArray, false); //false because items present events, not the array
  7254. arrayItem(curOffset); //so process the first item so every next() has event
  7255. break;
  7256. default:
  7257. state=valueStart;
  7258. ptElementType type = readValue(value.clear());
  7259. readNext();
  7260. beginNode(NULL, curOffset, type);
  7261. stateInfo->tagText.swapWith(value);
  7262. break;
  7263. }
  7264. }
  7265. void rootItem()
  7266. {
  7267. if ('\"'==nextChar)
  7268. namedItem();
  7269. else if ('{'==nextChar || '['==nextChar)
  7270. arrayItem(curOffset);
  7271. else
  7272. expecting("[{ or \"");
  7273. }
  7274. bool rootNext()
  7275. {
  7276. if (!noRoot)
  7277. return false;
  7278. if (!checkReadNext() || !checkSkipWS())
  7279. return true;
  7280. switch (nextChar)
  7281. {
  7282. case '{': //support files where root level objects are separated by whitespace (usually \n)
  7283. case '[':
  7284. case ',':
  7285. break;
  7286. default:
  7287. expecting(",");
  7288. }
  7289. return true;
  7290. }
  7291. void newNamedAttribute()
  7292. {
  7293. skipWS();
  7294. readValue(value.clear());
  7295. readNext();
  7296. stateInfo->childCount++;
  7297. iEvent->newAttribute(tag.str(), value.str());
  7298. }
  7299. bool endNode(offset_t offset, bool notify=true)
  7300. {
  7301. if (stack.ordinality()<2)
  7302. {
  7303. state = headerStart;
  7304. more = rootNext();
  7305. }
  7306. if (notify)
  7307. {
  7308. if (stateInfo->type==elementTypeNull)
  7309. iEvent->endNode(stateInfo->wnsTag, 0, "", false, offset);
  7310. else
  7311. iEvent->endNode(stateInfo->wnsTag, stateInfo->tagText.length(), stateInfo->tagText.str(), false, offset);
  7312. }
  7313. freeStateInfo.append(*stateInfo);
  7314. stack.pop();
  7315. stateInfo = (stack.ordinality()) ? &stack.tos() : NULL;
  7316. return true;
  7317. }
  7318. // IPullPTreeReader
  7319. virtual void load()
  7320. {
  7321. while (next()) {}
  7322. }
  7323. virtual void reset()
  7324. {
  7325. PARENT::reset();
  7326. resetState();
  7327. }
  7328. virtual offset_t queryOffset() { return curOffset; }
  7329. virtual bool next()
  7330. {
  7331. if (!more)
  7332. return false;
  7333. checkStartReadNext();
  7334. checkSkipWS();
  7335. switch (state)
  7336. {
  7337. case headerStart:
  7338. {
  7339. if (nextChar!='{' && nextChar!='[') //already positioned at start
  7340. {
  7341. if (!checkReadNext())
  7342. return false;
  7343. if (checkBOM())
  7344. if (!checkReadNext())
  7345. return false;
  7346. if (!checkSkipWS())
  7347. return false;
  7348. }
  7349. if (noRoot)
  7350. rootItem();
  7351. else
  7352. {
  7353. switch (nextChar)
  7354. {
  7355. case '{':
  7356. state=objAttributes;
  7357. readNext();
  7358. beginNode("__object__", curOffset, elementTypeObject);
  7359. break;
  7360. case '[':
  7361. state=valueStart;
  7362. readNext();
  7363. beginNode("__array__", curOffset, elementTypeArray);
  7364. break;
  7365. default:
  7366. expecting("{ or [");
  7367. break;
  7368. }
  7369. }
  7370. break;
  7371. }
  7372. case nameStart:
  7373. namedItem();
  7374. break;
  7375. case objAttributes:
  7376. {
  7377. if ('}'==nextChar)
  7378. {
  7379. state=itemEnd;
  7380. iEvent->beginNodeContent(stateInfo->wnsTag);
  7381. break;
  7382. }
  7383. checkDelimiter(", or }");
  7384. if (nextChar != '\"')
  7385. expecting("\"");
  7386. readName(tag.clear());
  7387. if (tag.charAt(0)=='@')
  7388. newNamedAttribute();
  7389. else
  7390. {
  7391. preReadItemName = true;
  7392. state=itemContent;
  7393. stateInfo->childCount=0;
  7394. iEvent->beginNodeContent(stateInfo->wnsTag);
  7395. }
  7396. break;
  7397. }
  7398. case valueStart:
  7399. state=itemContent;
  7400. iEvent->beginNodeContent(stateInfo->wnsTag);
  7401. break;
  7402. case itemContent:
  7403. {
  7404. switch (stateInfo->type)
  7405. {
  7406. case elementTypeBool:
  7407. case elementTypeString:
  7408. case elementTypeInteger:
  7409. case elementTypeReal:
  7410. case elementTypeNull:
  7411. return endNode(curOffset);
  7412. break;
  7413. case elementTypeArray:
  7414. if (']'!=nextChar)
  7415. checkDelimiter(", or ]");
  7416. return arrayItem(curOffset);
  7417. case elementTypeObject:
  7418. if ('}'!=nextChar)
  7419. {
  7420. checkDelimiter(", or }");
  7421. namedItem();
  7422. }
  7423. else
  7424. {
  7425. if (stack.ordinality()>1)
  7426. readNext();
  7427. return endNode(curOffset);
  7428. }
  7429. break;
  7430. }
  7431. break;
  7432. }
  7433. case itemEnd:
  7434. {
  7435. if (!stack.length())
  7436. {
  7437. if (!noRoot || !rootNext())
  7438. return false;
  7439. readNext();
  7440. skipWS();
  7441. rootItem();
  7442. }
  7443. else
  7444. {
  7445. readNext();
  7446. state = itemContent;
  7447. return endNode(curOffset);
  7448. }
  7449. break;
  7450. }
  7451. }
  7452. return true;
  7453. }
  7454. };
  7455. IPTreeReader *createJSONStreamReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions, size32_t bufSize)
  7456. {
  7457. class CJSONStreamReader : public CJSONReader<CInstStreamReader>
  7458. {
  7459. public:
  7460. CJSONStreamReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions, size32_t bufSize=0) : CJSONReader<CInstStreamReader>(stream, iEvent, readerOptions, bufSize) { }
  7461. };
  7462. return new CJSONStreamReader(stream, iEvent, readerOptions, bufSize);
  7463. }
  7464. IPTreeReader *createJSONStringReader(const char *json, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions)
  7465. {
  7466. class CJSONStringReader : public CJSONReader<CInstStringReader>
  7467. {
  7468. public:
  7469. CJSONStringReader(const void *json, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions) : CJSONReader<CInstStringReader>(json, iEvent, readerOptions) { }
  7470. };
  7471. if (NULL == json)
  7472. throw createPTreeReadException(PTreeRead_syntax, "Null string passed to createJSONStringReader", NULL, 0, 0);
  7473. return new CJSONStringReader(json, iEvent, readerOptions);
  7474. }
  7475. IPTreeReader *createJSONBufferReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions)
  7476. {
  7477. class CJSONBufferReader : public CJSONReader<CInstBufferReader>
  7478. {
  7479. public:
  7480. CJSONBufferReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions) : CJSONReader<CInstBufferReader>(buf, bufLength, iEvent, readerOptions) { }
  7481. };
  7482. return new CJSONBufferReader(buf, bufLength, iEvent, readerOptions);
  7483. }
  7484. IPullPTreeReader *createPullJSONStreamReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions, size32_t bufSize)
  7485. {
  7486. class CJSONStreamReader : public CPullJSONReader<CInstStreamReader>
  7487. {
  7488. public:
  7489. CJSONStreamReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions, size32_t bufSize=0) : CPullJSONReader<CInstStreamReader>(stream, iEvent, readerOptions, bufSize) { }
  7490. };
  7491. return new CJSONStreamReader(stream, iEvent, readerOptions, bufSize);
  7492. }
  7493. IPullPTreeReader *createPullJSONStringReader(const char *json, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions)
  7494. {
  7495. class CJSONStringReader : public CPullJSONReader<CInstStringReader>
  7496. {
  7497. public:
  7498. CJSONStringReader(const void *json, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions) : CPullJSONReader<CInstStringReader>(json, iEvent, readerOptions) { }
  7499. };
  7500. return new CJSONStringReader(json, iEvent, readerOptions);
  7501. }
  7502. IPullPTreeReader *createPullJSONBufferReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions)
  7503. {
  7504. class CJSONBufferReader : public CPullJSONReader<CInstBufferReader>
  7505. {
  7506. public:
  7507. CJSONBufferReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions) : CPullJSONReader<CInstBufferReader>(buf, bufLength, iEvent, readerOptions) { }
  7508. };
  7509. return new CJSONBufferReader(buf, bufLength, iEvent, readerOptions);
  7510. }
  7511. IPropertyTree *createPTreeFromJSONString(const char *json, byte flags, PTreeReaderOptions readFlags, IPTreeMaker *iMaker)
  7512. {
  7513. Owned<IPTreeMaker> _iMaker;
  7514. if (!iMaker)
  7515. {
  7516. iMaker = createDefaultPTreeMaker(flags, readFlags);
  7517. _iMaker.setown(iMaker);
  7518. }
  7519. Owned<IPTreeReader> reader = createJSONStringReader(json, *iMaker, readFlags);
  7520. reader->load();
  7521. return LINK(iMaker->queryRoot());
  7522. }
  7523. IPropertyTree *createPTreeFromJSONString(unsigned len, const char *json, byte flags, PTreeReaderOptions readFlags, IPTreeMaker *iMaker)
  7524. {
  7525. Owned<IPTreeMaker> _iMaker;
  7526. if (!iMaker)
  7527. {
  7528. iMaker = createDefaultPTreeMaker(flags, readFlags);
  7529. _iMaker.setown(iMaker);
  7530. }
  7531. Owned<IPTreeReader> reader = createJSONBufferReader(json, len, *iMaker, readFlags);
  7532. reader->load();
  7533. return LINK(iMaker->queryRoot());
  7534. }
  7535. static const char * nextHttpParameterTag(StringBuffer &tag, const char *path)
  7536. {
  7537. while (*path=='.')
  7538. path++;
  7539. const char *finger = strchr(path, '.');
  7540. if (finger)
  7541. {
  7542. tag.clear().append(finger - path, path);
  7543. finger++;
  7544. }
  7545. else
  7546. tag.set(path);
  7547. return finger;
  7548. }
  7549. static void ensureHttpParameter(IPropertyTree *pt, StringBuffer &tag, const char *path, const char *value, const char *fullpath)
  7550. {
  7551. if (!tag.length())
  7552. return;
  7553. unsigned idx = 1;
  7554. if (path && isdigit(*path))
  7555. {
  7556. StringBuffer pos;
  7557. path = nextHttpParameterTag(pos, path);
  7558. idx = (unsigned) atoi(pos.str())+1;
  7559. }
  7560. if ('@'==*tag)
  7561. {
  7562. if (path && *path)
  7563. throw MakeStringException(-1, "'@' not allowed in parent node of parameter path: %s", fullpath);
  7564. pt->setProp(tag, value);
  7565. return;
  7566. }
  7567. if (tag.charAt(tag.length()-1)=='$')
  7568. {
  7569. if (path && *path)
  7570. throw MakeStringException(-1, "'$' not allowed in parent node of parameter path: %s", fullpath);
  7571. tag.setLength(tag.length()-1);
  7572. StringArray values;
  7573. values.appendList(value, "\r");
  7574. ForEachItemIn(pos, values)
  7575. {
  7576. const char *itemValue = values.item(pos);
  7577. while (*itemValue=='\n')
  7578. itemValue++;
  7579. pt->addProp(tag, itemValue);
  7580. }
  7581. return;
  7582. }
  7583. unsigned count = pt->getCount(tag);
  7584. while (count++ < idx)
  7585. pt->addPropTree(tag, createPTree(tag));
  7586. StringBuffer xpath(tag);
  7587. xpath.append('[').append(idx).append(']');
  7588. pt = pt->queryPropTree(xpath);
  7589. if (!path || !*path)
  7590. {
  7591. pt->setProp(NULL, value);
  7592. return;
  7593. }
  7594. StringBuffer nextTag;
  7595. path = nextHttpParameterTag(nextTag, path);
  7596. ensureHttpParameter(pt, nextTag, path, value, fullpath);
  7597. }
  7598. static void ensureHttpParameter(IPropertyTree *pt, const char *path, const char *value)
  7599. {
  7600. const char *fullpath = path;
  7601. StringBuffer tag;
  7602. path = nextHttpParameterTag(tag, path);
  7603. ensureHttpParameter(pt, tag, path, value, fullpath);
  7604. }
  7605. bool checkParseUrlPathNodeValue(const char *s, StringBuffer &name, StringAttr &value)
  7606. {
  7607. s = skipWhitespace(s);
  7608. const char *pn = strchr(s, '(');
  7609. if (pn) //strict format param('value') so we can extend later
  7610. {
  7611. const char *vp = pn + 1;
  7612. if (*vp!='\'')
  7613. return false;
  7614. const char *end =strchr(++vp, '\'');
  7615. if (!end || *(end+1)!=')')
  7616. return false;
  7617. if (!validateXMLTag(name.append(pn-s, s).trim()))
  7618. return false;
  7619. value.set(vp, end-vp);
  7620. }
  7621. else
  7622. {
  7623. if (!validateXMLTag(name.append(s).trim()))
  7624. return false;
  7625. }
  7626. return true;
  7627. }
  7628. IPropertyTree *createPTreeFromHttpPath(const char *nameWithAttrs, IPropertyTree *content, bool nestedRoot, ipt_flags flags)
  7629. {
  7630. StringArray nameAttrList;
  7631. nameAttrList.appendList(nameWithAttrs, "/");
  7632. if (!nameAttrList.ordinality())
  7633. return NULL;
  7634. Owned<IPropertyTree> pt = createPTree(nameAttrList.item(0), flags);
  7635. for (aindex_t pos=1; nameAttrList.isItem(pos); pos++)
  7636. {
  7637. StringBuffer name;
  7638. StringAttr value;
  7639. if (!checkParseUrlPathNodeValue(nameAttrList.item(pos), name, value))
  7640. throw MakeStringException(-1, "Invalid URL parameter format %s", nameAttrList.item(pos));
  7641. StringBuffer xpath("@");
  7642. xpath.append(name.str());
  7643. if (!value.get())
  7644. pt->setPropBool(xpath, true);
  7645. else
  7646. pt->setProp(xpath, value);
  7647. }
  7648. IPropertyTree *parent = pt;
  7649. const char *input = pt->queryProp("@input");
  7650. if (input)
  7651. {
  7652. StringArray inputNodes;
  7653. inputNodes.appendList(input, ".");
  7654. ForEachItemIn(in, inputNodes)
  7655. {
  7656. const char *tag = inputNodes.item(in);
  7657. if (!validateXMLTag(tag))
  7658. throw MakeStringException(-1, "Invalid REST query input specifier %s", input);
  7659. parent = parent->addPropTree(tag, createPTree(tag, flags));
  7660. }
  7661. }
  7662. if (streq("__array__", content->queryName()))
  7663. {
  7664. Owned<IAttributeIterator> aiter = content->getAttributes();
  7665. ForEach (*aiter)
  7666. parent->addProp(aiter->queryName(), aiter->queryValue());
  7667. Owned<IPropertyTreeIterator> iter = content->getElements("__item__");
  7668. ForEach (*iter)
  7669. {
  7670. IPropertyTree &e = iter->query();
  7671. e.renameProp("/", "Row");
  7672. parent->addPropTree("Row", LINK(&e));
  7673. }
  7674. }
  7675. else
  7676. mergePTree(parent, content);
  7677. if (nestedRoot)
  7678. {
  7679. Owned<IPropertyTree> root = createPTree(flags);
  7680. root->setPropTree(nameAttrList.item(0), pt.getClear());
  7681. return root.getClear();
  7682. }
  7683. return pt.getClear();
  7684. }
  7685. //URL node nameWithAttrs is of the form: "TagName/attr1('abc')/attr2/attr3('xyz')"
  7686. IPropertyTree *createPTreeFromHttpParameters(const char *nameWithAttrs, IProperties *parameters, bool skipLeadingDotParameters, bool nestedRoot, ipt_flags flags)
  7687. {
  7688. Owned<IPropertyTree> content = createPTree("content", flags);
  7689. Owned<IPropertyIterator> iter = parameters->getIterator();
  7690. ForEach(*iter)
  7691. {
  7692. StringBuffer key(iter->getPropKey());
  7693. if (!key.length() || key.charAt(key.length()-1)=='!')
  7694. continue;
  7695. if (skipLeadingDotParameters && key.charAt(0)=='.')
  7696. continue;
  7697. const char *value = parameters->queryProp(key);
  7698. if (!value || !*value)
  7699. continue;
  7700. ensureHttpParameter(content, key, value);
  7701. }
  7702. return createPTreeFromHttpPath(nameWithAttrs, content.getClear(), nestedRoot, flags);
  7703. }
  7704. IPropertyTree *createPTreeFromJSONFile(const char *filename, byte flags, PTreeReaderOptions readFlags, IPTreeMaker *iMaker)
  7705. {
  7706. Owned<IFile> in = createIFile(filename);
  7707. if (!in->exists())
  7708. return nullptr;
  7709. StringBuffer contents;
  7710. try
  7711. {
  7712. contents.loadFile(in);
  7713. }
  7714. catch (IException * e)
  7715. {
  7716. EXCLOG(e);
  7717. e->Release();
  7718. return nullptr;
  7719. }
  7720. return createPTreeFromJSONString(contents.length(), contents.str(), flags, readFlags, iMaker);
  7721. }
  7722. //---------------------------------------------------------------------------------------------------------------------
  7723. static constexpr const char * currentVersion = "1.0";
  7724. //---------------------------------------------------------------------------------------------------------------------
  7725. /*
  7726. * Use source to overwrite any changes in target
  7727. * Attributes are replaced
  7728. * Singleton elements are replaced.
  7729. * Entire arrays of scalar elements are replaced.
  7730. * Entire arrays of elements with no name attribute are replaced.
  7731. * Elements with a name attribute are matched by name. If there is a match it is merged. If there is no match it is added.
  7732. */
  7733. static bool checkInSequence(IPropertyTree & child, StringAttr &seqname, bool &first, bool &endprior)
  7734. {
  7735. first = false;
  7736. endprior = false;
  7737. if (seqname.length() && streq(seqname, child.queryName()))
  7738. return true;
  7739. endprior = !seqname.isEmpty();
  7740. if (child.isArray(nullptr))
  7741. {
  7742. first=true;
  7743. seqname.set(child.queryName());
  7744. return true;
  7745. }
  7746. seqname.clear();
  7747. return false;
  7748. }
  7749. inline bool isScalarItem(IPropertyTree &child)
  7750. {
  7751. if (child.hasChildren())
  7752. return false;
  7753. return child.getAttributeCount()==0;
  7754. }
  7755. static IPropertyTree *ensureMergeConfigTarget(IPropertyTree &target, const char *tag, const char *nameAttribute, const char *name, bool sequence)
  7756. {
  7757. StringBuffer tempPath;
  7758. const char * path = (sequence) ? nullptr : tag;
  7759. if (name && nameAttribute && *nameAttribute)
  7760. {
  7761. tempPath.append(tag).append("[").append(nameAttribute).append("=\'").append(name).append("']");
  7762. path = tempPath;
  7763. }
  7764. IPropertyTree * match = (path) ? target.queryPropTree(path) : nullptr;
  7765. if (!match)
  7766. {
  7767. if (sequence)
  7768. match = target.addPropTreeArrayItem(tag, createPTree(tag));
  7769. else
  7770. match = target.addPropTree(tag);
  7771. }
  7772. return match;
  7773. }
  7774. void mergeConfiguration(IPropertyTree & target, const IPropertyTree & source, const char *altNameAttribute, bool overwriteAttr)
  7775. {
  7776. Owned<IAttributeIterator> aiter = source.getAttributes();
  7777. ForEach(*aiter)
  7778. {
  7779. if (overwriteAttr || !target.hasProp(aiter->queryName()))
  7780. target.addProp(aiter->queryName(), aiter->queryValue());
  7781. }
  7782. StringAttr seqname;
  7783. Owned<IPropertyTreeIterator> iter = source.getElements("*");
  7784. ForEach(*iter)
  7785. {
  7786. IPropertyTree & child = iter->query();
  7787. const char * tag = child.queryName();
  7788. const char * name = child.queryProp("@name");
  7789. bool altname = false;
  7790. //Legacy support for old component configuration files that have repeated elements with no name tag but another unique id
  7791. if (!name && altNameAttribute && *altNameAttribute)
  7792. {
  7793. name = child.queryProp(altNameAttribute);
  7794. altname = name!=nullptr;
  7795. }
  7796. bool first = false;
  7797. bool endprior = false;
  7798. bool sequence = checkInSequence(child, seqname, first, endprior);
  7799. if (first && (!name || isScalarItem(child))) //arrays of unamed objects or scalars are replaced
  7800. target.removeProp(tag);
  7801. IPropertyTree * match = ensureMergeConfigTarget(target, tag, altname ? altNameAttribute : "@name", name, sequence);
  7802. mergeConfiguration(*match, child, altNameAttribute, overwriteAttr);
  7803. }
  7804. const char * sourceValue = source.queryProp("");
  7805. target.setProp("", sourceValue);
  7806. }
  7807. /*
  7808. * Load a json/yaml configuration file.
  7809. * If there is an extends tag in the root of the file then this file is applied as a delta to the base file
  7810. * the configuration is the contents of the tag within the file that matches the component tag.
  7811. */
  7812. static IPropertyTree * loadConfiguration(const char * filename, const char * componentTag, bool required, const char *altNameAttribute)
  7813. {
  7814. if (!checkFileExists(filename))
  7815. throw makeStringExceptionV(99, "Configuration file %s not found", filename);
  7816. const char * ext = pathExtension(filename);
  7817. Owned<IPropertyTree> configTree;
  7818. if (!ext || strieq(ext, ".yaml"))
  7819. {
  7820. try
  7821. {
  7822. configTree.setown(createPTreeFromYAMLFile(filename, 0, ptr_ignoreWhiteSpace, nullptr));
  7823. }
  7824. catch (IException *E)
  7825. {
  7826. StringBuffer msg;
  7827. E->errorMessage(msg);
  7828. ::Release(E);
  7829. throw makeStringExceptionV(99, "Error loading configuration file %s (invalid yaml): %s", filename, msg.str());
  7830. }
  7831. }
  7832. else
  7833. throw makeStringExceptionV(99, "Unrecognised file extension %s", ext);
  7834. if (!configTree)
  7835. throw makeStringExceptionV(99, "Error loading configuration file %s", filename);
  7836. IPropertyTree * config = configTree->queryPropTree(componentTag);
  7837. if (!config)
  7838. {
  7839. if (required)
  7840. throw makeStringExceptionV(99, "Section %s is missing from file %s", componentTag, filename);
  7841. return nullptr;
  7842. }
  7843. const char * base = configTree->queryProp("@extends");
  7844. if (!base)
  7845. return LINK(config);
  7846. StringBuffer baseFilename;
  7847. splitFilename(filename, &baseFilename, &baseFilename, nullptr, nullptr, false);
  7848. addNonEmptyPathSepChar(baseFilename);
  7849. baseFilename.append(base);
  7850. Owned<IPropertyTree> baseTree = loadConfiguration(baseFilename, componentTag, required, altNameAttribute);
  7851. mergeConfiguration(*baseTree, *config, altNameAttribute);
  7852. return LINK(baseTree);
  7853. }
  7854. static constexpr const char * envPrefix = "HPCC_CONFIG_";
  7855. static void applyEnvironmentConfig(IPropertyTree & target, const char * cptPrefix, const char * value)
  7856. {
  7857. const char * name = value;
  7858. if (!startsWith(name, envPrefix))
  7859. return;
  7860. name += strlen(envPrefix);
  7861. if (cptPrefix)
  7862. {
  7863. if (!startsWith(name, cptPrefix))
  7864. return;
  7865. name += strlen(cptPrefix);
  7866. if (*name++ != '_')
  7867. return;
  7868. }
  7869. StringBuffer propName;
  7870. if (startsWith(name, "PROP_"))
  7871. {
  7872. propName.append("@");
  7873. name += 5;
  7874. }
  7875. const char * eq = strchr(value, '=');
  7876. if (eq)
  7877. {
  7878. propName.append(eq - name, name);
  7879. target.setProp(propName, eq + 1);
  7880. }
  7881. else
  7882. {
  7883. propName.append(name);
  7884. target.setProp(propName, nullptr);
  7885. }
  7886. }
  7887. IPropertyTree * createPTreeFromYAML(const char * yaml)
  7888. {
  7889. if (*yaml == '{')
  7890. return createPTreeFromJSONString(yaml, 0, ptr_ignoreWhiteSpace, nullptr);
  7891. return createPTreeFromYAMLString(yaml, 0, ptr_ignoreWhiteSpace, nullptr);
  7892. }
  7893. static const char * extractOption(const char * option, const char * cur)
  7894. {
  7895. if (startsWith(cur, option))
  7896. {
  7897. cur += strlen(option);
  7898. if (*cur == '=')
  7899. return cur + 1;
  7900. if (*cur)
  7901. return nullptr;
  7902. return "1";
  7903. }
  7904. return nullptr;
  7905. }
  7906. static void applyCommandLineOption(IPropertyTree * config, const char * option, const char * value)
  7907. {
  7908. //Ignore -- with no following option.
  7909. if (isEmptyString(option))
  7910. return;
  7911. const char *tail;
  7912. while ((tail = strchr(option, '.')) != nullptr)
  7913. {
  7914. StringAttr elemName(option, tail-option);
  7915. if (!config->hasProp(elemName))
  7916. config = config->addPropTree(elemName);
  7917. else
  7918. {
  7919. config = config->queryPropTree(elemName);
  7920. if (!config)
  7921. throw makeStringExceptionV(99, "Cannot overriding scalar configuration element %s with structure", elemName.get());
  7922. }
  7923. option = tail+1;
  7924. }
  7925. if (!validateXMLTag(option))
  7926. throw makeStringExceptionV(99, "Invalid option name '%s'", option);
  7927. StringBuffer path;
  7928. path.append('@').append(option);
  7929. config->setProp(path, value);
  7930. }
  7931. static void applyCommandLineOption(IPropertyTree * config, const char * option, std::initializer_list<const char *> ignoreOptions)
  7932. {
  7933. const char * eq = strchr(option, '=');
  7934. StringBuffer name;
  7935. const char *val = nullptr;
  7936. if (eq)
  7937. {
  7938. name.append(eq - option, option);
  7939. option = name;
  7940. val = eq + 1;
  7941. }
  7942. else
  7943. {
  7944. //MORE: Support --x- and --x+?
  7945. val = "1";
  7946. }
  7947. if (stdContains(ignoreOptions, option))
  7948. return;
  7949. applyCommandLineOption(config, option, val);
  7950. }
  7951. static Owned<IPropertyTree> componentConfiguration;
  7952. static Owned<IPropertyTree> globalConfiguration;
  7953. MODULE_INIT(INIT_PRIORITY_STANDARD)
  7954. {
  7955. return true;
  7956. }
  7957. MODULE_EXIT()
  7958. {
  7959. componentConfiguration.clear();
  7960. globalConfiguration.clear();
  7961. }
  7962. IPropertyTree & queryComponentConfig()
  7963. {
  7964. if (!componentConfiguration)
  7965. throw makeStringException(99, "Configuration file has not yet been processed");
  7966. return *componentConfiguration;
  7967. }
  7968. IPropertyTree & queryGlobalConfig()
  7969. {
  7970. if (!globalConfiguration)
  7971. throw makeStringException(99, "Configuration file has not yet been processed");
  7972. return *globalConfiguration;
  7973. }
  7974. jlib_decl IPropertyTree * loadArgsIntoConfiguration(IPropertyTree *config, const char * * argv, std::initializer_list<const char *> ignoreOptions)
  7975. {
  7976. for (const char * * pArg = argv; *pArg; pArg++)
  7977. {
  7978. const char * cur = *pArg;
  7979. if (startsWith(cur, "--"))
  7980. applyCommandLineOption(config, cur + 2, ignoreOptions);
  7981. }
  7982. return config;
  7983. }
  7984. #ifdef _DEBUG
  7985. static void holdLoop()
  7986. {
  7987. DBGLOG("Component paused for debugging purposes, attach and set held=false to release");
  7988. bool held = true;
  7989. while (held)
  7990. Sleep(5);
  7991. }
  7992. #endif
  7993. jlib_decl IPropertyTree * loadConfiguration(IPropertyTree *componentDefault, const char * * argv, const char * componentTag, const char * envPrefix, const char * legacyFilename, IPropertyTree * (mapper)(IPropertyTree *), const char *altNameAttribute)
  7994. {
  7995. if (componentConfiguration)
  7996. throw makeStringExceptionV(99, "Configuration for component %s has already been initialised", componentTag);
  7997. Linked<IPropertyTree> config(componentDefault);
  7998. const char * optConfig = nullptr;
  7999. bool outputConfig = false;
  8000. #ifdef _DEBUG
  8001. bool held = false;
  8002. #endif
  8003. for (const char * * pArg = argv; *pArg; pArg++)
  8004. {
  8005. const char * cur = *pArg;
  8006. const char * matchConfig = extractOption("--config", cur);
  8007. if (matchConfig)
  8008. optConfig = matchConfig;
  8009. else if (strsame(cur, "--help"))
  8010. {
  8011. #if 0
  8012. //Better not to include this until it has been implemented, since it breaks eclcc
  8013. //MORE: displayHelp(config);
  8014. printf("%s <options>\n", argv[0]);
  8015. exit(0);
  8016. #endif
  8017. }
  8018. else if (strsame(cur, "--init"))
  8019. {
  8020. StringBuffer yamlText;
  8021. toYAML(componentDefault, yamlText, 0, YAML_SortTags);
  8022. printf("%s\n", yamlText.str());
  8023. exit(0);
  8024. }
  8025. else if (strsame(cur, "--outputconfig"))
  8026. {
  8027. outputConfig = true;
  8028. }
  8029. else
  8030. {
  8031. matchConfig = extractOption("--componentTag", cur);
  8032. if (matchConfig)
  8033. componentTag = matchConfig;
  8034. #ifdef _DEBUG
  8035. else
  8036. {
  8037. const char *matchHold = extractOption("--hold", cur);
  8038. if (matchHold)
  8039. {
  8040. if (strToBool(matchHold))
  8041. {
  8042. held = true;
  8043. holdLoop();
  8044. }
  8045. }
  8046. }
  8047. #endif
  8048. }
  8049. }
  8050. Owned<IPropertyTree> delta;
  8051. if (optConfig)
  8052. {
  8053. if (streq(optConfig, "1"))
  8054. throw makeStringExceptionV(99, "Name of configuration file omitted (use --config=<filename>)");
  8055. //--config= with no filename can be used to ignore the legacy configuration file
  8056. if (!isEmptyString(optConfig))
  8057. {
  8058. StringBuffer fullpath;
  8059. if (!isAbsolutePath(optConfig))
  8060. {
  8061. appendCurrentDirectory(fullpath, false);
  8062. addNonEmptyPathSepChar(fullpath);
  8063. }
  8064. fullpath.append(optConfig);
  8065. delta.setown(loadConfiguration(fullpath, componentTag, true, altNameAttribute));
  8066. globalConfiguration.setown(loadConfiguration(fullpath, "global", false, altNameAttribute));
  8067. }
  8068. }
  8069. else
  8070. {
  8071. if (legacyFilename && checkFileExists(legacyFilename))
  8072. delta.setown(createPTreeFromXMLFile(legacyFilename, ipt_caseInsensitive));
  8073. if (delta && mapper)
  8074. delta.setown(mapper(delta));
  8075. }
  8076. if (delta)
  8077. mergeConfiguration(*config, *delta, altNameAttribute);
  8078. const char * * environment = const_cast<const char * *>(getSystemEnv());
  8079. for (const char * * cur = environment; *cur; cur++)
  8080. {
  8081. applyEnvironmentConfig(*config, envPrefix, *cur);
  8082. }
  8083. if (outputConfig)
  8084. {
  8085. loadArgsIntoConfiguration(config, argv, { "config", "outputconfig" });
  8086. Owned<IPropertyTree> recreated = createPTree();
  8087. recreated->setProp("@version", currentVersion);
  8088. recreated->addPropTree(componentTag, LINK(config));
  8089. if (globalConfiguration)
  8090. recreated->addPropTree("global", globalConfiguration.getLink());
  8091. StringBuffer yamlText;
  8092. toYAML(recreated, yamlText, 0, YAML_SortTags);
  8093. printf("%s\n", yamlText.str());
  8094. exit(0);
  8095. }
  8096. else
  8097. loadArgsIntoConfiguration(config, argv);
  8098. //For legacy (and other weird cases) ensure there is a global section
  8099. if (!globalConfiguration)
  8100. globalConfiguration.setown(createPTree("global"));
  8101. #ifdef _DEBUG
  8102. // NB: don't re-hold, if CLI --hold already held.
  8103. if (!held && config->getPropBool("@hold"))
  8104. holdLoop();
  8105. #endif
  8106. unsigned ptreeMappingThreshold = globalConfiguration->getPropInt("@ptreeMappingThreshold", defaultSiblingMapThreshold);
  8107. setPTreeMappingThreshold(ptreeMappingThreshold);
  8108. componentConfiguration.set(config);
  8109. return config.getClear();
  8110. }
  8111. jlib_decl IPropertyTree * loadConfiguration(const char * defaultYaml, const char * * argv, const char * componentTag, const char * envPrefix, const char * legacyFilename, IPropertyTree * (mapper)(IPropertyTree *), const char *altNameAttribute)
  8112. {
  8113. if (componentConfiguration)
  8114. throw makeStringExceptionV(99, "Configuration for component %s has already been initialised", componentTag);
  8115. Owned<IPropertyTree> componentDefault;
  8116. if (defaultYaml)
  8117. {
  8118. Owned<IPropertyTree> defaultConfig = createPTreeFromYAML(defaultYaml);
  8119. componentDefault.set(defaultConfig->queryPropTree(componentTag));
  8120. if (!componentDefault)
  8121. throw makeStringExceptionV(99, "Default configuration does not contain the tag %s", componentTag);
  8122. }
  8123. else
  8124. componentDefault.setown(createPTree(componentTag));
  8125. return loadConfiguration(componentDefault, argv, componentTag, envPrefix, legacyFilename, mapper, altNameAttribute);
  8126. }
  8127. class CYAMLBufferReader : public CInterfaceOf<IPTreeReader>
  8128. {
  8129. protected:
  8130. Linked<IPTreeNotifyEvent> iEvent;
  8131. yaml_parser_t parser;
  8132. PTreeReaderOptions readerOptions = ptr_none;
  8133. bool noRoot = false;
  8134. public:
  8135. CYAMLBufferReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &_iEvent, PTreeReaderOptions _readerOptions) :
  8136. iEvent(&_iEvent), readerOptions(_readerOptions)
  8137. {
  8138. if (!yaml_parser_initialize(&parser))
  8139. throw makeStringException(99, "Filed to initialize libyaml parser");
  8140. yaml_parser_set_input_string(&parser, (const unsigned char *)buf, bufLength);
  8141. noRoot = 0 != ((unsigned)readerOptions & (unsigned)ptr_noRoot);
  8142. }
  8143. ~CYAMLBufferReader()
  8144. {
  8145. yaml_parser_delete(&parser);
  8146. }
  8147. yaml_event_type_t nextEvent(yaml_event_t &event, yaml_event_type_t final=YAML_NO_EVENT, yaml_event_type_t expected=YAML_NO_EVENT, const char *error="")
  8148. {
  8149. if (!yaml_parser_parse(&parser, &event))
  8150. throw makeStringExceptionV(99, "libyaml parser error %s", parser.problem);
  8151. if (event.type!=final && expected!=YAML_NO_EVENT && event.type!=expected)
  8152. throw makeStringExceptionV(99, "libyaml parser %s", error);
  8153. return event.type;
  8154. }
  8155. virtual void loadSequence(const char *tagname)
  8156. {
  8157. if (!tagname || !*tagname) //if unmapped (unnamed) sequences are possible have to decide how to name them in the ptree, later
  8158. throw makeStringException(99, "libyaml parser expected sequence name");
  8159. yaml_event_t event;
  8160. yaml_event_type_t eventType = YAML_NO_EVENT;
  8161. while (eventType!=YAML_SEQUENCE_END_EVENT)
  8162. {
  8163. eventType = nextEvent(event);
  8164. switch (eventType)
  8165. {
  8166. case YAML_MAPPING_START_EVENT: //child map
  8167. loadMap(tagname, true);
  8168. break;
  8169. case YAML_SEQUENCE_START_EVENT:
  8170. //todo
  8171. break;
  8172. case YAML_SCALAR_EVENT:
  8173. iEvent->beginNode(tagname, true, parser.offset);
  8174. iEvent->endNode(tagname, event.data.scalar.length, (const void *)event.data.scalar.value, false, parser.offset);
  8175. break;
  8176. case YAML_ALIAS_EVENT: //reference to an anchor, ignore for now
  8177. iEvent->beginNode(tagname, true, parser.offset);
  8178. iEvent->endNode(tagname, 0, nullptr, false, parser.offset);
  8179. break;
  8180. case YAML_SEQUENCE_END_EVENT: //done
  8181. break;
  8182. case YAML_NO_EVENT:
  8183. case YAML_MAPPING_END_EVENT:
  8184. case YAML_STREAM_START_EVENT:
  8185. case YAML_STREAM_END_EVENT:
  8186. case YAML_DOCUMENT_START_EVENT:
  8187. case YAML_DOCUMENT_END_EVENT:
  8188. default:
  8189. //shouldn't be here
  8190. break;
  8191. }
  8192. yaml_event_delete(&event);
  8193. }
  8194. }
  8195. virtual void loadMap(const char *tagname, bool sequence)
  8196. {
  8197. bool binaryContent = false;
  8198. StringBuffer content;
  8199. if (tagname && *tagname)
  8200. iEvent->beginNode(tagname, sequence, parser.offset);
  8201. yaml_event_t event;
  8202. yaml_event_type_t eventType = YAML_NO_EVENT;
  8203. while (eventType!=YAML_MAPPING_END_EVENT)
  8204. {
  8205. eventType = nextEvent(event, YAML_MAPPING_END_EVENT, YAML_SCALAR_EVENT, "expected map to start with scalar name");
  8206. if (eventType==YAML_MAPPING_END_EVENT)
  8207. {
  8208. yaml_event_delete(&event);
  8209. continue;
  8210. }
  8211. StringBuffer attname('@');
  8212. attname.append(event.data.scalar.length, (const char *)event.data.scalar.value);
  8213. const char *elname = attname.str()+1;
  8214. yaml_event_delete(&event);
  8215. eventType = nextEvent(event);
  8216. switch (eventType)
  8217. {
  8218. case YAML_MAPPING_START_EVENT: //child map
  8219. loadMap(elname, false);
  8220. break;
  8221. case YAML_SEQUENCE_START_EVENT:
  8222. loadSequence(elname);
  8223. break;
  8224. case YAML_SCALAR_EVENT:
  8225. {
  8226. //!el or !element will be our local tag (custom schema type) for an element
  8227. //ptree toYAML should set this for element scalars, and parent text content
  8228. const char *tag = (const char *)event.data.scalar.tag;
  8229. if (tag && (streq(tag, "!binary") || streq(tag, "!!binary")))
  8230. {
  8231. if (streq(elname, "^")) //text content of parent node
  8232. {
  8233. binaryContent = true;
  8234. JBASE64_Decode((const char *) event.data.scalar.value, content.clear());
  8235. }
  8236. else
  8237. {
  8238. StringBuffer decoded;
  8239. JBASE64_Decode((const char *) event.data.scalar.value, decoded);
  8240. iEvent->beginNode(elname, false, parser.offset);
  8241. iEvent->endNode(elname, decoded.length(), (const void *) decoded.str(), true, parser.offset);
  8242. }
  8243. }
  8244. else if (streq(elname, "^")) //text content of parent node
  8245. {
  8246. content.set((const char *) event.data.scalar.value);
  8247. }
  8248. else if (tag && (streq(tag, "!el") || streq(tag, "!element")))
  8249. {
  8250. iEvent->beginNode(elname, false, parser.offset);
  8251. iEvent->endNode(elname, event.data.scalar.length, (const void *) event.data.scalar.value, false, parser.offset);
  8252. }
  8253. else //by default all named scalars are ptree attributes
  8254. {
  8255. iEvent->newAttribute(attname, (const char *)event.data.scalar.value);
  8256. }
  8257. break;
  8258. }
  8259. case YAML_ALIAS_EVENT: //reference to an anchor, ignore for now
  8260. iEvent->beginNode(elname, false, parser.offset);
  8261. iEvent->endNode(elname, 0, nullptr, false, parser.offset);
  8262. break;
  8263. case YAML_MAPPING_END_EVENT: //done
  8264. break;
  8265. case YAML_NO_EVENT:
  8266. case YAML_SEQUENCE_END_EVENT:
  8267. case YAML_STREAM_START_EVENT:
  8268. case YAML_STREAM_END_EVENT:
  8269. case YAML_DOCUMENT_START_EVENT:
  8270. case YAML_DOCUMENT_END_EVENT:
  8271. default:
  8272. //shouldn't be here
  8273. break;
  8274. }
  8275. yaml_event_delete(&event);
  8276. }
  8277. if (tagname && *tagname)
  8278. iEvent->endNode(tagname, content.length(), content, binaryContent, parser.offset);
  8279. }
  8280. virtual void load() override
  8281. {
  8282. yaml_event_t event;
  8283. yaml_event_type_t eventType = YAML_NO_EVENT;
  8284. bool doc = false;
  8285. bool content = false;
  8286. while (eventType!=YAML_STREAM_END_EVENT)
  8287. {
  8288. eventType = nextEvent(event);
  8289. switch (eventType)
  8290. {
  8291. case YAML_MAPPING_START_EVENT:
  8292. //root content, the start of all mappings, should be only one at the root
  8293. if (content)
  8294. throw makeStringException(99, "YAML: Currently only support one content section (map) per stream");
  8295. loadMap(noRoot ? nullptr : "__object__", false); //root map
  8296. content=true;
  8297. break;
  8298. case YAML_SEQUENCE_START_EVENT:
  8299. //root content, sequence (array), should be only one at the root and can't mix with mappings
  8300. if (content)
  8301. throw makeStringException(99, "YAML: Currently only support one content section (sequence) per stream");
  8302. if (!noRoot)
  8303. iEvent->beginNode("__array__", false, 0);
  8304. loadSequence("__item__");
  8305. if (!noRoot)
  8306. iEvent->endNode("__array__", 0, nullptr, false, parser.offset);
  8307. content=true;
  8308. break;
  8309. case YAML_STREAM_START_EVENT:
  8310. case YAML_STREAM_END_EVENT:
  8311. //don't think we need to do anything... unless we start saving hints
  8312. break;
  8313. case YAML_DOCUMENT_START_EVENT:
  8314. //should only support one? multiple documents would imply an extra level of nesting (future flag?)
  8315. if (doc)
  8316. throw makeStringException(99, "YAML: Currently only support one document per stream");
  8317. doc=true;
  8318. break;
  8319. case YAML_DOCUMENT_END_EVENT:
  8320. break;
  8321. case YAML_NO_EVENT:
  8322. case YAML_ALIAS_EVENT: //root alias?
  8323. case YAML_MAPPING_END_EVENT:
  8324. case YAML_SCALAR_EVENT: //root unmapped (unnamed) scalars?
  8325. case YAML_SEQUENCE_END_EVENT:
  8326. //shouldn't be here
  8327. break;
  8328. default:
  8329. break;
  8330. }
  8331. yaml_event_delete(&event);
  8332. }
  8333. }
  8334. virtual offset_t queryOffset() override
  8335. {
  8336. return parser.offset;
  8337. }
  8338. };
  8339. IPTreeReader *createYAMLBufferReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions)
  8340. {
  8341. return new CYAMLBufferReader(buf, bufLength, iEvent, readerOptions);
  8342. }
  8343. IPropertyTree *createPTreeFromYAMLString(unsigned len, const char *yaml, byte flags, PTreeReaderOptions readFlags, IPTreeMaker *iMaker)
  8344. {
  8345. Owned<IPTreeMaker> _iMaker;
  8346. if (!iMaker)
  8347. {
  8348. iMaker = createDefaultPTreeMaker(flags, readFlags);
  8349. _iMaker.setown(iMaker);
  8350. }
  8351. Owned<IPTreeReader> reader = createYAMLBufferReader(yaml, len, *iMaker, readFlags);
  8352. reader->load();
  8353. return LINK(iMaker->queryRoot());
  8354. }
  8355. IPropertyTree *createPTreeFromYAMLString(const char *yaml, byte flags, PTreeReaderOptions readFlags, IPTreeMaker *iMaker)
  8356. {
  8357. return createPTreeFromYAMLString(strlen(yaml), yaml, flags, readFlags, iMaker);
  8358. }
  8359. IPropertyTree *createPTreeFromYAMLFile(const char *filename, byte flags, PTreeReaderOptions readFlags, IPTreeMaker *iMaker)
  8360. {
  8361. Owned<IFile> in = createIFile(filename);
  8362. if (!in->exists())
  8363. return nullptr;
  8364. StringBuffer contents;
  8365. try
  8366. {
  8367. contents.loadFile(in);
  8368. }
  8369. catch (IException * e)
  8370. {
  8371. EXCLOG(e);
  8372. e->Release();
  8373. return nullptr;
  8374. }
  8375. return createPTreeFromYAMLString(contents.length(), contents, flags, readFlags, iMaker);
  8376. }
  8377. static int yaml_write_iiostream(void *data, unsigned char *buffer, size_t size)
  8378. {
  8379. IIOStream *out = (IIOStream *) data;
  8380. out->write(size, (void *)buffer);
  8381. out->flush();
  8382. return 1;
  8383. }
  8384. class YAMLEmitter
  8385. {
  8386. yaml_emitter_t emitter;
  8387. yaml_event_t event;
  8388. IIOStream &out;
  8389. public:
  8390. YAMLEmitter(IIOStream &ios, int indent) : out(ios)
  8391. {
  8392. if (!yaml_emitter_initialize(&emitter))
  8393. throw MakeStringException(0, "YAMLEmitter: failed to initialize");
  8394. yaml_emitter_set_output(&emitter, yaml_write_iiostream, &out);
  8395. yaml_emitter_set_canonical(&emitter, false);
  8396. yaml_emitter_set_unicode(&emitter, true);
  8397. yaml_emitter_set_indent(&emitter, indent);
  8398. beginStream();
  8399. beginDocument();
  8400. }
  8401. ~YAMLEmitter()
  8402. {
  8403. endDocument();
  8404. endStream();
  8405. yaml_emitter_delete(&emitter);
  8406. }
  8407. yaml_char_t *getTag(bool binary, bool element)
  8408. {
  8409. if (binary)
  8410. return (yaml_char_t *) "!binary";
  8411. if (element)
  8412. return (yaml_char_t *) "!el";
  8413. return nullptr;
  8414. }
  8415. void emit()
  8416. {
  8417. yaml_emitter_emit(&emitter, &event);
  8418. }
  8419. void checkInit(int success, const char *descr)
  8420. {
  8421. if (success==0)
  8422. throw MakeStringException(0, "YAMLEmitter: %s failed", descr);
  8423. }
  8424. void writeValue(const char *value, bool element, bool hidden, bool binary)
  8425. {
  8426. yaml_scalar_style_t style = binary ? YAML_LITERAL_SCALAR_STYLE : YAML_ANY_SCALAR_STYLE;
  8427. const yaml_char_t *tag = getTag(binary, element);
  8428. bool implicit = tag==nullptr;
  8429. StringBuffer s;
  8430. if (!value)
  8431. value = "null";
  8432. else if (hidden)
  8433. value = (binary) ? "KioqKg==" : s.appendN(strlen(value), '*').str(); //KioqKg== is base64 of ****
  8434. checkInit(yaml_scalar_event_initialize(&event, nullptr, tag, (yaml_char_t *) value, -1, implicit, implicit, style), "yaml_scalar_event_initialize");
  8435. emit();
  8436. }
  8437. void writeName(const char *name)
  8438. {
  8439. dbgassertex(name!=nullptr);
  8440. return writeValue(name, false, false,false);
  8441. }
  8442. void writeNamedValue(const char *name, const char *value, bool element, bool hidden)
  8443. {
  8444. writeName(name);
  8445. writeValue(value, element, hidden, false);
  8446. }
  8447. void writeAttribute(const char *name, const char *value, bool hidden)
  8448. {
  8449. writeNamedValue(name, value, false, hidden);
  8450. }
  8451. void beginMap()
  8452. {
  8453. checkInit(yaml_mapping_start_event_initialize(&event, nullptr, nullptr, 0, YAML_BLOCK_MAPPING_STYLE), "yaml_mapping_start_event_initialize");
  8454. emit();
  8455. }
  8456. void endMap()
  8457. {
  8458. checkInit(yaml_mapping_end_event_initialize(&event), "yaml_mapping_end_event_initialize");
  8459. emit();
  8460. }
  8461. void beginSequence(const char *name)
  8462. {
  8463. if (name)
  8464. writeName(name);
  8465. checkInit(yaml_sequence_start_event_initialize(&event, nullptr, nullptr, 0, YAML_ANY_SEQUENCE_STYLE), "yaml_sequence_start_event_initialize");
  8466. emit();
  8467. }
  8468. void endSequence()
  8469. {
  8470. checkInit(yaml_sequence_end_event_initialize(&event), "yaml_sequence_end_event_initialize");
  8471. emit();
  8472. }
  8473. void beginDocument()
  8474. {
  8475. checkInit(yaml_document_start_event_initialize(&event, nullptr, nullptr, nullptr, true), "yaml_document_start_event_initialize");
  8476. emit();
  8477. }
  8478. void endDocument()
  8479. {
  8480. checkInit(yaml_document_end_event_initialize(&event, true), "yaml_document_end_event_initialize");
  8481. emit();
  8482. }
  8483. void beginStream()
  8484. {
  8485. checkInit(yaml_stream_start_event_initialize(&event, YAML_UTF8_ENCODING), "yaml_stream_start_event_initialize");
  8486. emit();
  8487. }
  8488. void endStream()
  8489. {
  8490. checkInit(yaml_stream_end_event_initialize(&event), "yaml_stream_end_event_initialize");
  8491. emit();
  8492. }
  8493. };
  8494. static void _toYAML(const IPropertyTree *tree, YAMLEmitter &yaml, byte flags, bool root=false, bool isArrayItem=false)
  8495. {
  8496. const char *name = tree->queryName();
  8497. if (!root && !isArrayItem)
  8498. {
  8499. if (!name || !*name)
  8500. name = "__unnamed__";
  8501. yaml.writeName(name);
  8502. }
  8503. Owned<IAttributeIterator> it = tree->getAttributes(true);
  8504. bool hasAttributes = it->first();
  8505. bool complex = (hasAttributes || tree->hasChildren());
  8506. bool hiddenRootArrayObject = isRootArrayObjectHidden(root, name, flags);
  8507. if (!hiddenRootArrayObject)
  8508. {
  8509. if (complex)
  8510. yaml.beginMap();
  8511. if (hasAttributes)
  8512. {
  8513. ForEach(*it)
  8514. {
  8515. const char *key = it->queryName()+1;
  8516. const char *val = it->queryValue();
  8517. yaml.writeAttribute(key, val, isSanitizedAndHidden(val, flags, true));
  8518. }
  8519. }
  8520. }
  8521. StringBuffer _content;
  8522. const char *content = nullptr; // to avoid uninitialized warning
  8523. bool isBinary = tree->isBinary(NULL);
  8524. bool isNull = true;
  8525. if (!hiddenRootArrayObject)
  8526. {
  8527. if (isBinary)
  8528. {
  8529. MemoryBuffer thislevelbin;
  8530. isNull = (!tree->getPropBin(NULL, thislevelbin))||(thislevelbin.length()==0);
  8531. if (!isNull)
  8532. JBASE64_Encode(thislevelbin.toByteArray(), thislevelbin.length(), _content, true);
  8533. content = _content.str();
  8534. }
  8535. else if (tree->isCompressed(NULL))
  8536. {
  8537. isNull = false; // can't be empty if compressed;
  8538. verifyex(tree->getProp(NULL, _content));
  8539. content = _content.str();
  8540. }
  8541. else
  8542. isNull = (NULL == (content = tree->queryProp(NULL)));
  8543. if (isNull && !root && !complex)
  8544. {
  8545. yaml.writeValue("null", false, false, false);
  8546. return;
  8547. }
  8548. }
  8549. Owned<IPropertyTreeIterator> sub = tree->getElements(hiddenRootArrayObject ? "__item__" : "*", 0 != (flags & YAML_SortTags) ? iptiter_sort : iptiter_null);
  8550. //note that detection of repeating elements relies on the fact that ptree elements
  8551. //of the same name will be grouped together
  8552. StringAttr seqname;
  8553. bool sequence = false;
  8554. ForEach(*sub)
  8555. {
  8556. IPropertyTree &element = sub->query();
  8557. bool first = false;
  8558. bool endprior = false;
  8559. sequence = checkInSequence(element, seqname, first, endprior);
  8560. if (endprior)
  8561. yaml.endSequence();
  8562. if (first)
  8563. yaml.beginSequence(hiddenRootArrayObject ? nullptr : element.queryName());
  8564. _toYAML(&element, yaml, flags, false, sequence);
  8565. }
  8566. if (sequence)
  8567. yaml.endSequence();
  8568. if (!isNull)
  8569. {
  8570. if (complex)
  8571. yaml.writeName("^");
  8572. //repeating/array/sequence items are implicitly elements, no need for tag
  8573. yaml.writeValue(content, isArrayItem ? false : true, isSanitizedAndHidden(content, flags, false), isBinary);
  8574. }
  8575. if (!hiddenRootArrayObject && complex)
  8576. yaml.endMap();
  8577. }
  8578. static void _toYAML(const IPropertyTree *tree, IIOStream &out, unsigned indent, byte flags, bool root=false, bool isArrayItem=false)
  8579. {
  8580. YAMLEmitter yaml(out, indent);
  8581. _toYAML(tree, yaml, flags, true, false);
  8582. }
  8583. jlib_decl StringBuffer &toYAML(const IPropertyTree *tree, StringBuffer &ret, unsigned indent, byte flags)
  8584. {
  8585. CStringBufferMarkupIOAdapter adapter(ret);
  8586. _toYAML(tree->queryBranch(NULL), adapter, indent, flags, true);
  8587. return ret;
  8588. }
  8589. void toYAML(const IPropertyTree *tree, IIOStream &out, unsigned indent, byte flags)
  8590. {
  8591. _toYAML(tree, out, indent, flags, true);
  8592. }
  8593. void printYAML(const IPropertyTree *tree, unsigned indent, unsigned flags)
  8594. {
  8595. StringBuffer yaml;
  8596. toYAML(tree, yaml, indent, flags);
  8597. printf("%s", yaml.str());
  8598. }
  8599. void dbglogYAML(const IPropertyTree *tree, unsigned indent, unsigned flags)
  8600. {
  8601. StringBuffer yaml;
  8602. toYAML(tree, yaml, indent, flags);
  8603. DBGLOG("%s", yaml.str());
  8604. }
  8605. void saveYAML(const char *filename, const IPropertyTree *tree, unsigned indent, unsigned flags)
  8606. {
  8607. OwnedIFile ifile = createIFile(filename);
  8608. saveYAML(*ifile, tree, indent, flags);
  8609. }
  8610. void saveYAML(IFile &ifile, const IPropertyTree *tree, unsigned indent, unsigned flags)
  8611. {
  8612. OwnedIFileIO ifileio = ifile.open(IFOcreate);
  8613. if (!ifileio)
  8614. throw MakeStringException(0, "saveXML: could not find %s to open", ifile.queryFilename());
  8615. saveYAML(*ifileio, tree, indent, flags);
  8616. }
  8617. void saveYAML(IFileIO &ifileio, const IPropertyTree *tree, unsigned indent, unsigned flags)
  8618. {
  8619. Owned<IIOStream> stream = createIOStream(&ifileio);
  8620. stream.setown(createBufferedIOStream(stream));
  8621. saveYAML(*stream, tree, indent, flags);
  8622. }
  8623. void saveYAML(IIOStream &stream, const IPropertyTree *tree, unsigned indent, unsigned flags)
  8624. {
  8625. toYAML(tree, stream, indent, flags);
  8626. }
  8627. jlib_decl IPropertyTree * queryCostsConfiguration()
  8628. {
  8629. return queryComponentConfig().queryPropTree("costs");
  8630. }
  8631. void copyPropIfMissing(IPropertyTree & target, const char * targetName, IPropertyTree & source, const char * sourceName)
  8632. {
  8633. if (source.hasProp(sourceName) && !target.hasProp(targetName))
  8634. {
  8635. if (source.isBinary(sourceName))
  8636. {
  8637. MemoryBuffer value;
  8638. source.getPropBin(sourceName, value);
  8639. target.setPropBin(targetName, value.length(), value.toByteArray());
  8640. }
  8641. else
  8642. target.setProp(targetName, source.queryProp(sourceName));
  8643. }
  8644. }