123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753 |
- /*##############################################################################
- HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- ############################################################################## */
- #include "jmisc.hpp"
- #include "jfile.hpp"
- #include "jiter.ipp"
- #include "jexcept.hpp"
- #include "jmutex.hpp"
- #include "jsort.hpp"
- #include "jutil.hpp"
- #include "hql.hpp"
- #include "hqlexpr.ipp"
- #include "hqlgram.hpp"
- #include "hqlfold.hpp"
- #include "hqlthql.hpp"
- #include "hqlpmap.hpp"
- #include <math.h>
- #include "hqlerrors.hpp"
- #include "hqlerror.hpp"
- #include "hqlplugins.hpp"
- #include "hqltrans.ipp"
- #include "hqlutil.hpp"
- #include "eclrtl.hpp"
- #include "hqlattr.hpp"
- #include "hqlmeta.hpp"
- static CriticalSection * attributeCS;
- MODULE_INIT(INIT_PRIORITY_HQLINTERNAL)
- {
- attributeCS = new CriticalSection;
- return true;
- }
- MODULE_EXIT()
- {
- delete attributeCS;
- }
- // This file should contain most of the derived attribute calculation for nodes in the expression tree,
- // Other candidates are
- // checkConstant, getChilddatasetType(), getNumChildTables
- // queryHasRows, definesColumnList(), queryTransformIndex
- // initFlagsBefore(), updatFlagsAfter()
- // getCachedEclCRC(), cacheTablesUsed(), isIndependentOfScope()
- // logic inside createDataset
- //Originally the idea was to have a class instance for each kind of opcode, and to call opcode[op]->evaluateAttrXXXXXX(this);
- //to evaluate the attribute. However because there are so many opcodes I'm not convinced this is the best way.
- //Better may be to model it more on the way queryRecordCount() is implemented.
- //This switch statement provides an entry of each opcode grouped according to its function.
- //within each group try and maintain alphabetical ordering
- unsigned getOperatorMetaFlags(node_operator op)
- {
- switch (op)
- {
- case no_none:
- case no_nobody:
- //Records/types
- case no_field:
- case no_record:
- case no_type:
- case no_ifblock:
- case no_enum:
- case no_selfref:
- case no_typedef:
- //Simple arithmetic expressions with no children:
- case no_constant:
- case no_variable:
- case no_quoted: // codegen only
- case no_getresult:
- case no_matched:
- case no_matchtext:
- case no_matchlength:
- case no_matchposition:
- case no_failcode:
- case no_failmessage:
- case no_id2blob:
- case no_blob2id:
- case no_clustersize:
- case no_loopcounter:
- case no_callback:
- case no_assertwild:
- case no_eventname:
- case no_eventextra:
- case no_debug_option_value:
- //Arithmetic operators
- case no_mul:
- case no_div:
- case no_modulus:
- case no_negate:
- case no_add:
- case no_sub:
- case no_exp:
- case no_power:
- case no_round:
- case no_roundup:
- case no_ln:
- case no_log10:
- case no_sin:
- case no_cos:
- case no_tan:
- case no_asin:
- case no_acos:
- case no_atan:
- case no_atan2:
- case no_sinh:
- case no_cosh:
- case no_tanh:
- case no_sqrt:
- case no_truncate:
- case no_cast:
- case no_implicitcast:
- case no_abs:
- case no_charlen:
- case no_sizeof:
- case no_offsetof:
- case no_nameof:
- case no_band:
- case no_bor:
- case no_bxor:
- case no_bnot:
- case no_order: //?? also a comparison
- case no_rank:
- case no_ranked:
- case no_hash:
- case no_typetransfer:
- case no_lshift:
- case no_rshift:
- case no_crc:
- case no_random:
- case no_counter:
- case no_address:
- case no_hash32:
- case no_hash64:
- case no_wuid:
- case no_countdict:
- case no_existslist:
- case no_countlist:
- case no_maxlist:
- case no_minlist:
- case no_sumlist:
- case no_unicodeorder:
- case no_assertkeyed:
- case no_hashmd5:
- case no_pure:
- case no_sequence:
- case no_getenv:
-
- //Selection operators - could be arithmetic, string, dataset etc.
- case no_map:
- case no_if:
- case no_choose:
- case no_which:
- case no_rejected:
- case no_mapto:
- case no_case:
- //String operators:
- case no_concat:
- case no_substring:
- case no_asstring:
- case no_intformat:
- case no_realformat:
- case no_trim:
- case no_fromunicode:
- case no_tounicode:
- case no_keyunicode:
- case no_rowdiff:
- case no_xmltext:
- case no_xmlunicode:
- case no_xmldecode:
- case no_xmlencode:
- case no_matchunicode:
- case no_matchutf8:
- case no_regex_find:
- case no_regex_replace:
- case no_toxml:
- //Boolean operators:
- case no_eq:
- case no_ne:
- case no_lt:
- case no_le:
- case no_gt:
- case no_ge:
- case no_not:
- case no_notnot:
- case no_and:
- case no_or:
- case no_xor:
- case no_notin:
- case no_in:
- case no_notbetween:
- case no_between:
- case no_is_valid:
- case no_indict:
- //Lists/Sets etc.
- case no_list:
- case no_all:
- case no_addsets:
- case no_createset:
- case no_rowset:
- case no_rowsetindex:
- case no_rowsetrange:
- case no_sortlist:
- case no_recordlist:
- case no_datasetlist:
- case no_transformlist:
- //Aggregate operators
- case no_count:
- case no_exists:
- case no_existsdict:
- case no_max:
- case no_min:
- case no_sum:
- case no_ave:
- case no_variance:
- case no_covariance:
- case no_correlation:
- case no_countgroup:
- case no_existsgroup:
- case no_maxgroup:
- case no_mingroup:
- case no_sumgroup:
- case no_avegroup:
- case no_vargroup:
- case no_covargroup:
- case no_corrgroup:
- case no_within:
- case no_notwithin:
- case no_countcompare:
- //Selectors
- case no_left:
- case no_right:
- case no_self:
- case no_activetable:
- case no_activerow:
- case no_top:
- //Transforms
- case no_transform:
- case no_assign:
- case no_assignall:
- case no_newtransform:
- //Rows
- case no_selectmap:
- case no_selectnth:
- case no_matchrow:
- case no_matchattr: // and scalar
- case no_projectrow:
- case no_createrow:
- case no_newrow:
- case no_temprow:
- //Dictionaries
- case no_createdictionary:
- //Datasets [see also selection operators]
- case no_rollup:
- case no_iterate:
- case no_hqlproject:
- case no_group:
- case no_cogroup:
- case no_cosort:
- case no_index:
- case no_table:
- case no_keyindex:
- case no_temptable:
- case no_usertable:
- case no_choosen:
- case no_filter:
- case no_fetch:
- case no_join:
- case no_sort:
- case no_subsort:
- case no_sorted:
- case no_dedup:
- case no_enth:
- case no_sample:
- case no_selectfields:
- case no_addfiles:
- case no_distribute:
- case no_normalize:
- case no_distributed:
- case no_preservemeta:
- case no_grouped:
- case no_denormalize:
- case no_newusertable:
- case no_newaggregate:
- case no_aggregate:
- case no_choosesets:
- case no_workunit_dataset:
- case no_split:
- case no_spill:
- case no_readspill:
- case no_writespill:
- case no_commonspill:
- case no_parse:
- case no_newparse:
- case no_throughaggregate:
- case no_compound_diskread:
- case no_compound_disknormalize:
- case no_compound_diskaggregate:
- case no_compound_diskcount:
- case no_compound_diskgroupaggregate:
- case no_compound_indexread:
- case no_compound_indexnormalize:
- case no_compound_indexaggregate:
- case no_compound_indexcount:
- case no_compound_indexgroupaggregate:
- case no_compound_childread:
- case no_compound_childnormalize:
- case no_compound_childaggregate:
- case no_compound_childcount:
- case no_compound_childgroupaggregate:
- case no_compound_inline:
- case no_getgraphresult:
- case no_compound_fetch:
- case no_topn:
- case no_newxmlparse:
- case no_httpcall:
- case no_soapcall:
- case no_soapcall_ds:
- case no_newsoapcall:
- case no_newsoapcall_ds:
- case no_nonempty:
- case no_filtergroup:
- case no_limit:
- case no_catchds:
- case no_loop:
- case no_forcenolocal:
- case no_allnodes:
- case no_selfjoin:
- case no_process:
- case no_thisnode:
- case no_getgraphloopresult:
- case no_graphloop:
- case no_assertstepped:
- case no_assertsorted:
- case no_assertgrouped:
- case no_assertdistributed:
- case no_mergejoin:
- case no_nwayjoin:
- case no_nwaymerge:
- case no_stepped:
- case no_datasetfromrow:
- case no_datasetfromdictionary:
- case no_assert_ds:
- case no_combine:
- case no_rollupgroup:
- case no_regroup:
- case no_combinegroup:
- case no_inlinetable:
- case no_denormalizegroup:
- case no_xmlproject:
- case no_spillgraphresult:
- case no_rows:
- case no_keyedlimit:
- case no_compound_selectnew:
- case no_getgraphloopresultset:
- case no_preload:
- case no_merge:
- case no_keyeddistribute:
- case no_newkeyindex:
- case no_anon:
- case no_pseudods:
- case no_deserialize:
- case no_serialize:
- case no_forcegraph:
- case no_related:
- case no_executewhen:
- case no_callsideeffect:
- case no_fromxml:
- case no_xmlparse:
- case no_normalizegroup:
- case no_owned_ds:
- case no_dataset_alias:
- case no_chooseds:
- //Multiple different kinds of values
- case no_select:
- case no_indirect:
- case no_selectindirect:
- case no_null:
- case no_globalscope:
- case no_nothor:
- case no_embedbody:
- case no_alias_scope:
- case no_evalonce:
- case no_forcelocal:
- case no_cluster:
- //Parser only - not in normalized expression trees
- case no_evaluate:
- case no_macro:
- case no_transformebcdic:
- case no_transformascii:
- case no_metaactivity:
- case no_loadxml:
- case no_fieldmap:
- case no_template_context:
- case no_processing:
- case no_merge_pending:
- case no_merge_nomatch:
- case no_namedactual:
- case no_assertconstant:
- case no_assertconcrete:
- case no_delayedscope:
- //Code generator only - only created once code is being generated.
- case no_postinc:
- case no_postdec:
- case no_preinc:
- case no_predec:
- case no_pselect:
- case no_deref:
- case no_ordered:
- case no_decimalstack:
- case no_translated:
- case no_filepos:
- case no_file_logicalname:
- case no_reference:
- case no_assign_addfiles:
- case no_nullptr:
- case no_childquery:
- //Workflow
- case no_stored:
- case no_failure:
- case no_success:
- case no_recovery:
- case no_wait:
- case no_event:
- case no_persist:
- case no_when:
- case no_setconditioncode:
- case no_priority:
- case no_colon:
- case no_setworkflow_cond:
- case no_global:
- case no_workflow:
- case no_workflow_action:
- case no_checkpoint:
- case no_define:
- case no_independent:
- case no_catch:
- case no_once:
- //Patterns
- case no_pat_select:
- case no_pat_const:
- case no_pat_pattern:
- case no_pat_follow:
- case no_pat_first:
- case no_pat_last:
- case no_pat_repeat:
- case no_pat_instance:
- case no_pat_anychar:
- case no_pat_token:
- case no_pat_imptoken:
- case no_pat_set:
- case no_pat_checkin:
- case no_pat_x_before_y:
- case no_pat_x_after_y:
- case no_pat_index:
- case no_pat_beginpattern:
- case no_pat_endpattern:
- case no_pat_checklength:
- case no_pat_featureparam:
- case no_pat_featureactual:
- case no_pat_featuredef:
- case no_pat_validate:
- case no_pat_use:
- case no_featuretype:
- case no_pat_guard:
- case no_penalty:
- case no_pat_case:
- case no_pat_nocase:
- case no_pat_before_y:
- case no_pat_after_y:
- case no_pat_production:
- case no_pat_or:
- //Pseudo-Attributes
- case no_csv:
- case no_sql:
- case no_thor:
- case no_flat:
- case no_pipe:
- case no_joined:
- case no_any:
- case no_xml:
- case no_distributer:
- case no_keyed:
- case no_sortpartition:
- //Multiple types
- case no_outofline:
- case no_create_initializer:
- //Actions
- case no_buildindex:
- case no_output:
- case no_apply:
- case no_fail:
- case no_distribution:
- case no_ensureresult:
- case no_setresult:
- case no_sequential:
- case no_parallel:
- case no_actionlist:
- case no_soapaction_ds:
- case no_newsoapaction_ds:
- case no_keydiff:
- case no_keypatch:
- case no_returnresult:
- case no_outputscalar:
- case no_evaluate_stmt:
- case no_return_stmt:
- case no_setgraphloopresult:
- case no_skip:
- case no_assert:
- case no_notify:
- case no_setgraphresult:
- case no_extractresult:
- case no_unused81:
- case no_definesideeffect:
- // Scopes etc.
- case no_scope:
- case no_forwardscope:
- case no_remotescope:
- case no_privatescope:
- case no_virtualscope:
- case no_concretescope:
- case no_mergedscope:
- //Used for representing functional attributes
- case no_service:
- case no_external:
- case no_funcdef:
- case no_externalcall:
- case no_libraryselect:
- case no_bound_func:
- case no_purevirtual:
- case no_internalselect:
- case no_delayedselect:
- case no_unboundselect:
- case no_libraryscope:
- case no_libraryscopeinstance:
- case no_libraryinput:
- case no_call:
- case no_attrname:
- // Other
- case no_comma:
- case no_uncommoned_comma:
- case no_compound:
- case no_param:
- case no_setmeta:
- case no_omitted:
- case no_range:
- case no_rangeto:
- case no_rangefrom:
- case no_rangecommon:
- case no_nofold:
- case no_nohoist:
- case no_section:
- case no_sectioninput:
- case no_alias:
- case no_unknown: // used for callbacks
- case no_attr:
- case no_attr_link:
- case no_attr_expr:
- case no_cachealias:
- case no_subgraph:
- case no_rowvalue:
- case no_loopbody:
- case no_complex:
- //Not implemented anywhere:
- case no_impure: // not really used
- case no_dependenton:
- case no_alias_project:
- case no_nolink:
- case no_joincount:
- case no_guard:
- case no_hint:
- case no_cloned:
- case no_childdataset:
- case no_envsymbol:
- case no_bound_type:
- case no_mix:
- case no_persist_check:
- case no_dataset_from_transform:
- case no_id:
- case no_unused6:
- case no_unused13: case no_unused14: case no_unused15:
- case no_unused24: case no_unused25: case no_unused28: case no_unused29:
- case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38:
- case no_unused40: case no_unused41: case no_unused42: case no_unused43: case no_unused44: case no_unused45: case no_unused46: case no_unused47: case no_unused48: case no_unused49:
- case no_unused50: case no_unused52:
- case no_unused80: case no_unused83:
- case no_unused101: case no_unused102:
- case no_is_null:
- case no_position:
- case no_current_time:
- case no_current_date:
- case no_current_timestamp:
- case no_update:
- //The following never get created IHqlExpressions, they are used as constants in the PARSE internal structures.
- case no_pat_compound:
- case no_pat_begintoken:
- case no_pat_endtoken:
- case no_pat_begincheck:
- case no_pat_endcheckin:
- case no_pat_endchecklength:
- case no_pat_beginseparator:
- case no_pat_endseparator:
- case no_pat_separator:
- case no_pat_beginvalidate:
- case no_pat_endvalidate:
- case no_pat_dfa:
- case no_pat_singlechar:
- case no_pat_beginrecursive:
- case no_pat_endrecursive:
- case no_pat_utf8single:
- case no_pat_utf8lead:
- case no_pat_utf8follow:
- case no_eclcrc:
- return 0;
- case no_isomitted:
- return 0;
- default:
- DBGLOG("**** Missing meta flags for operator %d ***", (int)op);
- return 0;
- }
- }
- //---------------------------------------------------------------------------------
- inline unsigned truncMaxlength(unsigned __int64 value)
- {
- return (value > MAX_MAXLENGTH) ? MAX_MAXLENGTH : (unsigned)value;
- }
- static unsigned getMaxSize(ITypeInfo * type, IHqlExpression * maxLength, IHqlExpression * maxSize, IHqlExpression * maxCount)
- {
- unsigned size = type->getSize();
- if (size != UNKNOWN_LENGTH)
- return size;
- if (!maxLength) maxLength = queryPropertyChild(type, maxLengthAtom, 0);
- if (!maxSize) maxSize = queryPropertyChild(type, maxSizeAtom, 0);
- if (!maxCount) maxCount = queryPropertyChild(type, maxCountAtom, 0);
- if (maxSize)
- return (unsigned)getIntValue(maxSize, 0);
- if (maxLength)
- {
- unsigned __int64 len = (unsigned)getIntValue(maxLength, 0);
- switch (type->getTypeCode())
- {
- case type_string:
- case type_data:
- return truncMaxlength(sizeof(size32_t) + len);
- case type_unicode:
- return truncMaxlength(sizeof(size32_t) + len*sizeof(UChar));
- case type_qstring:
- return truncMaxlength(sizeof(size32_t) + rtlQStrSize((unsigned)len));
- case type_varstring:
- return truncMaxlength(len + 1);
- case type_varunicode:
- return truncMaxlength((len + 1) * sizeof(UChar));
- case type_utf8:
- return truncMaxlength(sizeof(size32_t) + (len * 4));
- case type_set:
- return truncMaxlength(len);
- }
- }
- if (maxCount)
- {
- unsigned __int64 count = getIntValue(maxCount, 0);
- switch (type->getTypeCode())
- {
- case type_set:
- {
- ITypeInfo * childType = type->queryChildType();
- if (!childType)
- break;
- unsigned elemSize = getMaxSize(childType, NULL, NULL, NULL);
- if (elemSize != UNKNOWN_LENGTH)
- return truncMaxlength(sizeof(bool) + sizeof(size32_t) + count * elemSize);
- break;
- }
- }
- }
- return UNKNOWN_LENGTH;
- }
- static unsigned getMaxSize(IHqlExpression * field)
- {
- switch (field->getOperator())
- {
- case no_select:
- return getMaxSize(field->queryChild(1));
- case no_indirect:
- return getMaxSize(field->queryChild(0));
- }
- ITypeInfo * type = field->queryType();
- IHqlExpression * maxLength = queryPropertyChild(field, maxLengthAtom, 0);
- IHqlExpression * maxSize = queryPropertyChild(field, maxSizeAtom, 0);
- IHqlExpression * maxCount = queryPropertyChild(field, maxCountAtom, 0);
- unsigned max = getMaxSize(type, maxLength, maxSize, maxCount);
- if (max != UNKNOWN_LENGTH)
- return max;
- ITypeInfo * indirect = queryModifier(type, typemod_indirect);
- if (indirect)
- {
- IHqlExpression * original = static_cast<IHqlExpression *>(indirect->queryModifierExtra());
- return getMaxSize(original);
- }
- return max;
- }
- //Some arbitrary guess at the size of a variable length string field.
- static double twoThirds = 2.0/3.0;
- static unsigned guessSize(unsigned minLen, unsigned maxLen)
- {
- if (maxLen == UNKNOWN_LENGTH)
- maxLen = 4096;
- if (maxLen < minLen)
- return minLen;
- double value = pow((double)(maxLen-minLen), twoThirds);
- return truncMaxlength(minLen + (unsigned __int64)ceil(value));
- }
- static IHqlExpression * querySerializedForm(IHqlExpression * expr, IAtom * variation)
- {
- if (expr)
- {
- ExprPropKind kind;
- if (variation == diskAtom)
- kind = EPdiskserializedForm;
- else if (variation == internalAtom)
- kind = EPinternalserializedForm;
- else
- throwUnexpected();
- IHqlExpression * attr = expr->queryAttribute(kind);
- if (attr)
- return attr;
- }
- return expr;
- }
- static HqlTransformerInfo serializedRecordCreatorInfo("SerializedRecordCreator");
- class SerializedRecordCreator : public QuickHqlTransformer
- {
- public:
- SerializedRecordCreator(IAtom * _variety) : QuickHqlTransformer(serializedRecordCreatorInfo, NULL), variety(_variety) {}
- virtual IHqlExpression * createTransformedBody(IHqlExpression * expr)
- {
- switch (expr->getOperator())
- {
- case no_field:
- {
- if (expr->hasProperty(_linkCounted_Atom))
- {
- OwnedHqlExpr transformed = QuickHqlTransformer::createTransformedBody(expr);
- return removeProperty(transformed, _linkCounted_Atom);
- }
- break;
- }
- }
- return QuickHqlTransformer::createTransformedBody(expr);
- }
- virtual ITypeInfo * transformType(ITypeInfo * type)
- {
- Owned<ITypeInfo> transformed = QuickHqlTransformer::transformType(type);
- return getSerializedForm(transformed, variety);
- }
- protected:
- IAtom * variety;
- };
- static IHqlExpression * evaluateSerializedRecord(IHqlExpression * expr, IAtom * variation)
- {
- SerializedRecordCreator transformer(variation);
- return transformer.transform(expr);
- }
- //---------------------------------------------------------------------------------
- class CHqlExprMeta
- {
- public:
- static inline IHqlExpression * addAttribute(IHqlExpression * expr, ExprPropKind kind, IHqlExpression * value)
- {
- CHqlExpression * cexpr = static_cast<CHqlExpression *>(expr);
- cexpr->addAttribute(kind, value);
- return value;
- }
- static inline IHqlExpression * queryExistingAttribute(IHqlExpression * expr, ExprPropKind kind)
- {
- CHqlExpression * cexpr = static_cast<CHqlExpression *>(expr);
- return cexpr->queryExistingAttribute(kind);
- }
- } meta;
- //-- Attribute: serialized form -------------------------------------------------------------------------------
- static IHqlExpression * evaluateAttrSerializedForm(IHqlExpression * expr, ExprPropKind kind, IAtom * variation)
- {
- if (expr->getOperator() == no_record || expr->getOperator() == no_field)
- {
- OwnedHqlExpr serialized = evaluateSerializedRecord(expr, variation);
- if (serialized != expr)
- {
- //Tag serialized form so don't re-evaluated
- meta.addAttribute(serialized, kind, serialized);
- }
- return meta.addAttribute(expr, kind, serialized);
- }
- return NULL;
- }
- //-- Attribute: size -------------------------------------------------------------------------------
- //no_field
- static IHqlExpression * evaluateFieldAttrSize(IHqlExpression * expr)
- {
- ITypeInfo * type = expr->queryType();
- unsigned minSize = UNKNOWN_LENGTH;
- unsigned maxSize = 0;
- unsigned thisSize = type->getSize();
- OwnedHqlExpr thisMaxSizeExpr;
- if (expr->hasProperty(_isBlobInIndex_Atom))
- {
- thisSize = sizeof(unsigned __int64);
- }
- else
- {
- switch (type->getTypeCode())
- {
- case type_bitfield:
- {
- thisSize = type->queryChildType()->getSize();
- break;
- }
- case type_record:
- case type_row:
- {
- if (hasReferenceModifier(type))
- thisSize = sizeof(void *);
- else
- {
- IHqlExpression * ret = expr->queryRecord()->queryAttribute(EPsize);
- return meta.addAttribute(expr, EPsize, ret);
- }
- break;
- }
- case type_dictionary:
- case type_groupedtable:
- case type_table:
- {
- if (expr->hasProperty(_linkCounted_Atom))
- {
- thisSize = sizeof(size32_t) + sizeof(byte * *);
- break;
- }
- IHqlExpression * count = NULL;
- IHqlExpression * size = NULL;
- IHqlExpression * maxLength = NULL;
- IHqlExpression * maxCount = NULL;
- ForEachChild(i, expr)
- {
- IHqlExpression * attr = expr->queryChild(i);
- if (attr->isAttribute())
- {
- IAtom * name = attr->queryName();
- if (name == countAtom)
- count = attr->queryChild(0);
- else if (name == sizeofAtom)
- size = attr->queryChild(0);
- else if (name == maxLengthAtom)
- maxLength = attr->queryChild(0);
- else if (name == maxSizeAtom)
- maxLength = attr->queryChild(0);
- else if (name == maxCountAtom)
- maxCount = attr->queryChild(0);
- else if ((name == choosenAtom) && attr->queryChild(0)->queryValue())
- maxCount = attr->queryChild(0);
- }
- }
- IHqlExpression * record = expr->queryRecord();
- IHqlExpression * childRecordSizeExpr = record->queryAttribute(EPsize);
- unsigned childExpectedSize = (unsigned)getIntValue(childRecordSizeExpr->queryChild(0));
- unsigned childMinimumSize = (unsigned)getIntValue(childRecordSizeExpr->queryChild(1));
- IHqlExpression * childMaximumSizeExpr = childRecordSizeExpr->queryChild(2);
- unsigned childMaximumSize = (unsigned)getIntValue(childMaximumSizeExpr, UNKNOWN_LENGTH);
- ITypeInfo * sizetType = childMaximumSizeExpr->queryType();
- if (count || size)
- {
- minSize = 0;
- if (size && size->queryValue())
- thisSize = (unsigned)getIntValue(size);
- else if (count && count->queryValue())
- {
- unsigned __int64 num = (unsigned)getIntValue(count);
- thisSize = truncMaxlength(num * childExpectedSize);
- minSize = truncMaxlength(num * childMinimumSize);
- if (childMaximumSize != UNKNOWN_LENGTH)
- maxSize = truncMaxlength(num * childMaximumSize);
- else
- thisMaxSizeExpr.setown(createValue(no_mul, LINK(sizetType), ensureExprType(count, sizetType), LINK(childMaximumSizeExpr)));
- }
- else
- {
- thisSize = UNKNOWN_LENGTH;
- if (maxLength)
- maxSize = (unsigned)getIntValue(maxLength);
- else if (maxCount)
- {
- if (childMaximumSize != UNKNOWN_LENGTH)
- maxSize = truncMaxlength((unsigned __int64)getIntValue(maxCount) * childMaximumSize);
- else
- thisMaxSizeExpr.setown(createValue(no_mul, LINK(sizetType), ensureExprType(maxCount, sizetType), LINK(childMaximumSizeExpr)));
- }
- else
- maxSize = UNKNOWN_LENGTH;
- }
- }
- else
- {
- minSize = sizeof(size32_t);
- if (maxLength)
- maxSize = (unsigned)getIntValue(maxLength);
- else if (maxCount)
- {
- if (childMaximumSize != UNKNOWN_LENGTH)
- maxSize = truncMaxlength(sizeof(size32_t) + (unsigned __int64)getIntValue(maxCount) * childMaximumSize);
- else
- thisMaxSizeExpr.setown(createValue(no_add, LINK(sizetType), getSizetConstant(sizeof(size32_t)),
- createValue(no_mul, LINK(sizetType), ensureExprType(maxCount, sizetType), LINK(childMaximumSizeExpr))));
- }
- else
- maxSize = UNKNOWN_LENGTH;
- }
- break;
- }
- case type_string:
- case type_data:
- case type_unicode:
- case type_qstring:
- case type_utf8:
- if (thisSize == UNKNOWN_LENGTH)
- {
- minSize = sizeof(size32_t);
- maxSize = getMaxSize(expr);
- }
- break;
- case type_varstring:
- if (thisSize == UNKNOWN_LENGTH)
- {
- minSize = 1;
- maxSize = getMaxSize(expr);
- }
- break;
- case type_varunicode:
- if (thisSize == UNKNOWN_LENGTH)
- {
- minSize = sizeof(UChar);
- maxSize = getMaxSize(expr);
- }
- break;
- case type_set:
- if (thisSize == UNKNOWN_LENGTH)
- {
- minSize = sizeof(size32_t)+sizeof(bool);
- maxSize = getMaxSize(expr);
- }
- break;
- case type_alien:
- {
- IHqlAlienTypeInfo * alien = queryAlienType(type);
- thisSize = alien->getPhysicalTypeSize();
-
- if (thisSize == UNKNOWN_LENGTH)
- {
- IHqlExpression * lengthAttr = queryUncastExpr(alien->queryLengthFunction());
- if (lengthAttr->isConstant() && !lengthAttr->isFunction())
- {
- OwnedHqlExpr folded = foldHqlExpression(lengthAttr);
- if (folded->queryValue())
- thisSize = (unsigned)getIntValue(folded);
- }
- }
- if (thisSize == UNKNOWN_LENGTH)
- {
- minSize = 0;
- IHqlExpression * maxSizeExpr = expr->queryProperty(maxSizeAtom);
- if (!maxSizeExpr)
- maxSizeExpr = expr->queryProperty(maxLengthAtom);
- if (maxSizeExpr)
- maxSize = (unsigned)getIntValue(maxSizeExpr->queryChild(0));
- else
- maxSize = alien->getMaxSize();
- }
- break;
- }
- case type_packedint:
- minSize = 1;
- maxSize = (type->queryPromotedType()->getSize()+1);
- thisSize = (maxSize > 2) ? 2 : 1;
- break;
- case type_any:
- minSize = 1;
- maxSize = getMaxSize(expr);
- break;
- default:
- assertex(thisSize != UNKNOWN_LENGTH);
- break;
- }
- }
- if (thisMaxSizeExpr)
- maxSize = UNKNOWN_LENGTH;
- if (thisSize == UNKNOWN_LENGTH)
- thisSize = guessSize(minSize, maxSize);
- else
- {
- if (minSize == UNKNOWN_LENGTH)
- minSize = thisSize;
- if (maxSize == 0)
- maxSize = thisSize;
- }
- if ((thisSize == minSize) && (minSize == maxSize))
- {
- OwnedHqlExpr attr = getFixedSizeAttr(thisSize);
- return meta.addAttribute(expr, EPsize, attr);
- }
- if (!thisMaxSizeExpr)
- thisMaxSizeExpr.setown((maxSize == UNKNOWN_LENGTH) ? createAttribute(unknownSizeFieldAtom) : getSizetConstant(maxSize));
- OwnedHqlExpr attr = createExprAttribute(_propSize_Atom, getSizetConstant(thisSize), getSizetConstant(minSize), thisMaxSizeExpr.getClear());
- return meta.addAttribute(expr, EPsize, attr);
- }
- //no_ifblock
- static IHqlExpression * evaluateIfBlockAttrSize(IHqlExpression * expr)
- {
- IHqlExpression * size = expr->queryChild(1)->queryAttribute(EPsize);
- unsigned averageSize = (unsigned)getIntValue(size->queryChild(0), 0)/2;
- OwnedHqlExpr attr = createExprAttribute(_propSize_Atom, getSizetConstant(averageSize), getSizetConstant(0), LINK(size->queryChild(2)));
- return meta.addAttribute(expr, EPsize, attr);
- }
- //no_record
- static IHqlExpression * evaluateRecordAttrSize(IHqlExpression * expr)
- {
- unsigned __int64 expectedSize = 0;
- unsigned __int64 minimumSize = 0;
- unsigned __int64 maximumSize = 0;
- OwnedHqlExpr maximumSizeExpr;
- OwnedHqlExpr derivedSizeExpr;
- bool hasUnknownMaxSizeField = false;
- BitfieldPacker packer;
- ForEachChild(i, expr)
- {
- IHqlExpression * cur = expr->queryChild(i);
- ITypeInfo * type = cur->queryType();
- if (type && type->getTypeCode() == type_bitfield)
- {
- unsigned thisBitOffset, thisBits;
- if (!packer.checkSpaceAvailable(thisBitOffset, thisBits, type))
- {
- size32_t thisSize = type->queryChildType()->getSize();
- expectedSize += thisSize;
- minimumSize += thisSize;
- maximumSize += thisSize;
- }
- }
- else
- {
- packer.reset();
- IHqlExpression * size = cur->queryAttribute(EPsize);
- if (size)
- {
- expectedSize += (size32_t)getIntValue(size->queryChild(0));
- minimumSize += (size32_t)getIntValue(size->queryChild(1));
- IHqlExpression * maxExpr = size->queryChild(2);
- if (maxExpr->queryValue())
- maximumSize += (size32_t)getIntValue(maxExpr);
- else if (maxExpr->isAttribute())
- {
- assertex(maxExpr->queryName() == unknownSizeFieldAtom);
- hasUnknownMaxSizeField = true;
- }
- else
- extendAdd(maximumSizeExpr, maxExpr);
- }
- }
- }
- if ((minimumSize != maximumSize) || maximumSizeExpr || hasUnknownMaxSizeField)
- {
- IHqlExpression * maxLength = queryPropertyChild(expr, maxLengthAtom, 0);
- if (maxLength)
- {
- if (!hasUnknownMaxSizeField)
- {
- if (maximumSize || !maximumSizeExpr)
- {
- OwnedHqlExpr maxExpr = getSizetConstant(truncMaxlength(maximumSize));
- extendAdd(maximumSizeExpr, maxExpr);
- }
- derivedSizeExpr.set(maximumSizeExpr);
- }
- maximumSize = (unsigned)getIntValue(maxLength, UNKNOWN_LENGTH);
- maximumSizeExpr.clear();
- if (derivedSizeExpr)
- {
- //If not a constant then it is derived from the default maxlength, so the explicit maxlength is better
- //otherwise use the minimum value
- if (derivedSizeExpr->queryValue())
- {
- unsigned maxDerived = (unsigned)getIntValue(derivedSizeExpr);
- if (maximumSize > maxDerived)
- maximumSize = maxDerived;
- }
- }
- }
- else if (hasUnknownMaxSizeField)
- {
- OwnedHqlExpr maxExpr = getSizetConstant(truncMaxlength(maximumSize));
- extendAdd(maximumSizeExpr, maxExpr);
- maximumSize = 0;
- //?create an expression to represent
- // if (totalSize * 2 > defaultMaxRecordSize, totalSize + defaultMaxRecordSize / 2, defaultMaxRecordSize);
- IHqlExpression * defaultMaxLength = queryDefaultMaxRecordLengthExpr();
- OwnedHqlExpr minmax = LINK(maximumSizeExpr);
- OwnedHqlExpr minMaxTimes2 = createValue(no_mul, defaultMaxLength->getType(), LINK(minmax), getSizetConstant(2));
- OwnedHqlExpr cond = createBoolExpr(no_gt, LINK(minMaxTimes2), LINK(defaultMaxLength));
- OwnedHqlExpr trueExpr = createValue(no_add, defaultMaxLength->getType(), LINK(minmax), createValue(no_div, defaultMaxLength->getType(), LINK(defaultMaxLength), getSizetConstant(2)));
- maximumSizeExpr.setown(createValue(no_if, defaultMaxLength->getType(), LINK(cond), LINK(trueExpr), LINK(defaultMaxLength)));
- }
- }
- if ((maximumSize == 0) && expr->hasProperty(_nonEmpty_Atom))
- {
- expectedSize = 1;
- minimumSize = 1;
- maximumSize = 1;
- }
- if (maximumSize || !maximumSizeExpr)
- {
- OwnedHqlExpr maxExpr = getSizetConstant(truncMaxlength(maximumSize));
- extendAdd(maximumSizeExpr, maxExpr);
- }
- HqlExprArray args;
- args.append(*getSizetConstant(truncMaxlength(expectedSize)));
- args.append(*getSizetConstant(truncMaxlength(minimumSize)));
- args.append(*LINK(maximumSizeExpr));
- if (derivedSizeExpr)
- args.append(*LINK(derivedSizeExpr));
- OwnedHqlExpr sizeAttr = createExprAttribute(_propSize_Atom, args);
- return meta.addAttribute(expr, EPsize, sizeAttr);
- }
- static IHqlExpression * evaluateAttrSize(IHqlExpression * expr)
- {
- switch (expr->getOperator())
- {
- case no_field:
- return evaluateFieldAttrSize(expr);
- case no_ifblock:
- return evaluateIfBlockAttrSize(expr);
- case no_record:
- return evaluateRecordAttrSize(expr);
- case no_transform:
- case no_newtransform:
- {
- //MORE: This could calculate a better estimate for the size of the record by taking into account any constant values or datasets that are assigned.
- IHqlExpression * record = expr->queryRecord();
- IHqlExpression * recordSize = record->queryAttribute(EPsize);
- return meta.addAttribute(expr, EPsize, recordSize);
- }
- }
- IHqlExpression * record = expr->queryRecord();
- if (record)
- return record->queryAttribute(EPsize);
- return NULL;
- }
- IHqlExpression * getSerializedForm(IHqlExpression * expr, IAtom * variation)
- {
- return LINK(querySerializedForm(expr, variation));
- }
- ITypeInfo * getSerializedForm(ITypeInfo * type, IAtom * variation)
- {
- if (!type)
- return NULL;
- switch (type->getTypeCode())
- {
- case type_record:
- {
- IHqlExpression * record = queryRecord(queryUnqualifiedType(type));
- IHqlExpression * serializedRecord = querySerializedForm(record, variation);
- if (record == serializedRecord)
- return LINK(type);
- return cloneModifiers(type, serializedRecord->queryType());
- }
- case type_row:
- case type_transform:
- case type_table:
- case type_groupedtable:
- {
- //MORE: If (variant == internalAtom) consider using a format that prefixes the dataset with a count instead of a size
- OwnedITypeInfo noOutOfLineType = removeModifier(type, typemod_outofline);
- OwnedITypeInfo noLinkCountType = removeProperty(noOutOfLineType, _linkCounted_Atom);
- ITypeInfo * childType = noLinkCountType->queryChildType();
- OwnedITypeInfo newChild = getSerializedForm(childType, variation);
- return replaceChildType(noLinkCountType, newChild);
- }
- case type_dictionary:
- {
- OwnedITypeInfo noOutOfLineType = removeModifier(type, typemod_outofline);
- OwnedITypeInfo noLinkCountType = removeProperty(noOutOfLineType, _linkCounted_Atom);
- ITypeInfo * childType = noLinkCountType->queryChildType();
- OwnedITypeInfo newChild = getSerializedForm(childType, variation);
- if (variation == internalAtom)
- return replaceChildType(noLinkCountType, newChild);
- OwnedITypeInfo datasetType = makeTableType(LINK(newChild), NULL, NULL, NULL);
- return cloneModifiers(noLinkCountType, datasetType);
- }
- }
- return LINK(type);
- }
- //-- Attribute: unadorned form (no annotations) -------------------------------------------------------------------------------
- //Use a transformer to implement the mapping - since it contains the logic for processing types etc, but use the attributes as an extra cache.
- class HqlCachedAttributeTransformer : public QuickHqlTransformer
- {
- public:
- HqlCachedAttributeTransformer(HqlTransformerInfo & _transformInfo, ExprPropKind _propKind);
- virtual IHqlExpression * transform(IHqlExpression * expr);
- protected:
- ExprPropKind propKind;
- };
- HqlCachedAttributeTransformer::HqlCachedAttributeTransformer(HqlTransformerInfo & _transformInfo, ExprPropKind _propKind)
- : QuickHqlTransformer(_transformInfo, NULL), propKind(_propKind)
- {
- }
- IHqlExpression * HqlCachedAttributeTransformer::transform(IHqlExpression * expr)
- {
- IHqlExpression * match = meta.queryExistingAttribute(expr, propKind);
- if (match)
- return LINK(match);
- OwnedHqlExpr transformed = QuickHqlTransformer::transform(expr);
- if (transformed != expr)
- {
- //Tag serialized form so don't re-evaluate
- meta.addAttribute(transformed, propKind, transformed);
- }
- meta.addAttribute(expr, propKind, transformed);
- return transformed.getClear();
- }
- class HqlUnadornedNormalizer : public HqlCachedAttributeTransformer
- {
- public:
- HqlUnadornedNormalizer();
- virtual IHqlExpression * createTransformed(IHqlExpression * expr);
- virtual ITypeInfo * transformType(ITypeInfo * type);
- };
- static HqlTransformerInfo hqlUnadornedInfo("HqlUnadornedNormalizer");
- HqlUnadornedNormalizer::HqlUnadornedNormalizer() : HqlCachedAttributeTransformer(hqlUnadornedInfo, EPunadorned)
- {
- }
- ITypeInfo * HqlUnadornedNormalizer::transformType(ITypeInfo * type)
- {
- return HqlCachedAttributeTransformer::transformType(queryUnqualifiedType(type));
- }
- IHqlExpression * HqlUnadornedNormalizer::createTransformed(IHqlExpression * expr)
- {
- IHqlExpression * body = expr->queryBody(false);
- if (expr != body)
- return transform(body);
- node_operator op = expr->getOperator();
- switch (op)
- {
- case no_field:
- {
- //Remove the default values...
- HqlExprArray children;
- bool same = true;
- ForEachChild(idx, expr)
- {
- IHqlExpression * cur = expr->queryChild(idx);
- if (cur->isAttribute())
- {
- IHqlExpression * mapped = transform(cur);
- children.append(*mapped);
- if (mapped != cur)
- same = false;
- }
- else
- same = false;
- }
- ITypeInfo * type = expr->queryType();
- OwnedITypeInfo newType = transformType(type);
- IIdAtom * id = expr->queryId();
- //Fields names compare case-insignificantly therefore the field name is converted to lower case so that
- //equivalent fields are mapped to the same normalized expression.
- IIdAtom * newid = createIdAtom(id->lower()->str());
- if ((type != newType) || (id != newid))
- return createField(newid, newType.getClear(), children);
- if (same)
- return LINK(expr);
- return expr->clone(children);
- }
- case no_param:
- {
- ITypeInfo * type = expr->queryType();
- OwnedITypeInfo newType = transformType(type);
- HqlExprArray children;
- transformChildren(expr, children); // could just unwind
- return createParameter(expr->queryId(), UnadornedParameterIndex, newType.getClear(), children);
- }
- }
- return HqlCachedAttributeTransformer::createTransformed(expr);
- }
- static IHqlExpression * evaluateAttrUnadorned(IHqlExpression * expr)
- {
- HqlUnadornedNormalizer normalizer;
- //NB: Also has the side-effect of adding any missing attributes
- OwnedHqlExpr dummy = normalizer.transform(expr);
- return meta.queryExistingAttribute(expr, EPunadorned);
- }
- //---------------------------------------------------------------------------------
- static unsigned queryFieldAlignment(ITypeInfo * type)
- {
- unsigned size = type->getSize();
- type_t tc = type->getTypeCode();
- switch (tc)
- {
- case type_int:
- case type_swapint:
- switch (size)
- {
- case 2:
- case 4:
- case 8:
- return size;
- }
- return 1;
- case type_boolean:
- case type_real:
- return size;
- case type_string:
- case type_varstring:
- case type_data:
- if (hasLinkCountedModifier(type))
- return sizeof(void *);
- return 1;
- case type_char:
- case type_decimal:
- case type_packedint:
- case type_set: // would be nicer if properly aligned. Even nicer if the (isall,size) were separate
- case type_utf8:
- case type_qstring:
- case type_any:
- return 1;
- case type_bitfield:
- case type_array:
- case type_enumerated:
- return queryFieldAlignment(type->queryChildType());
- case type_pointer:
- return sizeof(void *);
- case type_table:
- case type_row:
- case type_groupedtable:
- if (hasLinkCountedModifier(type))
- return sizeof(void *);
- return 1;
- case type_alien:
- return queryFieldAlignment(type->queryChildType());
- case type_unicode:
- case type_varunicode:
- return sizeof(UChar);
- default:
- throwUnexpectedType(type);
- }
- }
- static unsigned queryFieldAlignment(IHqlExpression * field)
- {
- return queryFieldAlignment(field->queryType());
- }
- class FieldAlignCompare : public ICompare
- {
- public:
- FieldAlignCompare(const HqlExprCopyArray & _original) : original(_original) {}
- virtual int docompare(const void * inleft, const void * inright) const
- {
- IInterface * pleft = (IInterface *)(inleft);
- IInterface * pright = (IInterface *)(inright);
- IHqlExpression * left = static_cast<IHqlExpression *>(pleft);
- IHqlExpression * right = static_cast<IHqlExpression *>(pright);
- IHqlExpression * leftSizeAttr = left->queryAttribute(EPsize);
- IHqlExpression * rightSizeAttr = right->queryAttribute(EPsize);
- bool leftIsFixedSize = leftSizeAttr->queryChild(1) == leftSizeAttr->queryChild(2);
- bool rightIsFixedSize = rightSizeAttr->queryChild(1) == rightSizeAttr->queryChild(2);
- if (leftIsFixedSize && rightIsFixedSize)
- {
- bool leftIsBitfield = (left->queryType()->getTypeCode() == type_bitfield);
- bool rightIsBitfield = (right->queryType()->getTypeCode() == type_bitfield);
- if (!leftIsBitfield && !rightIsBitfield)
- {
- //First choose the largest alignment first
- unsigned leftAlign = queryFieldAlignment(left);
- unsigned rightAlign = queryFieldAlignment(right);
- if (leftAlign != rightAlign)
- return (int)(rightAlign - leftAlign);
- #if 0
- //Then choose smallest item next - so access is more compact
- unsigned leftSize = getIntValue(leftSizeAttr->queryChild(0));
- unsigned rightSize = getIntValue(rightSizeAttr->queryChild(0));
- if (leftSize != rightSize)
- return (int)(leftSize - rightSize);
- #endif
- //fall through to default
- }
- else if (!leftIsBitfield)
- return -1;
- else if (!rightIsBitfield)
- return +1;
- else
- {
- //Two bitfields - need better handling
- }
- }
- else if (leftIsFixedSize)
- return -1;
- else if (rightIsFixedSize)
- return +1;
- else
- {
- //both variable size
- }
- //default processing currently by name - may change to use original order
- return original.find(*left) - original.find(*right);
- // return stricmp(left->queryName()->str(), right->queryName()->str());
- }
- protected:
- const HqlExprCopyArray & original;
- } ;
- static bool optimizeFieldOrder(HqlExprArray & out, const HqlExprCopyArray & in)
- {
- HqlExprCopyArray sorted;
- appendArray(sorted, in);
- FieldAlignCompare compare(in);
- qsortvec((void * *)sorted.getArray(), sorted.ordinality(), compare);
- ForEachItemIn(i, sorted)
- out.append(OLINK(sorted.item(i)));
- return true;
- }
- static IHqlExpression * evaluateAttrAligned(IHqlExpression * expr)
- {
- bool same = true;
- HqlExprArray result;
- HqlExprCopyArray reorder;
- assertex(expr->getOperator() == no_record);
- ForEachChild(i, expr)
- {
- IHqlExpression * cur = expr->queryChild(i);
- switch (cur->getOperator())
- {
- case no_field:
- reorder.append(*cur);
- break;
- case no_ifblock:
- case no_record:
- default:
- if (optimizeFieldOrder(result, reorder))
- same = false;
- result.append(*LINK(cur));
- reorder.kill();
- break;
- }
- }
- if (optimizeFieldOrder(result, reorder))
- same = false;
- OwnedHqlExpr newRecord = same ? LINK(expr) : expr->clone(result);
- if (expr == newRecord)
- return meta.addAttribute(expr, EPaligned, queryAlignedAttr());
- meta.addAttribute(newRecord, EPaligned, queryAlignedAttr());
-
- OwnedHqlExpr alignAttr = createExprAttribute(_propAligned_Atom, newRecord.getClear());
- return meta.addAttribute(expr, EPaligned, alignAttr);
- }
- //---------------------------------------------------------------------------------
- MODULE_INIT(INIT_PRIORITY_HQLMETA)
- {
- for (node_operator op = (node_operator)(no_none+1); op < no_last_op; op = (node_operator)(op+1))
- getOperatorMetaFlags(op);
- return true;
- }
- MODULE_EXIT()
- {
- }
- //---------------------------------------------------------------------------------
- // Functions that provide simple information about an operator, that don't require tree traversal.
- bool isLocalActivity(IHqlExpression * expr)
- {
- switch (expr->getOperator())
- {
- case no_distribute:
- case no_keyeddistribute:
- case no_if:
- case no_chooseds:
- return false;
- case no_forcelocal:
- case no_combinegroup:
- case no_regroup:
- return true;
- //local makes no sense for the following
- case no_throughaggregate:
- case no_filter:
- case no_related:
- return false;
- case no_group:
- case no_grouped:
- case no_dedup:
- case no_cogroup:
- case no_cosort:
- case no_sort:
- case no_subsort:
- case no_sorted:
- case no_topn:
- case no_iterate:
- case no_rollup:
- case no_newaggregate:
- case no_merge:
- case no_choosen:
- case no_choosesets:
- case no_enth:
- case no_sample:
- case no_buildindex:
- case no_limit:
- case no_catchds:
- case no_newkeyindex:
- case no_table:
- case no_process:
- case no_assertsorted:
- case no_assertgrouped:
- case no_nonempty:
- case no_loop:
- case no_graphloop:
- case no_aggregate:
- case no_combine:
- assertex(localChangesActivity(expr));
- return expr->hasProperty(localAtom);
- case no_newusertable:
- if (isAggregateDataset(expr))
- return expr->hasProperty(localAtom);
- return false;
- case no_hqlproject: // count project may result in distributed output, but not be local(!)
- if (expr->hasProperty(_countProject_Atom))
- return expr->hasProperty(localAtom);
- return false;
- case no_denormalize:
- case no_denormalizegroup:
- case no_join:
- case no_mergejoin: //???
- case no_nwayjoin:
- case no_nwaymerge:
- case no_selfjoin:
- case no_joincount:
- assertex(localChangesActivity(expr));
- return expr->hasProperty(localAtom);
- case no_compound:
- return isLocalActivity(expr->queryChild(1));
- case no_compound_diskread:
- case no_compound_disknormalize:
- case no_compound_diskaggregate:
- case no_compound_diskcount:
- case no_compound_diskgroupaggregate:
- case no_compound_indexread:
- case no_compound_indexnormalize:
- case no_compound_indexaggregate:
- case no_compound_indexcount:
- case no_compound_indexgroupaggregate:
- {
- if (expr->hasProperty(localAtom))
- return true;
- IHqlExpression * root = queryRoot(expr);
- while (root->getOperator() == no_select)
- {
- bool isNew;
- IHqlExpression * ds = querySelectorDataset(root, isNew);
- if (!isNew)
- break;
- root = queryRoot(ds);
- }
- return isLocalActivity(root);
- }
- case no_compound_childread:
- case no_compound_childnormalize:
- case no_compound_childaggregate:
- case no_compound_childcount:
- case no_compound_childgroupaggregate:
- case no_compound_selectnew:
- case no_compound_inline:
- return true;
- default:
- {
- assertex(!localChangesActivity(expr));
- ITypeInfo * exprType = expr->queryType();
- if (exprType && (exprType->queryDistributeInfo() != NULL))
- return !isGroupedActivity(expr);
- return false;
- }
- }
- }
- bool isGroupedAggregateActivity(IHqlExpression * expr, IHqlExpression * grouping)
- {
- if (grouping && !grouping->isAttribute())
- return expr->hasProperty(groupedAtom);
- return isGrouped(expr->queryChild(0));
- }
- bool isGroupedActivity(IHqlExpression * expr)
- {
- switch (expr->getOperator())
- {
- case no_group:
- case no_enth:
- case no_distribute:
- case no_fetch:
- case no_keyeddistribute:
- case no_merge:
- case no_graphloop:
- return false;
- case no_denormalize:
- case no_denormalizegroup:
- case no_regroup:
- case no_addfiles:
- case no_join:
- case no_mergejoin:
- case no_nwayjoin:
- case no_nwaymerge:
- case no_selfjoin:
- case no_combine:
- case no_combinegroup:
- case no_if:
- case no_chooseds:
- case no_case:
- case no_map:
- case no_loop:
- case no_choosen:
- case no_process:
- case no_nonempty:
- case no_related:
- case no_pipe:
- return isGrouped(expr->queryType());
- case no_selectfields:
- case no_usertable:
- return isGroupedAggregateActivity(expr, expr->queryChild(2));
- case no_aggregate:
- case no_newaggregate:
- case no_newusertable:
- return isGroupedAggregateActivity(expr, expr->queryChild(3));
- case no_null:
- case no_anon:
- case no_pseudods:
- case no_fail:
- case no_skip:
- case no_all:
- case no_workunit_dataset:
- case no_getgraphresult:
- case no_getgraphloopresult:
- case no_getresult:
- case no_rows:
- case no_internalselect:
- case no_delayedselect:
- case no_unboundselect:
- case no_libraryselect:
- case no_purevirtual:
- case no_libraryinput:
- //All the source activities
- return isGrouped(expr->queryType());
- case no_compound:
- return isGroupedActivity(expr->queryChild(1));
- case no_output:
- return expr->hasProperty(groupedAtom) && isGroupedActivity(expr->queryChild(0));
- default:
- if (getNumChildTables(expr) == 1)
- return isGrouped(expr->queryChild(0));
- return false;
- }
- }
- bool localChangesActivityData(IHqlExpression * expr)
- {
- switch (expr->getOperator())
- {
- case no_compound_diskread:
- case no_compound_disknormalize:
- case no_compound_diskaggregate:
- case no_compound_diskcount:
- case no_compound_diskgroupaggregate:
- case no_compound_indexread:
- case no_compound_indexnormalize:
- case no_compound_indexaggregate:
- case no_compound_indexcount:
- case no_compound_indexgroupaggregate:
- case no_newkeyindex:
- case no_table:
- return true;
- case no_denormalize:
- case no_denormalizegroup:
- case no_join:
- case no_joincount:
- return isKeyedJoin(expr); // keyed join, local means only look at the local key part.
- //case no_fetch:////????
- }
- return false;
- }
- bool localChangesActivityAction(IHqlExpression * expr)
- {
- switch (expr->getOperator())
- {
- case no_dedup:
- case no_group:
- case no_grouped:
- case no_cogroup:
- case no_cosort:
- case no_sort:
- case no_subsort:
- case no_sorted:
- case no_topn:
- case no_iterate:
- case no_rollup:
- case no_newaggregate:
- case no_aggregate:
- case no_merge:
- case no_choosen:
- case no_choosesets:
- case no_enth:
- case no_sample:
- case no_buildindex:
- case no_limit:
- case no_catchds:
- case no_compound_diskaggregate:
- case no_compound_diskgroupaggregate:
- case no_compound_indexaggregate:
- case no_compound_indexgroupaggregate:
- case no_process:
- case no_assertsorted:
- case no_assertgrouped:
- case no_nonempty:
- case no_loop:
- case no_graphloop:
- case no_combine:
- return true;
- case no_hqlproject:
- return expr->hasProperty(_countProject_Atom);
- case no_newusertable:
- return isAggregateDataset(expr);
- case no_denormalize:
- case no_denormalizegroup:
- case no_join:
- case no_mergejoin: //???
- case no_nwayjoin:
- case no_nwaymerge:
- case no_selfjoin:
- case no_joincount:
- return !isKeyedJoin(expr); // Keyed joins always
- }
- return false;
- }
- bool localChangesActivity(IHqlExpression * expr)
- {
- return localChangesActivityData(expr) || localChangesActivityAction(expr);
- }
- unsigned isStreamingActivity(IHqlExpression * expr)
- {
- switch (expr->getOperator())
- {
- case no_sort:
- case no_topn:
- if (isGrouped(expr))
- return 0;
- return 1;
- case no_join:
- case no_denormalize:
- if (isKeyedJoin(expr))
- return 0;
- if (expr->hasProperty(lookupAtom))
- return 2;
- return 3; // ok if lhs/rhs are sorted...
- case no_selfjoin:
- return 1; // ok if sorted.
- case no_dedup:
- if (isGrouped(expr))
- return 0;
- if (expr->hasProperty(hashAtom) || expr->hasProperty(allAtom))
- return false;
- break;
- case no_addfiles:
- //if ordered and same item is read by lhs and rhs
- // ordered addfiles?
- break;
- case no_libraryselect:
- //????
- return 1;
- case no_spillgraphresult:
- case no_setgraphresult:
- case no_setgraphloopresult:
- break; //except for default loop output because likely to be read by a child as a whole
- }
- return 0;
- }
- // More complex derived information which requires tree traversal.
- bool isInlineTrivialDataset(IHqlExpression * expr)
- {
- loop
- {
- switch (expr->getOperator())
- {
- case no_selectnth:
- switch (expr->queryChild(1)->getOperator())
- {
- case no_constant:
- case no_counter:
- break;
- default:
- return false;
- }
- expr = expr->queryChild(0);
- break;
- case no_workunit_dataset:
- case no_getresult:
- case no_null:
- return true;
- case no_getgraphresult:
- return !expr->hasProperty(_distributed_Atom);
- default:
- return false;
- }
- }
- }
- bool isTrivialDataset(IHqlExpression * expr)
- {
- loop
- {
- if (isInlineTrivialDataset(expr))
- return true;
- switch (expr->getOperator())
- {
- case no_translated:
- case no_null:
- case no_temprow:
- case no_projectrow:
- case no_left:
- case no_right:
- case no_id2blob:
- case no_activerow:
- case no_typetransfer:
- case no_rows:
- case no_skip:
- case no_matchattr:
- case no_matchrow:
- case no_libraryinput:
- case no_workunit_dataset:
- case no_activetable:
- case no_top:
- return true;
- case no_select:
- if (!isNewSelector(expr))
- return false;
- if (expr->isDataset())
- return true;
- expr = expr->queryChild(0);
- break;
- case no_selectnth:
- case no_alias:
- case no_sorted:
- case no_distributed:
- case no_grouped:
- case no_preservemeta:
- case no_dataset_alias:
- case no_filter:
- expr = expr->queryChild(0);
- break;
- case no_inlinetable:
- return isConstantDataset(expr);
- default:
- return false;
- }
- }
- }
- static unsigned estimateRowSize(IHqlExpression * record)
- {
- IHqlExpression * size = record->queryAttribute(EPsize);
- if (!size || !size->queryChild(2)->queryValue())
- return UNKNOWN_LENGTH;
- return (unsigned)getIntValue(size->queryChild(0));
- }
- bool reducesRowSize(IHqlExpression * expr)
- {
- //More: This should be improved...., but slightly tricky without doing lots more processing.
- IHqlExpression * transform = queryNewColumnProvider(expr);
- IHqlExpression * prevRecord = expr->queryChild(0)->queryRecord();
- unsigned newRowSize = estimateRowSize(transform->queryRecord());
- unsigned prevRowSize = estimateRowSize(prevRecord);
- if ((newRowSize != UNKNOWN_LENGTH) && (prevRowSize != UNKNOWN_LENGTH))
- return newRowSize < prevRowSize;
- IHqlExpression * record = expr->queryRecord();
- if (getFlatFieldCount(record) < getFlatFieldCount(prevRecord))
- return true;
- return false;
- }
- bool increasesRowSize(IHqlExpression * expr)
- {
- IHqlExpression * transform = queryNewColumnProvider(expr);
- IHqlExpression * prevRecord = expr->queryChild(0)->queryRecord();
- unsigned newRowSize = estimateRowSize(transform);
- unsigned prevRowSize = estimateRowSize(prevRecord);
- if ((newRowSize != UNKNOWN_LENGTH) && (prevRowSize != UNKNOWN_LENGTH))
- return newRowSize > prevRowSize;
- IHqlExpression * record = expr->queryRecord();
- if (getFlatFieldCount(record) > getFlatFieldCount(prevRecord))
- return true;
- return false;
- }
- bool isLimitedDataset(IHqlExpression * expr, bool onFailOnly)
- {
- loop
- {
- if (expr->hasProperty(limitAtom))
- return true;
- switch (expr->getOperator())
- {
- case no_choosen:
- case no_limit:
- // case no_keyedlimit: // not included because it is done before everything else, so filters can be merged in
- if (!onFailOnly || expr->hasProperty(onFailAtom))
- return true;
- break;
- case no_table:
- case no_newkeyindex:
- return false;
- default:
- if (getNumChildTables(expr) != 1)
- return false;
- break;
- }
- expr = expr->queryChild(0);
- }
- }
- bool containsAnyActions(IHqlExpression * expr)
- {
- switch (expr->getOperator())
- {
- case no_comma:
- case no_compound:
- case no_actionlist:
- {
- ForEachChild(i, expr)
- {
- if (containsAnyActions(expr->queryChild(i)))
- return true;
- }
- return false;
- }
- case no_setmeta:
- return false;
- default:
- return true;
- }
- }
- //-- Attribute: record count -------------------------------------------------------------------------------
- unsigned getCardinality(IHqlExpression * expr)
- {
- loop
- {
- switch (expr->getOperator())
- {
- case no_select:
- expr = expr->queryChild(1);
- break;
- case no_constant:
- return 1;
- case no_field:
- {
- IHqlExpression * cardinality = queryPropertyChild(expr, cardinalityAtom, 0);
- if (cardinality)
- return (unsigned)getIntValue(cardinality);
- }
- //fall through:
- default:
- return expr->queryType()->getCardinality();
- }
- }
- }
- bool isSmallGrouping(IHqlExpression * sortlist)
- {
- unsigned __int64 totalCardinality = 1;
- unsigned max = sortlist->numChildren();
- for (unsigned idx = 0; idx < max; idx++)
- {
- IHqlExpression * cur = sortlist->queryChild(idx);
- unsigned cardinality = getCardinality(cur);
- if (!cardinality)
- return false;
- totalCardinality *= cardinality;
- //don't use hash aggregation if larger than 100,000 potential elements
- if (totalCardinality >= 100000)
- return false;
- }
- return true;
- }
- //An estimate of the order of magnitude of the number of rows in a dataset. See function below for artificial thresholds.
- const static unsigned __int64 RCtinyLimit = 10;
- const static unsigned __int64 RCgroupLimit = 1000;
- const static unsigned __int64 RCfewLimit = 100000;
- const static unsigned __int64 RCmemoryLimit = 50000000;
- const static unsigned __int64 RCclusterSizeEstimate = 5000;
- enum RowCountMagnitude
- {
- RCMnone, // 0
- RCMtiny, // < 10
- RCMgroup, // < 1000
- RCMfew, // < 100,000
- RCMmemory, // < memory
- RCMdisk, // who knows?
- RCMunknown
- };
- const char * const magnitudeText[] = {
- "empty",
- "tiny",
- "group",
- "few",
- "memory",
- "disk",
- "unknown"
- };
- inline RowCountMagnitude getRowCountMagnitude(__int64 num)
- {
- if (num == 0)
- return RCMnone;
- if (num <= RCtinyLimit)
- return RCMtiny;
- if (num <= RCgroupLimit)
- return RCMgroup;
- if (num <= RCfewLimit)
- return RCMfew;
- if (num <= RCmemoryLimit)
- return RCMmemory;
- return RCMdisk;
- }
- static IHqlExpression * makeConstant(__int64 value)
- {
- if ((value >= 0) && (size32_t)value == value)
- return getSizetConstant((size32_t)value);
- return createConstant(value);
- }
- struct HqlRowCountInfo
- {
- public:
- HqlRowCountInfo() { setUnknown(RCMnone); }
- void applyChoosen(__int64 limit, bool isLocal);
- void combineAlternatives(const HqlRowCountInfo & other);
- void combineBoth(const HqlRowCountInfo & other);
- bool extractHint(IHqlExpression * hint);
- void limitMin(__int64 value);
- void setEstimate(__int64 n);
- void scaleFixed(__int64 scale);
- void scaleRange(__int64 scale);
- void setMin(__int64 n) { min.setown(makeConstant(n)); }
- void setN(__int64 n);
- void setRange(__int64 low, __int64 high);
- void setUnknown(RowCountMagnitude _magnitude);
- void setMaxMagnitude(RowCountMagnitude _magnitude)
- {
- if (magnitude > _magnitude)
- magnitude = _magnitude;
- }
- IHqlExpression * createRecordCountAttr()
- {
- return createExprAttribute(_propRecordCount_Atom, makeConstant(magnitude), LINK(min), LINK(max));// , LINK(estimate));
- }
- void extract(IHqlExpression * attr)
- {
- assertex(attr->queryName() == _propRecordCount_Atom);
- magnitude = (RowCountMagnitude)getIntValue(attr->queryChild(0));
- min.set(attr->queryChild(1));
- max.set(attr->queryChild(2));
- //estimate.set(attr->queryChild(3));
- }
- inline void setSingleRow() { setN(1); }
- void getText(StringBuffer & text) const;
- __int64 getMin() const { return getIntValue(min); }
- inline bool isSingleRow() const
- {
- return matchesConstantValue(min, 1) && matchesConstantValue(max, 1);
- }
- inline bool alwaysHasRow() const
- {
- return !matchesConstantValue(min, 0);
- }
- public:
- OwnedHqlExpr min; // Absolute minimum - can't be fewer records
- OwnedHqlExpr max; // Absolute maximum - can't be more records
- RowCountMagnitude magnitude; // Expected magnitude. Normally matches max, but may occasionally diverge,.
- //It might be possible to calculate an estimate of the number of rows, but I'm not sure if it
- //is possible to make it significantly more useful than the magnitude.
- // OwnedHqlExpr estimate;
- };
- void HqlRowCountInfo::applyChoosen(__int64 limit, bool isLocal)
- {
- if (getMin() > limit)
- min.setown(makeConstant(limit));
- __int64 maxLimit = isLocal ? RCclusterSizeEstimate*limit : limit;
- if (getIntValue(max, maxLimit+1) > maxLimit)
- max.setown(makeConstant(maxLimit));
- RowCountMagnitude newMagnitude = getRowCountMagnitude(maxLimit);
- if (magnitude > newMagnitude)
- magnitude = newMagnitude;
- }
- void HqlRowCountInfo::combineAlternatives(const HqlRowCountInfo & other)
- {
- if (other.getMin() < getMin())
- min.set(other.min);
- IValue * maxValue = max->queryValue();
- if (maxValue)
- {
- IValue * otherMaxValue = other.max->queryValue();
- if (!otherMaxValue || (otherMaxValue->getIntValue() > maxValue->getIntValue()))
- max.set(other.max);
- }
- if (magnitude < other.magnitude)
- magnitude = other.magnitude;
- }
- void HqlRowCountInfo::combineBoth(const HqlRowCountInfo & other)
- {
- min.setown(makeConstant(getMin()+other.getMin()));
- IValue * maxValue = max->queryValue();
- IValue * otherMaxValue = other.max->queryValue();
- if (!otherMaxValue)
- max.set(other.max);
- else if (maxValue)
- {
- __int64 newMax = maxValue->getIntValue()+otherMaxValue->getIntValue();
- max.setown(makeConstant(newMax));
- }
- //Appending shouldn't change to a larger magnitude.
- if (magnitude < other.magnitude)
- magnitude = other.magnitude;
- }
- bool HqlRowCountInfo::extractHint(IHqlExpression * hint)
- {
- IHqlExpression * arg = hint->queryChild(0);
- if (!arg)
- return false;
- switch (arg->getOperator())
- {
- case no_constant:
- setN(getIntValue(arg));
- return true;
- case no_rangeto:
- setRange(0, getIntValue(arg->queryChild(0)));
- return true;
- case no_range:
- setRange(getIntValue(arg->queryChild(0)), getIntValue(arg->queryChild(1)));
- return true;
- case no_attr:
- {
- IAtom * name = arg->queryName();
- RowCountMagnitude magnitude = RCMnone;
- if (name == tinyAtom)
- magnitude = RCMtiny;
- else if (name == groupAtom)
- magnitude = RCMgroup;
- else if (name == fewAtom)
- magnitude = RCMfew;
- else if (name == memoryAtom)
- magnitude = RCMmemory;
- if (magnitude != RCMnone)
- {
- setUnknown(magnitude);
- return true;
- }
- break;
- }
- }
- return false;
- }
- void HqlRowCountInfo::getText(StringBuffer & text) const
- {
- min->queryValue()->generateECL(text);
- text.append("..");
- if (max->queryValue())
- max->queryValue()->generateECL(text);
- else
- text.append("?");
- text.append("[").append(magnitudeText[magnitude]).append("]");
- }
- void HqlRowCountInfo::limitMin(__int64 value)
- {
- if (getMin() > value)
- min.setown(makeConstant(value));
- }
- void HqlRowCountInfo::scaleFixed(__int64 scale)
- {
- __int64 minValue = getMin();
- __int64 maxValue = getIntValue(max, 0);
- if (maxValue)
- {
- setRange(minValue * scale, maxValue * scale); // MORE: Worry about 64bit overflow
- }
- else
- {
- setUnknown(RCMdisk);
- setMin(minValue * scale);
- }
- }
- void HqlRowCountInfo::scaleRange(__int64 scale)
- {
- scaleFixed(scale);
- min.setown(makeConstant(0));
- }
- void HqlRowCountInfo::setEstimate(__int64 n)
- {
- magnitude = getRowCountMagnitude(n);
- }
- void HqlRowCountInfo::setN(__int64 n)
- {
- setMin(n);
- max.set(min);
- magnitude = getRowCountMagnitude(n);
- }
- void HqlRowCountInfo::setRange(__int64 low, __int64 high)
- {
- min.setown(makeConstant(low));
- max.setown(makeConstant(high));
- magnitude = getRowCountMagnitude(high);
- }
- void HqlRowCountInfo::setUnknown(RowCountMagnitude _magnitude)
- {
- min.setown(getSizetConstant(0));
- max.setown(getUnknownAttribute());
- magnitude = _magnitude;
- }
- //MORE: This information should be cached in an attribute, once it is working, and used in more than one place.
- void retrieveRowInformation(HqlRowCountInfo & info, IHqlExpression * expr)
- {
- IHqlExpression * attr = expr->queryAttribute(EPrecordCount);
- info.extract(attr);
- }
- static void calcIntersectingRowInformation(HqlRowCountInfo & info, IHqlExpression * expr, unsigned firstDs)
- {
- retrieveRowInformation(info, expr->queryChild(firstDs));
- ForEachChildFrom(i, expr, firstDs+1)
- {
- IHqlExpression * cur = expr->queryChild(i);
- if (!cur->isAttribute())
- {
- HqlRowCountInfo nextInfo;
- retrieveRowInformation(nextInfo, cur);
- info.combineBoth(nextInfo);
- }
- }
- }
- //MORE: This would benefit from knowing if the target is hthor/roxie (or a thoir child query) so it could tell if local means
- //anything. The best solution is to annotate the graph with _global_ for thor, or _single_ for the others. One day....
- IHqlExpression * calcRowInformation(IHqlExpression * expr)
- {
- HqlRowCountInfo info;
- IHqlExpression * hint = queryHint(expr, outputAtom);
- if (hint && info.extractHint(hint))
- return info.createRecordCountAttr();
- IHqlExpression * ds = expr->queryChild(0);
- node_operator op = expr->getOperator();
- switch (op)
- {
- case no_nothor:
- case no_thor:
- case no_compound_diskread:
- case no_compound_disknormalize:
- case no_compound_diskaggregate:
- case no_compound_diskcount:
- case no_compound_diskgroupaggregate:
- case no_compound_indexread:
- case no_compound_indexnormalize:
- case no_compound_indexaggregate:
- case no_compound_indexcount:
- case no_compound_indexgroupaggregate:
- case no_compound_childread:
- case no_compound_childnormalize:
- case no_compound_childaggregate:
- case no_compound_childcount:
- case no_compound_childgroupaggregate:
- case no_compound_inline:
- case no_compound_selectnew:
- case no_compound_fetch:
- case no_alias:
- case no_forcelocal:
- case no_distribute:
- case no_distributed:
- case no_preservemeta:
- case no_keyeddistribute:
- case no_sorted:
- case no_stepped:
- case no_assertsorted:
- case no_assertgrouped:
- case no_assertdistributed:
- case no_sort:
- case no_subsort:
- case no_nohoist:
- case no_section:
- case no_sectioninput:
- case no_assert_ds:
- case no_readspill:
- case no_writespill:
- case no_commonspill:
- case no_forcegraph:
- case no_split:
- case no_spill:
- case no_spillgraphresult:
- case no_outofline:
- case no_globalscope:
- case no_throughaggregate:
- case no_alias_scope:
- case no_thisnode:
- case no_preload:
- case no_combine:
- case no_catchds:
- case no_metaactivity:
- case no_cosort:
- case no_serialize:
- case no_deserialize:
- case no_executewhen:
- case no_owned_ds:
- case no_dataset_alias:
- {
- return getRecordCountInfo(ds);
- }
- case no_allnodes:
- {
- retrieveRowInformation(info, ds);
- info.scaleRange(RCclusterSizeEstimate);
- break;
- }
- case no_limit:
- case no_keyedlimit:
- {
- retrieveRowInformation(info, ds);
- __int64 limit = getIntValue(expr->queryChild(1), 0);
- if ((limit != 0) && !isGrouped(expr))
- info.applyChoosen(limit, isLocalActivity(expr));
- else
- info.limitMin(limit);
- break;
- }
- case no_hqlproject:
- case no_iterate:
- {
- retrieveRowInformation(info, ds);
- if (transformContainsSkip(expr->queryChild(1)))
- info.limitMin(0);
- break;
- }
- case no_fetch:
- {
- retrieveRowInformation(info, expr->queryChild(1));
- if (transformContainsSkip(expr->queryChild(3)))
- info.limitMin(0);
- break;
- }
- case no_dedup:
- {
- retrieveRowInformation(info, ds);
- //Only affect minimum => Grouped, local and non grouped may all reduce to 1
- info.limitMin(1);
- break;
- }
- case no_rollup:
- case no_rollupgroup:
- {
- //rollup on a single row is a single row, rollup on non single may or may not be.
- retrieveRowInformation(info, ds);
- if (transformContainsSkip(queryNewColumnProvider(expr)))
- info.limitMin(0);
- else
- info.limitMin(1);
- break;
- }
- case no_aggregate:
- case no_newaggregate:
- case no_newusertable:
- case no_selectfields:
- case no_usertable:
- {
- retrieveRowInformation(info, ds);
- if (isAggregateDataset(expr))
- {
- IHqlExpression * grouping = queryDatasetGroupBy(expr);
- if (!grouping)
- grouping = queryGrouping(ds);
- if (grouping)
- {
- //Either aggregate grouped dataset, or grouping supplied. Similar semantics.
- //minimum is 1 unless inputs has minimum of 0
- info.limitMin(1);
- if (expr->hasProperty(fewAtom))
- info.setMaxMagnitude(RCMfew);
- else if (isSmallGrouping(grouping))
- info.setMaxMagnitude(RCMfew);
- }
- else if (isLocalActivity(expr))
- {
- info.setRange(1, RCclusterSizeEstimate); // local,ungrouped -> one per node
- }
- else
- info.setSingleRow();
- }
- else
- {
- if (transformContainsSkip(queryNewColumnProvider(expr)))
- info.limitMin(0);
- }
- break; // maybe a project of an aggregate
- }
- case no_selectnth:
- case no_datasetfromrow:
- case no_activerow:
- {
- info.setSingleRow();
- break;
- }
- case no_rows:
- {
- info.setUnknown(RCMgroup);
- break;
- }
- case no_rowsetindex:
- case no_rowsetrange:
- {
- info.setUnknown(RCMmemory);
- break;
- }
- case no_workunit_dataset:
- case no_getgraphresult:
- case no_getgraphloopresult:
- case no_getresult:
- {
- IHqlExpression * attr = expr->queryProperty(_propRecordCount_Atom);
- if (attr)
- return LINK(attr);
- if (expr->isDatarow() || expr->hasProperty(rowAtom))
- {
- info.setSingleRow();
- }
- else
- {
- if (expr->hasProperty(_distributed_Atom))
- info.setUnknown(RCMdisk);
- else
- info.setUnknown(RCMfew);
- }
- break;
- }
- case no_table:
- case no_keyindex:
- case no_newkeyindex:
- {
- IHqlExpression * attr = expr->queryProperty(_propRecordCount_Atom);
- if (attr)
- return LINK(attr);
- if (expr->isDatarow() || expr->hasProperty(rowAtom))
- {
- info.setSingleRow();
- }
- else
- {
- info.setUnknown(RCMdisk);
- //Allow an annotation on a dataset to specify exact and ranges of counts.
- IHqlExpression * count = queryPropertyChild(expr, countAtom, 0);
- IHqlExpression * maxCount = queryPropertyChild(expr, maxCountAtom, 0);
- IHqlExpression * aveCount = queryPropertyChild(expr, aveAtom, 0);
- if (count)
- info.setN(getIntValue(count));
- else if (maxCount)
- info.setRange(0, getIntValue(maxCount));
- else if (aveCount)
- info.setEstimate(getIntValue(aveCount));
- }
- break;
- }
- case no_filter:
- case no_filtergroup:
- case no_sample:
- {
- retrieveRowInformation(info, ds);
- info.limitMin(0);
- //More sample could potentially reduce the magnitude
- break;
- }
- case no_temptable:
- {
- IHqlExpression * values = expr->queryChild(0);
- if (values->getOperator() == no_recordlist)
- info.setN(values->numChildren());
- else
- info.setUnknown(RCMfew);
- break;
- }
- case no_inlinetable:
- {
- IHqlExpression * transforms = expr->queryChild(0);
- unsigned maxValue = transforms->numChildren();
- unsigned minValue = 0;
- for (unsigned i=0; i < maxValue; i++)
- {
- if (!containsSkip(transforms->queryChild(i)))
- minValue++;
- }
- info.setRange(minValue, maxValue);
- break;
- }
- case no_dataset_from_transform:
- {
- // only if the count is a constant value
- IHqlExpression * count = expr->queryChild(0);
- IValue * value = count->queryValue();
- if (value)
- {
- IHqlExpression * transform = expr->queryChild(1);
- __int64 maxCount = value->getIntValue();
- if (containsSkip(transform))
- info.setRange(0, maxCount);
- else
- info.setN(maxCount);
- }
- // leave it be, if it's a constant expression or a variable
- break;
- }
- case no_null:
- info.setN(expr->isDatarow() ? 1 : 0);
- break;
- case no_fail:
- info.setN(0);
- break;
- case no_if:
- {
- retrieveRowInformation(info, expr->queryChild(1));
- IHqlExpression * rhs = expr->queryChild(2);
- if (rhs)
- {
- HqlRowCountInfo rhsInfo;
- retrieveRowInformation(rhsInfo, rhs);
- info.combineAlternatives(rhsInfo);
- }
- else
- {
- info.min.setown(getSizetConstant(0));
- }
- break;
- }
- case no_nonempty:
- {
- retrieveRowInformation(info, ds);
- //Go through the children so we get a sensible value for the magnitude
- unsigned max = expr->numChildren();
- for (unsigned i=1; i< max; i++)
- {
- if (!isZero(info.min))
- break;
- IHqlExpression * cur = expr->queryChild(i);
- if (!cur->isAttribute())
- {
- HqlRowCountInfo nextInfo;
- retrieveRowInformation(nextInfo, cur);
- info.min.set(nextInfo.min);
- info.combineAlternatives(nextInfo);
- }
- }
- break;
- }
- case no_chooseds:
- case no_regroup:
- case no_combinegroup:
- case no_addfiles:
- case no_merge:
- {
- unsigned firstDataset = getFirstActivityArgument(expr);
- calcIntersectingRowInformation(info, expr, firstDataset);
- break;
- }
- case no_choosen:
- {
- retrieveRowInformation(info, ds);
- __int64 choosenLimit = getIntValue(expr->queryChild(1), 0);
- if (choosenLimit == CHOOSEN_ALL_LIMIT)
- info.limitMin(0); // play safe - could be clever if second value is constant, and min/max known.
- else if ((choosenLimit != 0) && !isGrouped(expr))
- info.applyChoosen(choosenLimit, isLocalActivity(expr));
- else
- info.limitMin(choosenLimit);
- }
- break;
- case no_topn:
- {
- retrieveRowInformation(info, ds);
- __int64 choosenLimit = getIntValue(expr->queryChild(2), 0);
- if ((choosenLimit > 0) && !isGrouped(expr))
- info.applyChoosen(choosenLimit, isLocalActivity(expr));
- else
- info.limitMin(choosenLimit);
- }
- break;
- case no_select:
- {
- bool isNew;
- IHqlExpression * realDs = querySelectorDataset(expr, isNew);
- if (isNew)
- retrieveRowInformation(info, realDs);
- else
- info.setSingleRow();
- if (!expr->isDatarow())
- {
- IHqlExpression * field = expr->queryChild(1);
- __int64 count = getIntValue(queryPropertyChild(field, countAtom, 0), 0);
- __int64 maxcount = getIntValue(queryPropertyChild(field, maxCountAtom, 0), 0);
- if (count)
- info.scaleFixed(count);
- else if (maxcount)
- info.scaleRange(maxcount);
- else if (info.isSingleRow())
- info.setUnknown(RCMfew);
- else
- info.setUnknown(RCMdisk);
- }
- break;
- }
- case no_normalize:
- {
- retrieveRowInformation(info, ds);
- IValue * numRows = expr->queryChild(1)->queryValue();
- if (numRows)
- {
- __int64 scale = numRows->getIntValue();
- if (containsSkip(expr->queryChild(2)))
- info.scaleRange(scale);
- else
- info.scaleFixed(scale);
- }
- else
- info.setUnknown(RCMdisk);
- break;
- }
- case no_group:
- case no_grouped:
- //MORE: Not completely sure how we should handle groups.
- return getRecordCountInfo(ds);
- case no_join:
- case no_selfjoin:
- {
- bool maxSingleRowOut = false;
- if (expr->hasProperty(leftonlyAtom))
- maxSingleRowOut = true;
- else if (isLeftJoin(expr) || isInnerJoin(expr))
- {
- IHqlExpression * keep = queryPropertyChild(expr, keepAtom, 0);
- if (matchesConstantValue(keep, 1))
- maxSingleRowOut = true;
- }
- if (maxSingleRowOut)
- {
- retrieveRowInformation(info, ds);
- if (!expr->hasProperty(leftouterAtom) || containsSkip(expr->queryChild(3)))
- info.limitMin(0);
- }
- else
- info.setUnknown(RCMdisk);
- break;
- }
- case no_denormalize:
- case no_denormalizegroup:
- {
- retrieveRowInformation(info, ds);
- if (containsSkip(expr->queryChild(3)))
- info.limitMin(0);
- break;
- }
- case no_mergejoin:
- case no_nwayjoin:
- case no_nwaymerge:
- info.setUnknown(RCMdisk);
- break;
- case no_loop:
- case no_graphloop:
- case no_libraryselect:
- case no_libraryinput:
- case no_param:
- case no_anon:
- case no_nofold: // assume nothing - to stop subsequent optimizations
- case no_delayedselect:
- case no_unboundselect:
- case no_internalselect:
- info.setUnknown(RCMdisk);
- break;
- case no_parse:
- case no_newparse:
- case no_xmlparse:
- case no_newxmlparse:
- case no_soapcall:
- case no_soapcall_ds:
- case no_newsoapcall:
- case no_newsoapcall_ds:
- case no_httpcall:
- case no_process:
- case no_pipe:
- case no_translated:
- case no_datasetfromdictionary:
- //MORE could improve each of these
- info.setUnknown(RCMdisk);
- break;
- case no_map:
- case no_case:
- {
- if (expr->isDatarow())
- {
- info.setSingleRow();
- break;
- }
- //This is primarily implemented so the annotations in the graph look correct
- unsigned start = (op == no_case) ? 1 : 0;
- IHqlExpression * dft = NULL;
- ForEachChildFrom(i1, expr, start)
- {
- IHqlExpression * cur = expr->queryChild(i1);
- if (cur->getOperator() != no_mapto)
- {
- if (!cur->isAttribute())
- dft = cur;
- break;
- }
- }
- if (dft)
- retrieveRowInformation(info, dft);
- else
- info.setN(0);
- ForEachChildFrom(i2, expr, start)
- {
- IHqlExpression * cur = expr->queryChild(i2);
- if (cur->getOperator() == no_mapto)
- {
- HqlRowCountInfo rhsInfo;
- retrieveRowInformation(rhsInfo, cur->queryChild(1));
- info.combineAlternatives(rhsInfo);
- }
- }
- break;
- }
- case no_id2blob:
- case no_xmlproject:
- case no_call:
- case no_externalcall:
- info.setUnknown(RCMfew);
- break;
- case no_colon:
- {
- IHqlExpression * workflow = expr->queryChild(1);
- //For either of
- if (queryOperatorInList(no_stored, workflow) || queryOperatorInList(no_recovery, workflow))
- {
- info.setUnknown(RCMdisk);
- break;
- }
- //MORE: Could restrict based on few flags
- return getRecordCountInfo(ds);
- }
- case no_choosesets:
- case no_enth:
- //MORE: Could sum the numbers to return
- return getRecordCountInfo(ds);
- case no_compound:
- return getRecordCountInfo(expr->queryChild(1));
- default:
- if (expr->isDataset())
- UNIMPLEMENTED_XY("Record count calculation for operator", getOpString(op));
- if (expr->isDatarow())
- info.setSingleRow();
- else
- info.setUnknown(RCMdisk); //Assume the worse case...
- break;
- }
- return info.createRecordCountAttr();
- }
- static IHqlExpression * evaluateAttrRecordCount(IHqlExpression * expr)
- {
- OwnedHqlExpr info = calcRowInformation(expr);
- return meta.addAttribute(expr, EPrecordCount, info);
- }
- void getRecordCountText(StringBuffer & result, IHqlExpression * expr)
- {
- HqlRowCountInfo info;
- retrieveRowInformation(info, expr);
- info.getText(result);
- }
- //---------------------------------------------------------------------------------
- bool hasFewRows(IHqlExpression * expr)
- {
- HqlRowCountInfo info;
- retrieveRowInformation(info, expr);
- return (info.magnitude <= RCMfew);
- }
- bool spillToWorkunitNotFile(IHqlExpression * expr, ClusterType platform)
- {
- if (platform == RoxieCluster)
- return true;
- if (isThorCluster(platform))
- {
- //In thor, all rows will get sent to master and written to dali, and then read back on slave 0
- //not likely to be more efficient unless only a single row - although the generated code accessing
- //from a child query is better
- return hasNoMoreRowsThan(expr, 1);
- }
- return hasFewRows(expr);
- }
- bool hasSingleRow(IHqlExpression * expr)
- {
- HqlRowCountInfo info;
- retrieveRowInformation(info, expr);
- return info.isSingleRow();
- }
- bool hasNoMoreRowsThan(IHqlExpression * expr, __int64 limit)
- {
- HqlRowCountInfo info;
- retrieveRowInformation(info, expr);
- return getIntValue(info.max, limit+1) <= limit;
- }
- // Functions for testing whether
- // Functions for accessing attributes from types etc.
- IHqlExpression * queryProperty(ITypeInfo * type, IAtom * search)
- {
- loop
- {
- typemod_t curModifier = type->queryModifier();
- switch (curModifier)
- {
- case typemod_none:
- return NULL;
- case typemod_attr:
- {
- IHqlExpression * prop = static_cast<IHqlExpression *>(type->queryModifierExtra());
- if (prop->queryName() == search)
- return prop;
- break;
- }
- case typemod_original:
- {
- IHqlExpression * original = static_cast<IHqlExpression *>(type->queryModifierExtra());
- IHqlExpression * match = original->queryProperty(search);
- if (match)
- return match;
- break;
- }
- }
- type = type->queryTypeBase();
- }
- }
- IHqlExpression * queryPropertyChild(ITypeInfo * type, IAtom * search, unsigned idx)
- {
- IHqlExpression * match = queryProperty(type, search);
- if (match)
- return match->queryChild(idx);
- return NULL;
- }
- // Functions for extracting and preserving attribute information on types and fields.
- void cloneFieldModifier(Shared<ITypeInfo> & type, ITypeInfo * donorType, IAtom * attr)
- {
- IHqlExpression * match = queryProperty(donorType, attr);
- if (!match)
- return;
- IHqlExpression * existing = queryProperty(type, attr);
- if (match == existing)
- return;
- type.setown(makeAttributeModifier(type.getClear(), LINK(match)));
- }
- ITypeInfo * cloneEssentialFieldModifiers(ITypeInfo * donor, ITypeInfo * rawtype)
- {
- Linked<ITypeInfo> type = rawtype;
- cloneFieldModifier(type, donor, maxLengthAtom);
- cloneFieldModifier(type, donor, maxSizeAtom);
- cloneFieldModifier(type, donor, maxCountAtom);
- return type.getClear();
- }
- ITypeInfo * removeProperty(ITypeInfo * t, IAtom * search)
- {
- typemod_t curModifier = t->queryModifier();
- if (curModifier == typemod_none)
- return LINK(t);
- ITypeInfo * base = t->queryTypeBase();
- if (curModifier == typemod_attr)
- {
- IHqlExpression * attr = (IHqlExpression *)t->queryModifierExtra();
- if (attr->queryName() == search)
- return LINK(base);
- }
- OwnedITypeInfo newBase = removeProperty(base, search);
- if (newBase == base)
- return LINK(t);
- return makeModifier(newBase.getClear(), curModifier, LINK(t->queryModifierExtra()));
- }
- bool isUninheritedFieldAttribute(IHqlExpression * expr)
- {
- if (expr->isAttribute())
- {
- IAtom * name = expr->queryName();
- //MORE: Attributes of datasets need a different representation - should probably be include in the type somehow...
- if ((name == virtualAtom) || (name == countAtom))
- return true;
- }
- return false;
- }
- bool hasUninheritedAttribute(IHqlExpression * field)
- {
- ForEachChild(i, field)
- if (isUninheritedFieldAttribute(field->queryChild(i)))
- return true;
- return false;
- }
- IHqlExpression * extractFieldAttrs(IHqlExpression * field)
- {
- IHqlExpression * attrs = NULL;
- ForEachChild(idx, field)
- {
- IHqlExpression * child = field->queryChild(idx);
- if (child->isAttribute())
- {
- //MORE: Attributes of datasets need a different representation - should probably be include in the type somehow...
- if (!isUninheritedFieldAttribute(child))
- {
- // which others should we ignore?
- attrs = createComma(attrs, LINK(child));
- }
- }
- }
- return attrs;
- }
- IHqlExpression * extractAttrsFromExpr(IHqlExpression * value)
- {
- if (!value)
- return NULL;
- if (value->getOperator() == no_select)
- value = value->queryChild(1);
- if (value->getOperator() == no_field)
- return extractFieldAttrs(value);
- return NULL;
- }
- // Type processing
- ITypeInfo * getPromotedECLType(ITypeInfo * lType, ITypeInfo * rType)
- {
- return ::getPromotedType(lType, rType);
- }
- ITypeInfo * getPromotedECLCompareType(ITypeInfo * lType, ITypeInfo * rType)
- {
- return ::getPromotedCompareType(lType, rType);
- }
- unsigned getMaxRecordSize(IHqlExpression * record, unsigned defaultMaxRecordSize, bool & hasKnownSize, bool & usedDefault)
- {
- IHqlExpression * size = record->queryAttribute(EPsize);
- IHqlExpression * minSizeExpr = size->queryChild(1);
- IHqlExpression * maxSizeExpr = size->queryChild(2);
- unsigned maxSize = (unsigned)getIntValue(maxSizeExpr, UNKNOWN_LENGTH);
- hasKnownSize = (minSizeExpr == maxSizeExpr);
- if (maxSize == UNKNOWN_LENGTH)
- {
- OwnedHqlExpr defaultExpr = getSizetConstant(defaultMaxRecordSize);
- OwnedHqlExpr value = replaceExpression(maxSizeExpr, queryDefaultMaxRecordLengthExpr(), defaultExpr);
- OwnedHqlExpr folded = foldHqlExpression(value);
- assertex(folded);
- maxSize = (unsigned)getIntValue(folded);
- unsigned minSize = getIntValue(minSizeExpr);
- if (maxSize < minSize)
- maxSize = minSize;
- usedDefault = true;
- }
- else
- usedDefault = false;
- return maxSize;
- }
- size32_t getExpectedRecordSize(IHqlExpression * record)
- {
- IHqlExpression * size = record->queryAttribute(EPsize);
- return size ? (size32_t)getIntValue(size->queryChild(0)) : 0;
- }
- size32_t getMinRecordSize(IHqlExpression * record)
- {
- IHqlExpression * size = record->queryAttribute(EPsize);
- return size ? (size32_t)getIntValue(size->queryChild(1)) : 0;
- }
- unsigned getMaxRecordSize(IHqlExpression * record, unsigned defaultMaxRecordSize)
- {
- bool isKnownSize, usedDefault;
- return getMaxRecordSize(record, defaultMaxRecordSize, isKnownSize, usedDefault);
- }
- bool maxRecordSizeUsesDefault(IHqlExpression * record)
- {
- IHqlExpression * maxSize = record->queryAttribute(EPsize)->queryChild(2);
- return (maxSize->queryValue() == NULL);
- }
- bool isVariableSizeRecord(IHqlExpression * record)
- {
- IHqlExpression * sizeAttr = record->queryAttribute(EPsize);
- return sizeAttr->queryChild(1) != sizeAttr->queryChild(2);
- }
- bool maxRecordSizeIsAmbiguous(IHqlExpression * record, size32_t & specifiedSize, size32_t & derivedSize)
- {
- IHqlExpression * sizeAttr = record->queryAttribute(EPsize);
- IHqlExpression * derivedSizeExpr = sizeAttr->queryChild(3);
- if (!derivedSizeExpr || !derivedSizeExpr->isConstant())
- return false;
- OwnedHqlExpr foldedDerivedSize = foldHqlExpression(derivedSizeExpr);
- if (!foldedDerivedSize->queryValue())
- return false;
- IHqlExpression * maxLength = sizeAttr->queryChild(2);
- OwnedHqlExpr foldedMaxLength = foldHqlExpression(maxLength);
- if (!foldedMaxLength->queryValue())
- return false;
- specifiedSize = (size32_t)foldedMaxLength->queryValue()->getIntValue();
- derivedSize = (size32_t) foldedDerivedSize->queryValue()->getIntValue();
- return derivedSize != specifiedSize;
- }
- bool maxRecordSizeCanBeDerived(IHqlExpression * record)
- {
- if (!isVariableSizeRecord(record))
- return true;
- if (record->hasProperty(maxLengthAtom))
- {
- IHqlExpression * sizeAttr = record->queryAttribute(EPsize);
- IHqlExpression * derivedSizeExpr = sizeAttr->queryChild(3);
- return (derivedSizeExpr != NULL);
- }
- return !maxRecordSizeUsesDefault(record);
- }
- //---------------------------------------------------------------------------------
- bool recordRequiresSerialization(IHqlExpression * expr, IAtom * serializeForm)
- {
- if (!expr)
- return false;
- if (querySerializedForm(expr, serializeForm) != expr)
- return true;
- return false;
- }
- bool recordRequiresDestructor(IHqlExpression * expr)
- {
- if (!expr)
- return false;
- //true if the internal serialized form is different
- if (querySerializedForm(expr, internalAtom) != expr)
- return true;
- return false;
- }
- bool recordRequiresLinkCount(IHqlExpression * expr)
- {
- //MORE: This should strictly speaking check if any of the child fields are link counted.
- //This function is a sufficient proxy at the moment
- return recordRequiresDestructor(expr);
- }
- bool recordSerializationDiffers(IHqlExpression * expr, IAtom * serializeForm1, IAtom * serializeForm2)
- {
- return querySerializedForm(expr, serializeForm1) != querySerializedForm(expr, serializeForm2);
- }
- extern HQL_API bool typeRequiresDeserialization(ITypeInfo * type, IAtom * serializeForm)
- {
- Owned<ITypeInfo> serializedType = getSerializedForm(type, serializeForm);
- if (queryUnqualifiedType(serializedType) == queryUnqualifiedType(type))
- return false;
- type_t stc = serializedType->getTypeCode();
- if (stc != type->getTypeCode())
- return true;
- if (stc == type_table)
- {
- if (recordTypesMatch(serializedType, type))
- return false;
- return true;
- }
- return true;
- }
- //---------------------------------------------------------------------------------
- IHqlExpression * queryRecordCountInfo(IHqlExpression * expr)
- {
- return expr->queryAttribute(EPrecordCount);
- }
- IHqlExpression * getRecordCountInfo(IHqlExpression * expr)
- {
- return LINK(expr->queryAttribute(EPrecordCount));
- }
- IHqlExpression * queryExpectedRecordCount(IHqlExpression * expr)
- {
- IHqlExpression * attr = expr->queryAttribute(EPrecordCount);
- return attr ? attr->queryChild(0) : NULL;
- }
- IHqlExpression * getPackedRecord(IHqlExpression * expr)
- {
- IHqlExpression * attr = expr->queryAttribute(EPaligned);
- IHqlExpression * packed = attr->queryChild(0);
- if (!packed) packed = expr;
- return LINK(packed);
- }
- /*
- * This function can be called while parsing (or later) to find a "normalized" version of a record or a field.
- * It ignores default values for fields, and removes named symbols/ other location specific information - so
- * that identical records defined in macros etc. are treated as identical.
- */
- IHqlExpression * getUnadornedRecordOrField(IHqlExpression * expr)
- {
- if (!expr)
- return NULL;
- IHqlExpression * attr = expr->queryAttribute(EPunadorned);
- return LINK(attr);
- }
- //---------------------------------------------------------------------------------
- inline bool isAlwaysLocationIndependent(IHqlExpression * expr)
- {
- switch (expr->getOperator())
- {
- case no_constant:
- case no_param:
- case no_quoted:
- case no_variable:
- return true;
- case no_attr:
- return (expr->numChildren() == 0);
- }
- return false;
- }
- class HqlLocationIndependentNormalizer : public QuickHqlTransformer
- {
- public:
- HqlLocationIndependentNormalizer();
- virtual IHqlExpression * createTransformed(IHqlExpression * expr);
- virtual ITypeInfo * transformType(ITypeInfo * type);
- protected:
- IHqlExpression * doCreateTransformed(IHqlExpression * expr);
- };
- static HqlTransformerInfo hqlLocationIndependentInfo("HqlLocationIndependentNormalizer");
- HqlLocationIndependentNormalizer::HqlLocationIndependentNormalizer() : QuickHqlTransformer(hqlLocationIndependentInfo, NULL)
- {
- }
- ITypeInfo * HqlLocationIndependentNormalizer::transformType(ITypeInfo * type)
- {
- switch (type->queryModifier())
- {
- case typemod_original:
- return transformType(type->queryTypeBase());
- case typemod_none:
- return QuickHqlTransformer::transformType(type);
- case typemod_indirect:
- {
- IHqlExpression * original = static_cast<IHqlExpression *>(type->queryModifierExtra());
- OwnedHqlExpr transformed = transform(original);
- return makeModifier(transformed->getType(), typemod_indirect, LINK(transformed));
- }
- default:
- {
- ITypeInfo * typeBase = type->queryTypeBase();
- Owned<ITypeInfo> newType = transformType(typeBase);
- if (typeBase == newType)
- return LINK(type);
- return cloneModifier(type, newType);
- }
- }
- }
- IHqlExpression * HqlLocationIndependentNormalizer::doCreateTransformed(IHqlExpression * expr)
- {
- node_operator op = expr->getOperator();
- switch (op)
- {
- case no_attr:
- {
- //Original attributes cause chaos => remove all children from attributes
- if (expr->numChildren() != 0)
- return createAttribute(expr->queryName());
- return LINK(expr);
- }
- case no_field:
- {
- //Remove the default values from fields since they just confuse.
- HqlExprArray children;
- bool same = true;
- ForEachChild(idx, expr)
- {
- IHqlExpression * cur = expr->queryChild(idx);
- if (cur->isAttribute())
- {
- IHqlExpression * mapped = transform(cur);
- children.append(*mapped);
- if (mapped != cur)
- same = false;
- }
- else
- same = false;
- }
- ITypeInfo * type = expr->queryType();
- OwnedITypeInfo newType = transformType(type);
- if (type != newType)
- return createField(expr->queryId(), newType.getClear(), children);
- if (same)
- return LINK(expr);
- return expr->clone(children);
- }
- }
- return QuickHqlTransformer::createTransformed(expr);
- }
- IHqlExpression * HqlLocationIndependentNormalizer::createTransformed(IHqlExpression * expr)
- {
- //Remove all annotations. It is vaguely possible there are some annotations we would want to retain, but I don't know of any
- IHqlExpression * body = expr->queryBody(false);
- if (expr != body)
- return transform(body);
- if (isAlwaysLocationIndependent(expr))
- return LINK(expr);
- IHqlExpression * match = meta.queryExistingAttribute(expr, EPlocationIndependent);
- if (match)
- return LINK(match);
- OwnedHqlExpr transformed = doCreateTransformed(expr);
- meta.addAttribute(expr, EPlocationIndependent, transformed);
- return transformed.getClear();
- }
- IHqlExpression * evaluateAttrLocationIndependent(IHqlExpression * expr)
- {
- if (isAlwaysLocationIndependent(expr))
- return expr->queryBody();
- //Because the transformers contain all the logic for how scopes etc. are transformed it is much better to
- //use a transformer which caches the result in the expression tree instead of trying to replicate
- //all the rules in some member functions.
- HqlLocationIndependentNormalizer normalizer;
- OwnedHqlExpr transformed = normalizer.transform(expr);
- return transformed; // NB: no getClear(). Because it is cached it is guaranteed to exist even when this link is released.
- }
- IHqlExpression * queryLocationIndependent(IHqlExpression * expr)
- {
- IHqlExpression * match = expr->queryAttribute(EPlocationIndependent);
- if (match)
- return match;
- return expr;
- }
- static void clonePropertyAsModifier(Owned<ITypeInfo> & type, IHqlExpression * donor, IAtom * attr)
- {
- if (queryProperty(type, attr))
- return;
- IHqlExpression * match = donor->queryProperty(attr);
- if (!match)
- return;
- type.setown(makeAttributeModifier(type.getClear(), LINK(match)));
- }
- ITypeInfo * preserveTypeQualifiers(ITypeInfo * ownedType, IHqlExpression * donor)
- {
- //The following would be a good idea, but it won't work until we introduce a recordof() operator
- //and use that whenever queryRecord() is currenly called (see bug46863)
- // type = makeModifier(type, typemod_indirect, LINK(arg));
- //Instead, just clone the attributes we need
- IHqlExpression * field = queryFieldFromExpr(donor);
- switch (field->getOperator())
- {
- case no_field:
- // case no_record:
- break;
- default:
- return ownedType;
- }
- OwnedITypeInfo type = ownedType;
- clonePropertyAsModifier(type, field, maxLengthAtom);
- clonePropertyAsModifier(type, field, maxSizeAtom);
- clonePropertyAsModifier(type, field, maxCountAtom);
- return type.getClear();
- }
- static bool cloneModifierAsProperty(HqlExprArray & args, ITypeInfo * donor, IAtom * attr)
- {
- IHqlExpression * match = queryProperty(donor, attr);
- if (!match)
- return true;
- if (queryProperty(attr, args))
- return true;
- args.append(*LINK(match));
- return false;
- }
- bool preserveTypeQualifiers(HqlExprArray & args, ITypeInfo * donor)
- {
- bool same = true;
- same = cloneModifierAsProperty(args, donor, maxLengthAtom) && same;
- same = cloneModifierAsProperty(args, donor, maxSizeAtom) && same;
- same = cloneModifierAsProperty(args, donor, maxCountAtom) && same;
- return same;
- }
- IHqlExpression * preserveTypeQualifiers(IHqlExpression * ownedField, ITypeInfo * donor)
- {
- OwnedHqlExpr field = ownedField;
- HqlExprArray args;
- unwindChildren(args, field);
- if (preserveTypeQualifiers(args, donor))
- return field.getClear();
- return field->clone(args);
- }
- bool isLinkedRowset(ITypeInfo * t)
- {
- switch (t->getTypeCode())
- {
- case type_table:
- case type_groupedtable:
- case type_dictionary:
- return hasLinkCountedModifier(t);
- }
- return false;
- }
- bool isArrayRowset(ITypeInfo * t)
- {
- switch (t->getTypeCode())
- {
- case type_table:
- case type_groupedtable:
- case type_array:
- case type_dictionary:
- {
- if (hasLinkCountedModifier(t))
- assertex(hasLinkCountedModifier(t->queryChildType()));
- if (hasOutOfLineModifier(t) || hasLinkCountedModifier(t))
- return true;
- ITypeInfo * rowType = t->queryChildType();
- if (hasOutOfLineModifier(rowType) || hasLinkCountedModifier(rowType))
- throwUnexpected();
- return false;
- }
- case type_row:
- throwUnexpected();
- }
- return false;
- }
- bool hasLinkedRow(ITypeInfo * t)
- {
- switch (t->getTypeCode())
- {
- case type_table:
- case type_groupedtable:
- case type_dictionary:
- return hasLinkedRow(t->queryChildType());
- case type_row:
- return hasLinkCountedModifier(t);
- }
- return false;
- }
- ITypeInfo * setLinkCountedAttr(ITypeInfo * _type, bool setValue)
- {
- Linked<ITypeInfo> type = _type;
- switch (type->getTypeCode())
- {
- case type_table:
- case type_groupedtable:
- case type_dictionary:
- {
- ITypeInfo * rowType = type->queryChildType();
- Owned<ITypeInfo> newRowType = setLinkCountedAttr(rowType, setValue);
- if (rowType != newRowType)
- type.setown(replaceChildType(type, newRowType));
- break;
- }
- case type_row:
- break;
- default:
- return type.getClear();
- }
- if (hasLinkCountedModifier(type))
- {
- if (setValue)
- return LINK(type);
- return removeProperty(type, _linkCounted_Atom);
- }
- else
- {
- if (setValue)
- return makeAttributeModifier(LINK(type), getLinkCountedAttr());
- return LINK(type);
- }
- }
- ITypeInfo * setStreamedAttr(ITypeInfo * _type, bool setValue)
- {
- Linked<ITypeInfo> type = _type;
- switch (type->getTypeCode())
- {
- case type_groupedtable:
- {
- ITypeInfo * dsType = type->queryChildType();
- Owned<ITypeInfo> newDsType = setStreamedAttr(dsType, setValue);
- if (dsType != newDsType)
- type.setown(replaceChildType(type, newDsType));
- break;
- }
- case type_table:
- break;
- default:
- return type.getClear();
- }
- if (hasStreamedModifier(type))
- {
- if (setValue)
- return LINK(type);
- return removeProperty(type, streamedAtom);
- }
- else
- {
- if (setValue)
- return makeAttributeModifier(LINK(type), getStreamedAttr());
- return LINK(type);
- }
- }
- //---------------------------------------------------------------------------------------------------------------------
- IHqlExpression * CHqlExpression::queryExistingAttribute(ExprPropKind propKind) const
- {
- CriticalBlock block(*attributeCS);
- CHqlDynamicAttribute * cur = attributes;
- while (cur)
- {
- if (cur->kind == propKind)
- {
- IHqlExpression * value = cur->value;
- if (value)
- return value;
- return const_cast<CHqlExpression *>(this);
- }
- cur = cur->next;
- }
- return NULL;
- }
- void CHqlExpression::addAttribute(ExprPropKind kind, IHqlExpression * value)
- {
- if (value == this)
- value = NULL;
- CriticalBlock block(*attributeCS);
- //theoretically we should test if the attribute has already been added by another thread, but in practice there is no
- //problem if the attribute is present twice.
- CHqlDynamicAttribute * attr = new CHqlDynamicAttribute(kind, value);
- attr->next = attributes;
- attributes = attr;
- }
- IHqlExpression * CHqlExpression::queryAttribute(ExprPropKind kind)
- {
- IHqlExpression * match = queryExistingAttribute(kind);
- if (match)
- return match;
- switch (kind)
- {
- case EPrecordCount:
- return evaluateAttrRecordCount(this);
- case EPdiskserializedForm:
- return evaluateAttrSerializedForm(this, kind, diskAtom);
- case EPinternalserializedForm:
- return evaluateAttrSerializedForm(this, kind, internalAtom);
- case EPsize:
- return evaluateAttrSize(this);
- case EPaligned:
- return evaluateAttrAligned(this);
- case EPunadorned:
- return evaluateAttrUnadorned(this);
- case EPlocationIndependent:
- return evaluateAttrLocationIndependent(this);
- }
- return NULL;
- }
|