hqlinline.cpp 75 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175
  1. /*##############################################################################
  2. HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. ############################################################################## */
  13. #include "jliball.hpp"
  14. #include "hql.hpp"
  15. #include "platform.h"
  16. #include "jlib.hpp"
  17. #include "jmisc.hpp"
  18. #include "jstream.ipp"
  19. #include "jdebug.hpp"
  20. #include "hql.hpp"
  21. #include "hqlcpp.ipp"
  22. #include "hqlhtcpp.ipp"
  23. #include "hqlutil.hpp"
  24. #include "hqlcpputil.hpp"
  25. #include "hqltcppc.ipp"
  26. #include "hqlwcpp.hpp"
  27. #include "hqlcatom.hpp"
  28. #include "hqlccommon.hpp"
  29. #include "hqlcerrors.hpp"
  30. #include "hqlpmap.hpp"
  31. #include "hqlinline.hpp"
  32. //I don't think the following is needed yet, but when it is, just enable following...
  33. //#define ONSTART_IN_NESTED_TOPLEVEL
  34. /*
  35. Allow the following operations to be evaluated in line:
  36. a) project of an dataset that can be iterated
  37. */
  38. enum
  39. {
  40. HEFprocessinline = 0x0001,
  41. HEFassigninline = 0x0002,
  42. HEFiterateinline = 0x0004,
  43. HEFevaluateinline = 0x0008, // can evaluate without any temporary dataset being created (temporary row is ok)
  44. HEFspillinline = 0x0010, // I'm not sure I can do this - because whether it spills depends on how it is being used.
  45. RETassign = HEFassigninline|HEFprocessinline,
  46. RETiterate = HEFiterateinline|HEFassigninline|HEFprocessinline,
  47. RETevaluate = HEFevaluateinline|HEFiterateinline|HEFassigninline|HEFprocessinline,
  48. };
  49. #define canAssignNoSpill(childFlags) ((childFlags & (HEFspillinline|HEFassigninline)) == HEFassigninline)
  50. #define canIterateNoSpill(childFlags) ((childFlags & (HEFspillinline|HEFiterateinline)) == HEFiterateinline)
  51. #define canEvaluateNoSpill(childFlags) ((childFlags & (HEFspillinline|HEFevaluateinline)) == HEFevaluateinline)
  52. // assign is superset of iterate, iterate is a superset of evaluate
  53. static unsigned getInlineFlags(BuildCtx * ctx, IHqlExpression * expr);
  54. static unsigned calcInlineFlags(BuildCtx * ctx, IHqlExpression * expr)
  55. {
  56. //The following improves a few graphs, but sometimes significantly (e.g., bc10.xhql, seep11.xhql)
  57. //But it would be really good if the code could be made context independent - then it could go in hqlattr and be cached.
  58. if (ctx)
  59. {
  60. if (expr->isDataset() || expr->isDictionary())
  61. {
  62. if (ctx->queryMatchExpr(expr))
  63. return RETevaluate;
  64. }
  65. else
  66. {
  67. if (ctx->queryAssociation(expr, AssocRow, NULL))
  68. return RETevaluate;
  69. }
  70. }
  71. node_operator op = expr->getOperator();
  72. switch (op)
  73. {
  74. case no_workunit_dataset:
  75. if (expr->hasAttribute(wuidAtom))
  76. return 0;
  77. return RETassign;
  78. case no_alias:
  79. {
  80. unsigned childFlags = getInlineFlags(ctx, expr->queryChild(0));
  81. if (childFlags == 0)
  82. return 0;
  83. if (isGrouped(expr))
  84. return RETevaluate;
  85. return RETevaluate|RETiterate;
  86. }
  87. case no_dataset_alias:
  88. return getInlineFlags(ctx, expr->queryChild(0));
  89. }
  90. if (isGrouped(expr))
  91. return 0;
  92. switch (op)
  93. {
  94. case no_select:
  95. //MORE: What about child datasets in nested records?
  96. if (isNewSelector(expr))
  97. {
  98. unsigned childFlags = getInlineFlags(ctx, expr->queryChild(0));
  99. if ((childFlags & HEFevaluateinline) && isMultiLevelDatasetSelector(expr, false))
  100. childFlags &= ~HEFevaluateinline;
  101. return childFlags;
  102. }
  103. return RETevaluate;
  104. case no_newaggregate:
  105. {
  106. unsigned childFlags = getInlineFlags(ctx, expr->queryChild(0));
  107. if ((childFlags == 0) || queryRealChild(expr, 3))
  108. return 0;
  109. if (canIterateNoSpill(childFlags))
  110. return RETevaluate;
  111. return RETevaluate|HEFspillinline;
  112. }
  113. case no_hqlproject:
  114. //can't do a skip inside an inline project - since the generated code doesn't allow "continue" to be used.
  115. if (transformContainsSkip(expr->queryChild(1)))
  116. return 0;
  117. //fallthrough
  118. case no_newusertable:
  119. {
  120. if (expr->hasAttribute(prefetchAtom))
  121. return 0;
  122. IHqlExpression * ds = expr->queryChild(0);
  123. unsigned childFlags = getInlineFlags(ctx, ds);
  124. if (childFlags == 0)
  125. return 0;
  126. if (hasSingleRow(ds))
  127. {
  128. if (canEvaluateNoSpill(childFlags))
  129. return RETevaluate;
  130. return RETevaluate|HEFspillinline;
  131. }
  132. if (canIterateNoSpill(childFlags))
  133. return RETiterate;
  134. return RETiterate|HEFspillinline;
  135. }
  136. case no_selectmap:
  137. case no_selectnth:
  138. {
  139. IHqlExpression * ds = expr->queryChild(0);
  140. unsigned childFlags = getInlineFlags(ctx, ds);
  141. if (childFlags == 0)
  142. {
  143. if (isSelectSortedTop(expr))
  144. {
  145. childFlags = getInlineFlags(ctx, ds->queryChild(0));
  146. if (childFlags == 0)
  147. return 0;
  148. if (canIterateNoSpill(childFlags))
  149. return RETevaluate;
  150. return RETevaluate|HEFspillinline;
  151. }
  152. return 0;
  153. }
  154. if (isTrivialSelectN(expr) && canEvaluateNoSpill(childFlags))
  155. return RETevaluate;
  156. if (canIterateNoSpill(childFlags))
  157. return RETevaluate;
  158. return RETevaluate|HEFspillinline;
  159. }
  160. case no_filter:
  161. {
  162. unsigned childFlags = getInlineFlags(ctx, expr->queryChild(0));
  163. if (childFlags == 0)
  164. return 0;
  165. if (canIterateNoSpill(childFlags))
  166. return RETiterate;
  167. if (filterIsTableInvariant(expr) && canAssignNoSpill(childFlags))
  168. return RETassign;
  169. return RETiterate|HEFspillinline;
  170. }
  171. case no_choosen:
  172. case no_index:
  173. {
  174. unsigned childFlags = getInlineFlags(ctx, expr->queryChild(0));
  175. if (childFlags == 0)
  176. return 0;
  177. if (canIterateNoSpill(childFlags))
  178. return RETiterate;
  179. return RETiterate|HEFspillinline;
  180. }
  181. case no_limit:
  182. {
  183. if (expr->hasAttribute(skipAtom) || expr->hasAttribute(onFailAtom))
  184. return 0;
  185. unsigned childFlags = getInlineFlags(ctx, expr->queryChild(0));
  186. if (childFlags == 0)
  187. return 0;
  188. unsigned flags = 0;
  189. if (canEvaluateNoSpill(childFlags))
  190. flags |= RETevaluate;
  191. if (canIterateNoSpill(childFlags))
  192. flags |= RETiterate;
  193. if (flags)
  194. return flags;
  195. return RETevaluate|HEFspillinline;
  196. }
  197. case no_fail:
  198. return RETevaluate;
  199. case no_catchds:
  200. return 0; // for the moment always do this out of line
  201. case no_table:
  202. return 0;
  203. case no_createdictionary:
  204. return RETassign;
  205. case no_datasetfromdictionary:
  206. return RETiterate;
  207. case no_owned_ds:
  208. {
  209. unsigned childFlags = getInlineFlags(ctx, expr->queryChild(0));
  210. if (childFlags == 0)
  211. return 0;
  212. return RETevaluate|HEFspillinline;
  213. }
  214. case no_addfiles:
  215. {
  216. unsigned ret = RETassign;
  217. for (unsigned i=0; i < 2; i++)
  218. {
  219. unsigned childFlags = getInlineFlags(ctx, expr->queryChild(i));
  220. if (childFlags == 0)
  221. return 0;
  222. if (childFlags & HEFspillinline)
  223. ret |= HEFspillinline;
  224. }
  225. return ret;
  226. }
  227. case no_if:
  228. case no_choose:
  229. case no_chooseds:
  230. {
  231. unsigned ret = expr->isDatarow() ? RETevaluate : RETassign;
  232. ForEachChildFrom(i, expr, 1)
  233. {
  234. IHqlExpression * cur = expr->queryChild(i);
  235. if (cur->isAttribute())
  236. continue;
  237. unsigned childFlags = getInlineFlags(ctx, cur);
  238. if (childFlags == 0)
  239. return 0;
  240. if (childFlags & HEFspillinline)
  241. ret |= HEFspillinline;
  242. }
  243. return ret;
  244. }
  245. case no_compound_childread:
  246. case no_compound_childnormalize:
  247. case no_compound_childaggregate:
  248. case no_compound_selectnew:
  249. case no_compound_inline:
  250. case no_datasetfromrow:
  251. case no_sorted:
  252. case no_distributed:
  253. case no_preservemeta:
  254. case no_nofold:
  255. case no_nohoist:
  256. case no_alias_scope:
  257. case no_serialize:
  258. case no_deserialize:
  259. case no_dataset_alias:
  260. return getInlineFlags(ctx, expr->queryChild(0));
  261. case no_forcegraph:
  262. return 0;
  263. case no_section:
  264. if (expr->hasAttribute(graphAtom)) // force it to appear in the graph
  265. return 0;
  266. return getInlineFlags(ctx, expr->queryChild(0));
  267. case no_call:
  268. case no_externalcall: // no so sure about this - should possibly be assignable only. (also no_call above)
  269. case no_getresult:
  270. return expr->isDatarow() ? RETevaluate : RETassign;
  271. case no_getgraphresult:
  272. if (expr->hasAttribute(_distributed_Atom))
  273. return 0;
  274. return expr->isDatarow() ? RETevaluate : RETassign;
  275. case no_temptable:
  276. return RETassign;
  277. case no_xmlproject:
  278. return RETiterate;
  279. case no_dataset_from_transform:
  280. {
  281. if (expr->hasAttribute(distributedAtom))
  282. return 0;
  283. if (transformContainsSkip(expr->queryChild(1)))
  284. return 0;
  285. return RETassign;
  286. }
  287. case no_inlinetable:
  288. {
  289. IHqlExpression * transforms = expr->queryChild(0);
  290. if (transformListContainsSkip(transforms))
  291. return 0;
  292. if (isConstantDataset(expr))
  293. return RETevaluate;
  294. return RETassign;
  295. }
  296. case no_createrow:
  297. //MORE: We should probably be able to handle this...
  298. if (containsSkip(expr->queryChild(0)))
  299. return 0;
  300. return RETevaluate;
  301. case no_translated:
  302. return RETassign|RETiterate|RETevaluate;
  303. case no_projectrow:
  304. {
  305. unsigned childFlags = getInlineFlags(ctx, expr->queryChild(0));
  306. if (childFlags == 0)
  307. return 0;
  308. return RETevaluate;
  309. }
  310. case no_null:
  311. case no_temprow:
  312. case no_left:
  313. case no_right:
  314. case no_top:
  315. case no_id2blob:
  316. case no_activerow:
  317. case no_typetransfer:
  318. case no_rows:
  319. case no_skip:
  320. case no_matchattr:
  321. case no_matchrow:
  322. case no_libraryinput:
  323. case no_fromxml:
  324. return RETevaluate;
  325. case no_apply:
  326. {
  327. unsigned childFlags = getInlineFlags(ctx, expr->queryChild(0));
  328. if (childFlags == 0)
  329. return 0;
  330. return RETevaluate|(childFlags & HEFspillinline);
  331. }
  332. case no_activetable:
  333. return RETassign;
  334. case no_join:
  335. {
  336. if (!expr->hasAttribute(allAtom) || isKeyedJoin(expr))
  337. return 0;
  338. //conservatively check we support the attributes.
  339. unsigned max = expr->numChildren();
  340. for (unsigned i=4; i < max; i++)
  341. {
  342. IHqlExpression * cur = expr->queryChild(i);
  343. if (!cur->isAttribute())
  344. return 0;
  345. IAtom * name = cur->queryName();
  346. //possibly implement keep as well. (Local on a child join does nothing.)
  347. if ((name != leftouterAtom) && (name != leftonlyAtom) && (name != innerAtom) && (name != allAtom) && (name != localAtom) && !isInternalAttributeName(name))
  348. return 0;
  349. }
  350. unsigned childLFlags = getInlineFlags(ctx, expr->queryChild(0));
  351. unsigned childRFlags = getInlineFlags(ctx, expr->queryChild(1));
  352. if ((childLFlags == 0) || (childRFlags == 0))
  353. return 0;
  354. return RETassign|((childLFlags|childRFlags) & HEFspillinline);
  355. }
  356. case no_compound:
  357. return getInlineFlags(ctx, expr->queryChild(1));
  358. default:
  359. return 0;
  360. }
  361. }
  362. unsigned getInlineFlags(BuildCtx * ctx, IHqlExpression * expr)
  363. {
  364. //This could one day use flags stored in CHqlExpression if it was considered necessary.
  365. return calcInlineFlags(ctx, expr);
  366. }
  367. bool canProcessInline(BuildCtx * ctx, IHqlExpression * expr)
  368. {
  369. return getInlineFlags(ctx, expr) != 0;
  370. }
  371. bool canIterateInline(BuildCtx * ctx, IHqlExpression * expr)
  372. {
  373. return canIterateNoSpill(getInlineFlags(ctx, expr));
  374. }
  375. bool canEvaluateInline(BuildCtx * ctx, IHqlExpression * expr)
  376. {
  377. return canEvaluateNoSpill(getInlineFlags(ctx, expr));
  378. }
  379. bool canAssignInline(BuildCtx * ctx, IHqlExpression * expr)
  380. {
  381. return canAssignNoSpill(getInlineFlags(ctx, expr));
  382. }
  383. bool canAssignNotEvaluateInline(BuildCtx * ctx, IHqlExpression * expr)
  384. {
  385. unsigned flags = getInlineFlags(ctx, expr);
  386. return canAssignNoSpill(flags) && !canEvaluateNoSpill(flags);
  387. }
  388. bool alwaysEvaluatesToBound(IHqlExpression * expr)
  389. {
  390. switch (expr->getOperator())
  391. {
  392. case no_select:
  393. return !isMultiLevelDatasetSelector(expr, false);
  394. case no_owned_ds:
  395. case no_translated:
  396. case no_alias:
  397. case no_call:
  398. case no_externalcall: // no so sure about this - should possibly be assignable only. (also no_call above)
  399. case no_getresult:
  400. case no_getgraphresult:
  401. case no_workunit_dataset:
  402. case no_activetable:
  403. return !hasStreamedModifier(expr->queryType());
  404. case no_temptable:
  405. //MORE! when we have constant datasets
  406. default:
  407. return canEvaluateInline(NULL, expr);
  408. }
  409. }
  410. //============================================================================
  411. //============================================================================
  412. /*
  413. Notes on Graph distribution/localisation etc.
  414. There are several different things to take into account when noting the localisation of an activity, including the problem
  415. that meaning of "local" is overloaded. Currently we have something like the following, for each logical instance of an activity:
  416. Roxie
  417. SingleNode - only evaluated on a single node
  418. [Split, filter, project, rollup, sort, simpleresult]
  419. SingleNodeLocal - evaluated on a single node, and only access local files.
  420. [local keyed join, local index read]
  421. PrimarySlave - Evaluated on many nodes, only using local file parts [if any], interaction between nodes may occur on the primary
  422. [keyed join, index read, remote]
  423. Thor
  424. Master - only evalauted on the master
  425. [simpleresult]
  426. Distributed - Evaluated on many nodes
  427. [sort, distribute, group]
  428. DistributedIndependent - Evaluated on many nodes, no interaction between nodes
  429. [local sort, local group]
  430. DistributedLocal- Evaluated on many nodes using local file parts
  431. [local index read, local keyed join]
  432. HThor only has singlenode as an option
  433. Child Queries can be essentially be any of the roxie options. In addition there is the question of where the child will be
  434. evaluated in relation to its parent. The options are
  435. NoAccess - no context dependent information in the helper
  436. CoLocal - Will be evaluated on the same slave node as the parent
  437. NonLocal - May be evaluated on a different slave node from the parent
  438. We'll use the following logical flags to keep track of the different options:
  439. _singlenode_|_independentnodes_|_
  440. independent - no interaction between records on the different nodes.
  441. correlated - interaction between records on the different nodes.
  442. _localparts_|_allparts_
  443. - access local file parts or all file parts.
  444. allfile - access all file parts.
  445. local - either access local file parts, or no interaction between nodes.
  446. Ideally we want the expression tree to be independent of the current engine type, and so that derived attributes and optimizations
  447. can work independently of the engine if possible.
  448. => spill needs to indicate whether it is a purely memory spill
  449. => sort order and local interactions need resolving.
  450. */
  451. inline GraphLocalisation mergeLocalisation(GraphLocalisation l, GraphLocalisation r)
  452. {
  453. //options are GraphCoLocal, GraphNonLocal, GraphCoNonLocal, GraphNoAccess
  454. if (l == GraphNeverAccess)
  455. l = GraphNoAccess;
  456. if (r == GraphNeverAccess)
  457. r = GraphNoAccess;
  458. if (l == GraphNoAccess)
  459. return r;
  460. if (r == GraphNoAccess)
  461. return l;
  462. if ((l == GraphCoNonLocal) || (r == GraphCoNonLocal))
  463. return GraphCoNonLocal;
  464. if (l != r)
  465. return GraphCoNonLocal;
  466. return l;
  467. }
  468. bool exprExcludingInputNeedsParent(IHqlExpression * expr)
  469. {
  470. unsigned first = getFirstActivityArgument(expr);
  471. unsigned last = first + getNumActivityArguments(expr);
  472. ForEachChild(i, expr)
  473. {
  474. if (i >= first && i < last)
  475. continue;
  476. IHqlExpression * cur = expr->queryChild(i);
  477. if (isGraphDependent(cur) || usesRuntimeContext(cur))
  478. return true;
  479. }
  480. if (!expr->isIndependentOfScopeIgnoringInputs())
  481. return true;
  482. return false;
  483. }
  484. bool exprIncludingInputNeedsParent(IHqlExpression * expr)
  485. {
  486. if (isGraphDependent(expr) || usesRuntimeContext(expr))
  487. return true;
  488. if (!expr->isIndependentOfScope())
  489. return true;
  490. return false;
  491. }
  492. bool activityNeedsParent(IHqlExpression * expr)
  493. {
  494. //This should always err on the side of yes...
  495. switch (expr->getOperator())
  496. {
  497. case no_select:
  498. case no_compound_diskread:
  499. case no_compound_disknormalize:
  500. case no_compound_diskaggregate:
  501. case no_compound_diskcount:
  502. case no_compound_diskgroupaggregate:
  503. case no_compound_indexread:
  504. case no_compound_indexnormalize:
  505. case no_compound_indexaggregate:
  506. case no_compound_indexcount:
  507. case no_compound_indexgroupaggregate:
  508. if (exprIncludingInputNeedsParent(expr))
  509. return true;
  510. break;
  511. case no_libraryinput:
  512. return true;
  513. case no_hqlproject:
  514. case no_newusertable:
  515. //Filters might be merged into projects, so need to walk the inputs.
  516. loop
  517. {
  518. if (activityNeedsParent(expr))
  519. return true;
  520. expr = expr->queryChild(0);
  521. switch (expr->getOperator())
  522. {
  523. case no_sorted:
  524. case no_filter:
  525. break;
  526. default:
  527. return false;
  528. }
  529. }
  530. case no_libraryscopeinstance:
  531. {
  532. //Obscure way to get at the name of the library being called.
  533. IHqlExpression * moduleFunction = expr->queryBody()->queryDefinition();
  534. IHqlExpression * module = moduleFunction->queryChild(0);
  535. assertex(module->getOperator() == no_libraryscope);
  536. IHqlExpression * nameAttr = module->queryAttribute(nameAtom);
  537. if (activityNeedsParent(nameAttr))
  538. return true;
  539. return exprExcludingInputNeedsParent(expr);
  540. }
  541. default:
  542. if (expr->isDatarow())
  543. {
  544. switch (expr->getOperator())
  545. {
  546. case no_split:
  547. case no_spill:
  548. case no_selectnth:
  549. if (exprExcludingInputNeedsParent(expr))
  550. return true;
  551. break;
  552. default:
  553. if (exprIncludingInputNeedsParent(expr))
  554. return true;
  555. break;
  556. }
  557. }
  558. else
  559. {
  560. if (exprExcludingInputNeedsParent(expr))
  561. return true;
  562. }
  563. break;
  564. }
  565. //Assume the worse for queries based on ds.childdataset
  566. if (getChildDatasetType(expr) & childdataset_hasdataset)
  567. {
  568. IHqlExpression * ds = queryRoot(expr);
  569. if (ds && ds->getOperator() == no_select)
  570. return true;
  571. }
  572. return false;
  573. }
  574. // A minimal function to catch the obviously context invariant items.
  575. static bool accessesData(IHqlExpression * expr)
  576. {
  577. switch (expr->getOperator())
  578. {
  579. case no_attr:
  580. case no_constant:
  581. case no_record:
  582. return false;
  583. case no_assign:
  584. return accessesData(expr->queryChild(1));
  585. case no_assignall:
  586. case no_transform:
  587. case no_newtransform:
  588. case no_transformlist:
  589. case no_attr_expr:
  590. case no_fail:
  591. {
  592. ForEachChild(i, expr)
  593. {
  594. if (accessesData(expr->queryChild(i)))
  595. return true;
  596. }
  597. return false;
  598. }
  599. default:
  600. return true;
  601. }
  602. }
  603. GraphLocalisation queryActivityLocalisation(IHqlExpression * expr, bool optimizeParentAccess)
  604. {
  605. node_operator op = expr->getOperator();
  606. switch (op)
  607. {
  608. case no_compound_diskread:
  609. {
  610. if (isLocalActivity(expr) || expr->hasAttribute(_colocal_Atom))
  611. return GraphCoLocal;
  612. //If a compound operation has been added, but with no other effect then don't allow that to change the localisation
  613. IHqlExpression * ds = expr->queryChild(0);
  614. if (ds->getOperator() == no_table)
  615. return queryActivityLocalisation(ds, optimizeParentAccess);
  616. return GraphNonLocal;
  617. }
  618. case no_table:
  619. if (expr->hasAttribute(_noAccess_Atom))
  620. return GraphNeverAccess;
  621. //fallthrough
  622. case no_keyindex:
  623. case no_newkeyindex:
  624. case no_compound_disknormalize:
  625. case no_compound_diskaggregate:
  626. case no_compound_diskcount:
  627. case no_compound_diskgroupaggregate:
  628. case no_compound_indexread:
  629. case no_compound_indexnormalize:
  630. case no_compound_indexaggregate:
  631. case no_compound_indexcount:
  632. case no_compound_indexgroupaggregate:
  633. if (!isLocalActivity(expr) && !expr->hasAttribute(_colocal_Atom))
  634. return GraphNonLocal;
  635. break;
  636. case no_compound_fetch:
  637. case no_fetch:
  638. //Maybe there should be a version that only fetches locally???
  639. return GraphNonLocal; // simplest if we do it this way
  640. case no_allnodes:
  641. return GraphNonLocal; // simplest if we do it this way
  642. case no_join:
  643. case no_denormalize:
  644. case no_denormalizegroup:
  645. if (isKeyedJoin(expr) && !expr->hasAttribute(localAtom))
  646. return GraphNonLocal;
  647. break;
  648. case no_output:
  649. if (expr->hasAttribute(_spill_Atom))
  650. return GraphNeverAccess;
  651. break;
  652. case no_setgraphresult:
  653. case no_spillgraphresult:
  654. case no_setgraphloopresult:
  655. case no_null:
  656. case no_split:
  657. case no_spill:
  658. case no_readspill:
  659. case no_writespill:
  660. case no_commonspill:
  661. case no_addfiles:
  662. case no_subgraph:
  663. case no_nofold:
  664. case no_nohoist:
  665. case no_regroup:
  666. case no_sorted:
  667. case no_distributed:
  668. case no_preservemeta:
  669. case no_grouped:
  670. case no_alias_scope:
  671. case no_sequential:
  672. case no_parallel:
  673. case no_deserialize:
  674. case no_serialize:
  675. case no_actionlist:
  676. case no_orderedactionlist:
  677. case no_definesideeffect:
  678. case no_dataset_alias:
  679. return GraphNeverAccess; // Will never access any data values from anywhere
  680. case no_hqlproject:
  681. case no_newusertable:
  682. //Many more of these could return GraphNoAccess if I determined that only constants and the input
  683. //row were accessed. Examples are project, sort, group etc. etc.
  684. //if (isSimpleProject(expr))
  685. /// return GraphNoAccess;
  686. return GraphCoLocal;
  687. case no_datasetfromrow:
  688. return GraphNeverAccess;
  689. case no_workunit_dataset:
  690. return GraphCoLocal; // weird exception in roxie
  691. case no_getgraphresult:
  692. case no_datasetfromdictionary:
  693. return GraphCoLocal;
  694. case no_createrow:
  695. case no_inlinetable:
  696. {
  697. unsigned max = expr->numChildren();
  698. for (unsigned i=0; i<max; i++)
  699. {
  700. if (accessesData(expr->queryChild(i)))
  701. return GraphCoLocal;
  702. }
  703. return GraphNeverAccess;
  704. }
  705. case no_group:
  706. case no_choosen:
  707. case no_limit:
  708. case no_catchds:
  709. case no_selectnth:
  710. {
  711. unsigned max = expr->numChildren();
  712. for (unsigned i=1; i<max; i++)
  713. {
  714. if (accessesData(expr->queryChild(i)))
  715. return GraphCoLocal;
  716. }
  717. return GraphNeverAccess;
  718. }
  719. case no_newaggregate:
  720. if (!queryRealChild(expr, 3))
  721. {
  722. node_operator op = querySimpleAggregate(expr, false, false);
  723. if (op == no_existsgroup || op == no_countgroup)
  724. return GraphNeverAccess;
  725. //Need to check if it accesses anything in the context!
  726. }
  727. break;
  728. }
  729. if (optimizeParentAccess && !activityNeedsParent(expr))
  730. {
  731. switch (op)
  732. {
  733. case no_if:
  734. case no_case:
  735. case no_map:
  736. //May be combined into a switch.
  737. break;
  738. default:
  739. return GraphNoAccess;
  740. }
  741. }
  742. return GraphCoLocal;
  743. }
  744. bool isNonLocal(IHqlExpression * expr, bool optimizeParentAccess)
  745. {
  746. return (queryActivityLocalisation(expr, optimizeParentAccess) == GraphNonLocal);
  747. }
  748. static GraphLocalisation doGetGraphLocalisation(IHqlExpression * expr, bool optimizeParentAccess);
  749. static GraphLocalisation queryGraphLocalisation(IHqlExpression * expr, bool optimizeParentAccess)
  750. {
  751. GraphLocalisation localisation = queryActivityLocalisation(expr, optimizeParentAccess);
  752. if (isSourceActivity(expr))
  753. return localisation;
  754. unsigned firstChild = 0;
  755. unsigned numChildren;
  756. switch (expr->getOperator())
  757. {
  758. case no_if:
  759. case no_chooseds:
  760. firstChild = 1;
  761. numChildren = expr->numChildren();
  762. break;
  763. case no_compound:
  764. case no_subgraph:
  765. numChildren = expr->numChildren();
  766. break;
  767. case no_attr:
  768. case no_attr_expr:
  769. return GraphNeverAccess;
  770. default:
  771. numChildren = getNumActivityArguments(expr);
  772. break;
  773. }
  774. for (unsigned i = firstChild; i < numChildren; i++)
  775. {
  776. localisation = mergeLocalisation(localisation, doGetGraphLocalisation(expr->queryChild(i), optimizeParentAccess));
  777. if (localisation == GraphCoNonLocal)
  778. return localisation;
  779. }
  780. return localisation;
  781. }
  782. static GraphLocalisation doGetGraphLocalisation(IHqlExpression * expr, bool optimizeParentAccess)
  783. {
  784. IHqlExpression * cached = (IHqlExpression *)expr->queryTransformExtra();
  785. if (cached)
  786. return (GraphLocalisation)cached->queryValue()->getIntValue();
  787. GraphLocalisation ret = queryGraphLocalisation(expr, optimizeParentAccess);
  788. if (ret == GraphNeverAccess)
  789. ret = GraphNoAccess;
  790. expr->setTransformExtraOwned(getSizetConstant((unsigned)ret));
  791. return ret;
  792. }
  793. static GraphLocalisation getGraphLocalisation(IHqlExpression * expr, bool optimizeParentAccess)
  794. {
  795. TransformMutexBlock lock;
  796. return doGetGraphLocalisation(expr, optimizeParentAccess);
  797. }
  798. bool HqlCppTranslator::isAlwaysCoLocal()
  799. {
  800. return targetHThor();
  801. }
  802. GraphLocalisation HqlCppTranslator::getGraphLocalisation(IHqlExpression * expr, bool isInsideChildQuery)
  803. {
  804. if (isAlwaysCoLocal()) return GraphCoLocal;
  805. if (targetThor() && !isInsideChildQuery)
  806. return GraphNonLocal;
  807. return ::getGraphLocalisation(expr, options.optimizeParentAccess);
  808. }
  809. bool HqlCppTranslator::isNeverDistributed(IHqlExpression * expr)
  810. {
  811. if (targetHThor())
  812. return true;
  813. switch (expr->getOperator())
  814. {
  815. case no_if:
  816. return expr->isAction();
  817. case no_setresult:
  818. return true;
  819. }
  820. return false;
  821. }
  822. //============================================================================
  823. ParentExtract::ParentExtract(HqlCppTranslator & _translator, PEtype _type, IHqlExpression * _graphId, GraphLocalisation _localisation, EvalContext * _container)
  824. : HqlExprAssociation(parentExtractMarkerExpr), translator(_translator), type(_type), graphId(_graphId)
  825. {
  826. localisation = _localisation;
  827. container = _container;
  828. if (!container)
  829. throwError(HQLERR_ExpectedParentContext);
  830. Owned<ITypeInfo> nullRow = makeRowType(queryNullRecord()->getType());
  831. Owned<ITypeInfo> declType = makeModifier(makeWrapperModifier(LINK(nullRow)), typemod_builder);
  832. StringBuffer extractName;
  833. translator.getUniqueId(extractName.append("ex"));
  834. boundBuilder.expr.setown(createVariable(extractName.str(), LINK(declType)));
  835. boundExtract.expr.setown(createVariable(extractName.str(), makeReferenceModifier(LINK(nullRow))));
  836. serialization = NULL;
  837. childSerialization = NULL;
  838. canDestroyExtract = false;
  839. }
  840. ParentExtract::~ParentExtract()
  841. {
  842. ::Release(serialization);
  843. }
  844. void ParentExtract::associateCursors(BuildCtx & declarectx, BuildCtx & evalctx, GraphLocalisation childLocalisation)
  845. {
  846. const CursorArray * boundCursors = NULL;
  847. switch (childLocalisation)
  848. {
  849. case GraphCoLocal:
  850. if (localisation == GraphRemote)
  851. boundCursors = &nonlocalBoundCursors;
  852. else if ((localisation == GraphCoLocal) || (localisation == GraphCoNonLocal))
  853. boundCursors = &colocalBoundCursors;
  854. else
  855. throwError2(HQLERR_InconsisentLocalisation, (int)childLocalisation, (int)localisation);
  856. break;
  857. case GraphNonLocal:
  858. case GraphRemote:
  859. if ((localisation == GraphCoLocal) || (localisation == GraphNoAccess))
  860. throwError2(HQLERR_InconsisentLocalisation, (int)childLocalisation, (int)localisation);
  861. boundCursors = &nonlocalBoundCursors;
  862. break;
  863. case GraphCoNonLocal:
  864. throwUnexpected();
  865. case GraphNoAccess:
  866. if (localisation == GraphRemote)
  867. boundCursors = &nonlocalBoundCursors;
  868. else
  869. {
  870. //Make sure we get an error if we do try and access something from the parent.
  871. return;
  872. }
  873. break;
  874. default:
  875. throwUnexpected();
  876. }
  877. ForEachItemIn(i, *boundCursors)
  878. {
  879. BoundRow & cur = boundCursors->item(i);
  880. IHqlExpression * aliasExpansion = cur.queryAliasExpansion();
  881. if (aliasExpansion)
  882. {
  883. //NB: If this ever generates any code then we will need to process it later...
  884. CHqlBoundExpr bound;
  885. translator.buildExpr(evalctx, aliasExpansion, bound);
  886. declarectx.associateOwn(*cur.clone(bound.expr));
  887. }
  888. else
  889. declarectx.associateOwn(OLINK(cur));
  890. }
  891. }
  892. void ParentExtract::beginCreateExtract(BuildCtx & ctx, bool doDeclare)
  893. {
  894. buildctx.setown(new BuildCtx(ctx));
  895. // Don't leak the serialization row into other sub calls - may want to do this a different way so
  896. // cses get commoned up by tagging the serialization somehow.
  897. //Probably do this later and allow it to be null. Will need a mechanism for calling some finalisation code
  898. //after all code is generated in order to do it.
  899. if (doDeclare)
  900. {
  901. BuildCtx * declarectx = buildctx;
  902. translator.getInvariantMemberContext(ctx, &declarectx, NULL, false, false);
  903. declarectx->addDeclare(boundBuilder.expr);
  904. }
  905. serialization = SerializationRow::create(translator, boundBuilder.expr, container ? container->queryActivity() : NULL);
  906. buildctx->associateOwn(*LINK(serialization));
  907. //Ensure the row is large enough to cope with any fixed fields - will only get relocated if variable fields are serialised
  908. HqlExprArray args;
  909. args.append(*LINK(serialization->queryBound()));
  910. args.append(*serialization->getFinalFixedSizeExpr());
  911. translator.callProcedure(*buildctx, ensureRowAvailableId, args);
  912. //Collect a list of cursors together... NB these are in reverse order..
  913. gatherActiveRows(*buildctx);
  914. childSerialization->setBuilder(this);
  915. }
  916. void ParentExtract::beginNestedExtract(BuildCtx & clonectx)
  917. {
  918. //Collect a list of cursors together... NB these are in reverse order..
  919. gatherActiveRows(clonectx);
  920. }
  921. void ParentExtract::beginReuseExtract()
  922. {
  923. //MORE: Should really check that the same rows are active...
  924. //Need to check if any additional rows, and if so bind them.
  925. childSerialization->setBuilder(this);
  926. }
  927. void ParentExtract::beginChildActivity(BuildCtx & declareCtx, BuildCtx & startCtx, GraphLocalisation childLocalisation, IHqlExpression * colocal, bool nested, bool ignoreSelf, ActivityInstance * activityRequiringCast)
  928. {
  929. if (type == PETcallback)
  930. assertex(ignoreSelf);
  931. //MORE: If we ever generate grand children - nested classes then the following isn't going to work
  932. //because accessing colocal->exNNN isn't going to get at the extract defined in the child class+passed to the grandchild.
  933. //Simplest would be to define the builders in the activity, and access them as activity->exXXX in the (grand)child.
  934. bool isColocal = (colocal != NULL);
  935. const CursorArray & boundCursors = isColocal ? colocalBoundCursors : nonlocalBoundCursors;
  936. ForEachItemIn(i, boundCursors)
  937. {
  938. BoundRow & cur = boundCursors.item(i);
  939. if (cur.isSerialization())
  940. {
  941. IHqlExpression * bound = cur.queryBound();
  942. if (!colocal && bound != boundExtract.expr)
  943. break;
  944. //If this is the parent extract for a local callback, it will be passed as a parameter, so it shouldn't
  945. //be added as a member of the child class
  946. if (ignoreSelf && (bound == boundExtract.expr))
  947. continue;
  948. declareCtx.addDeclare(bound);
  949. OwnedHqlExpr src;
  950. if (bound == boundExtract.expr)
  951. {
  952. //MORE: This cast is a hack. We should process const correctly in helper functions etc.
  953. if (!nested)
  954. src.setown(createVariable("(byte *)pe", bound->getType()));
  955. }
  956. if (!src)
  957. {
  958. src.setown(addMemberSelector(bound, colocal));
  959. //yuk yuk yuk.... A nasty solution to a nasty problem.
  960. //If the parent is a nested class with an extract (used by the compound group aggregate)
  961. //then any other extracts are going to be builders in the parent activity, but pointers in the nested
  962. //class. This means we need to add a .getBytes() call to extract the pointer
  963. //Also if we ever have grandchild nested classes they will need to change where the builder is defined.
  964. if (activityRequiringCast && (cur.queryActivity() == activityRequiringCast))
  965. {
  966. HqlExprArray args;
  967. args.append(*LINK(src));
  968. src.setown(translator.bindTranslatedFunctionCall(getBytesFromBuilderId, args));
  969. }
  970. }
  971. startCtx.addAssign(bound, src);
  972. }
  973. }
  974. associateCursors(declareCtx, startCtx, childLocalisation);
  975. }
  976. bool ParentExtract::canEvaluate(IHqlExpression * expr)
  977. {
  978. if (!buildctx)
  979. return false;
  980. return translator.canEvaluateInContext(*buildctx, expr);
  981. }
  982. void ParentExtract::endChildActivity()
  983. {
  984. }
  985. bool ParentExtract::requiresOnStart() const
  986. {
  987. switch (type)
  988. {
  989. case PETchild:
  990. case PETremote:
  991. case PETloop:
  992. case PETlibrary:
  993. return true;
  994. case PETnested:
  995. case PETcallback:
  996. return container->requiresOnStart();
  997. default:
  998. throwUnexpected();
  999. }
  1000. }
  1001. bool ParentExtract::insideChildQuery() const
  1002. {
  1003. switch (type)
  1004. {
  1005. case PETchild:
  1006. return true;
  1007. }
  1008. return container->insideChildQuery();
  1009. }
  1010. bool ParentExtract::areGraphResultsAccessible(IHqlExpression * searchGraphId) const
  1011. {
  1012. if (graphId == searchGraphId)
  1013. return true;
  1014. switch (type)
  1015. {
  1016. case PETloop:
  1017. return container->areGraphResultsAccessible(searchGraphId);
  1018. }
  1019. return false;
  1020. }
  1021. void ParentExtract::endCreateExtract(CHqlBoundExpr & boundExtract)
  1022. {
  1023. //NB: This can be called more than once - so need to be careful about any processing that
  1024. //is done in here.
  1025. childSerialization->setBuilder(NULL);
  1026. CHqlBoundExpr boundSize;
  1027. translator.getRecordSize(*buildctx, serialization->queryDataset(), boundSize);
  1028. boundExtract.length.set(boundSize.expr);
  1029. boundExtract.expr.set(boundBuilder.expr);
  1030. }
  1031. void ParentExtract::endUseExtract(BuildCtx & ctx)
  1032. {
  1033. childSerialization->finalize();
  1034. //Not so sure about the lifetime of this. If the extract was saved for a later occasion (e.g., prefetch project) then may be destroyed too soon.
  1035. if (canDestroyExtract)
  1036. translator.doGenerateMetaDestruct(ctx, serialization->queryDataset(), childSerialization->queryRecord());
  1037. }
  1038. void ParentExtract::buildAssign(IHqlExpression * serializedTarget, IHqlExpression * originalValue)
  1039. {
  1040. translator.buildAssign(*buildctx, serializedTarget, originalValue);
  1041. }
  1042. //This function converts an expression evaluated in the parent to an expression that can be evaluated in the child.
  1043. //Note, all extracts from all parent colocal activities are cloned into the child activity, so same expression is used to access them in the child
  1044. void ParentExtract::ensureAccessible(BuildCtx & ctx, IHqlExpression * expr, const CHqlBoundExpr & bound, CHqlBoundExpr & tgt, IHqlExpression * colocal)
  1045. {
  1046. //MORE: Need to know whether it is
  1047. //a) a member variable b) a serialization member or c) local and needs seerializing.
  1048. //so when then do
  1049. //a) add colcocal-> or create local references to the variables.
  1050. //b) don't need to do anything
  1051. //c) need to serialize and return value from that.
  1052. //Probably need to tag global declares with an attribute - e.g., typemod_member
  1053. //If activity is not colocal then always need to serialize
  1054. if (colocal)
  1055. {
  1056. //I wish I didn't have to do this. wrapper classes have a cast added when they are converted to
  1057. //expressions. I should probably look at removing it, but until that happens they need to be
  1058. //stripped so the member variables get matched correctly.
  1059. IHqlExpression * boundExpr = bound.expr;
  1060. if (boundExpr->getOperator() == no_implicitcast)
  1061. {
  1062. IHqlExpression * uncast = boundExpr->queryChild(0);
  1063. if (hasModifier(uncast->queryType(), typemod_wrapper) &&
  1064. (queryUnqualifiedType(boundExpr->queryType()) == queryUnqualifiedType(uncast->queryType())))
  1065. boundExpr = uncast;
  1066. }
  1067. //If already bound into a serialization then no need to do anything.
  1068. ITypeInfo * serializedModifier = queryModifier(boundExpr->queryType(), typemod_serialized);
  1069. if (serializedModifier)
  1070. {
  1071. //If serialized rebind the expression, rather than using the bound, otherwise temporary variables used for sizes
  1072. //of fields can get lost (see extractbug.hql) for an example
  1073. IHqlExpression * originalMapped = static_cast<IHqlExpression *>(serializedModifier->queryModifierExtra());
  1074. translator.buildAnyExpr(ctx, originalMapped, tgt);
  1075. tgt.expr.setown(addExpressionModifier(tgt.expr, typemod_serialized, originalMapped));
  1076. return;
  1077. }
  1078. //Special case where this extract is used for member functions within the same class
  1079. if (hasModifier(boundExpr->queryType(), typemod_member))
  1080. {
  1081. if (colocal == colocalSameClassPreserveExpr)
  1082. {
  1083. tgt.set(bound);
  1084. return;
  1085. }
  1086. //Need to create new (reference) members in the new class, and code to assign them in
  1087. //the setParent() call.
  1088. tgt.isAll.setown(addMemberSelector(bound.isAll, colocal));
  1089. tgt.count.setown(addMemberSelector(bound.count, colocal));
  1090. tgt.length.setown(addMemberSelector(bound.length, colocal));
  1091. tgt.expr.setown(addMemberSelector(boundExpr, colocal));
  1092. return;
  1093. }
  1094. }
  1095. //nonLocal child and local variables to a colocal child.
  1096. assertex(childSerialization);
  1097. OwnedHqlExpr value = bound.getTranslatedExpr();
  1098. OwnedHqlExpr mapped = childSerialization->ensureSerialized(value, canDestroyExtract ? colocal : NULL, false);
  1099. translator.buildAnyExpr(ctx, mapped, tgt);
  1100. //add a serialized modifier to the type so we can track expressions already serialized, the extra saves the original mappiing
  1101. tgt.expr.setown(addExpressionModifier(tgt.expr, typemod_serialized, mapped));
  1102. ctx.associateExpr(expr, tgt);
  1103. }
  1104. void ParentExtract::addSerializedExpression(IHqlExpression * value, ITypeInfo * type)
  1105. {
  1106. OwnedHqlExpr mapped = childSerialization->addSerializedValue(value, type, NULL, false);
  1107. }
  1108. AliasKind ParentExtract::evaluateExpression(BuildCtx & ctx, IHqlExpression * value, CHqlBoundExpr & tgt, IHqlExpression * colocal, bool evaluateLocally)
  1109. {
  1110. CHqlBoundExpr bound;
  1111. AliasKind kind;
  1112. if (buildctx)
  1113. kind = translator.doBuildAliasValue(*buildctx, value, bound);
  1114. else
  1115. kind = container->evaluateExpression(ctx, value, bound, evaluateLocally);
  1116. if (kind != NotFoundAlias)
  1117. ensureAccessible(ctx, value, bound, tgt, colocal);
  1118. if (!colocal)
  1119. return RuntimeAlias;
  1120. return kind;
  1121. }
  1122. void ParentExtract::gatherActiveRows(BuildCtx & ctx)
  1123. {
  1124. //Collect a list of cursors together... NB these are in reverse order..
  1125. CursorArray activeRows;
  1126. RowAssociationIterator iter(ctx);
  1127. ForEach(iter)
  1128. {
  1129. BoundRow & cur = iter.get();
  1130. if ((cur.querySide() != no_self) && !cur.isBuilder())
  1131. {
  1132. bool ok = true;
  1133. IHqlExpression * represents = cur.queryDataset();
  1134. switch(represents->getOperator())
  1135. {
  1136. case no_null:
  1137. ok = !represents->hasAttribute(clearAtom); // Don't serialize rows used as default clear rows
  1138. break;
  1139. case no_anon:
  1140. ok = !represents->hasAttribute(selfAtom);
  1141. break;
  1142. default:
  1143. if (cur.isResultAlias())
  1144. ok = false;
  1145. //MORE: This should only be done if the child query etc. actually references the datarow.
  1146. //ideally from a colocal activity.
  1147. #if 0
  1148. //Theoretically the following is true. However it can mean that for the cost of serializing an extra 4 bytes here and
  1149. //there you end up serializing several hundred in some other situations.
  1150. //So on balance, worth including them even if strictly unnecessary.
  1151. if (represents->isDatarow() && !isAlwaysActiveRow(represents))
  1152. ok = false;
  1153. #endif
  1154. break;
  1155. }
  1156. if (ok)
  1157. activeRows.append(OLINK(cur));
  1158. }
  1159. }
  1160. //NB: Serialization needs to be added processed first in the child scope
  1161. if (serialization)
  1162. {
  1163. childSerialization = (SerializationRow *)serialization->clone(boundExtract.expr);
  1164. colocalBoundCursors.append(*childSerialization);
  1165. nonlocalBoundCursors.append(*LINK(childSerialization));
  1166. }
  1167. //MORE: Should possibly create two sets of cursors one if children are colocal, other if children aren't
  1168. //so colocal cursors can be used wherever possible. Would change following to localisation != GraphNonLocal
  1169. //and remove else
  1170. if (localisation == GraphCoLocal || localisation == GraphCoNonLocal)
  1171. {
  1172. OwnedHqlExpr colocal = createQuoted("colocal", makeVoidType());
  1173. ForEachItemInRev(i, activeRows)
  1174. {
  1175. BoundRow & cur = activeRows.item(i);
  1176. IHqlExpression * bound = cur.queryBound();
  1177. OwnedHqlExpr colocalBound = addMemberSelector(bound, colocal);
  1178. BoundRow * newRow = NULL;
  1179. if (cur.isSerialization() || cur.queryDataset()->getOperator() == no_pseudods)
  1180. {
  1181. //an extract/serialization. Same cursor is added to all colocal children.
  1182. newRow = LINK(&cur);
  1183. }
  1184. else if (cur.queryAliasExpansion() || cur.isNonLocal())
  1185. {
  1186. //A cursor who's pointer has already been serialized into an extract
  1187. //NB: Alias expansions get rebound when they are bound into the context.
  1188. newRow = LINK(&cur);
  1189. }
  1190. else if (!cur.isBinary())
  1191. {
  1192. //CSV and xml datasets need their elements serialized into the parent extract
  1193. newRow = new NonLocalIndirectRow(cur, NULL, childSerialization);
  1194. }
  1195. else if (serialization)
  1196. {
  1197. //A cursor active in the current scope => add it to the extract, and create an alias in the
  1198. //child contexts.
  1199. //Need to add these fields to the extract record, and do the assignments at the point of call
  1200. Owned<ITypeInfo> fieldType = makeRowReferenceType(cur.queryDataset());
  1201. if (hasOutOfLineModifier(bound->queryType()))
  1202. fieldType.setown(makeAttributeModifier(LINK(fieldType), getLinkCountedAttr()));
  1203. OwnedHqlExpr expandedAlias = serialization->createField(NULL, fieldType);
  1204. OwnedHqlExpr castSource = createValue(no_implicitcast, LINK(fieldType), LINK(bound));
  1205. OwnedHqlExpr translated = createTranslated(castSource);
  1206. translator.buildAssign(ctx, expandedAlias, translated);
  1207. newRow = new BoundAliasRow(cur, NULL, expandedAlias);
  1208. newRow->setInherited(true);
  1209. }
  1210. else
  1211. {
  1212. //A row defined in a activity class instance, being used from a nested class.
  1213. //E.g., row used to clear a value. Should be possible to recreate in the nested class
  1214. if (cur.getKind() != AssocRow)
  1215. throwUnexpected();
  1216. }
  1217. if (newRow)
  1218. colocalBoundCursors.append(* newRow);
  1219. }
  1220. //May also need to associate any counters in scope. May be more than one!
  1221. //Possibly other things like filepositions, ..... How do I recognise them all?
  1222. //Use a dynamic offset map to represent whatever is serialized.
  1223. //MORE: This may actually mean we want to bind these non-row contexts on demand - but then it is harder to share.
  1224. }
  1225. if (localisation != GraphCoLocal)
  1226. {
  1227. ForEachItemInRev(i, activeRows)
  1228. {
  1229. BoundRow & cur = activeRows.item(i);
  1230. nonlocalBoundCursors.append(*new NonLocalIndirectRow(cur, NULL, childSerialization));
  1231. }
  1232. }
  1233. }
  1234. //----------------------------------------------------------------------------
  1235. ParentExtract * HqlCppTranslator::createExtractBuilder(BuildCtx & ctx, PEtype type, IHqlExpression * graphId, GraphLocalisation localisation, bool doDeclare)
  1236. {
  1237. ParentExtract * extractor = NULL;
  1238. // if (localisation == GraphCoLocal)
  1239. // extract = checkForPreexistingExtract - find a bound association before a row association is found;
  1240. if (!extractor)
  1241. {
  1242. extractor = new ParentExtract(*this, type, graphId, localisation, queryEvalContext(ctx));
  1243. extractor->beginCreateExtract(ctx, doDeclare);
  1244. }
  1245. else
  1246. extractor->beginReuseExtract();
  1247. return extractor;
  1248. }
  1249. ParentExtract * HqlCppTranslator::createExtractBuilder(BuildCtx & ctx, PEtype type, IHqlExpression * graphId, IHqlExpression * expr, bool doDeclare)
  1250. {
  1251. if (isAlwaysCoLocal())
  1252. return createExtractBuilder(ctx, type, graphId, GraphCoLocal, true);
  1253. bool isInsideChildQuery = (type == PETchild) || insideChildQuery(ctx);
  1254. return createExtractBuilder(ctx, type, graphId, getGraphLocalisation(expr, isInsideChildQuery), true);
  1255. }
  1256. //----------------------------------------------------------------------------
  1257. AliasKind HqlCppTranslator::buildExprInCorrectContext(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt, bool evaluateLocally)
  1258. {
  1259. EvalContext * instance = queryEvalContext(ctx);
  1260. if (instance)
  1261. return instance->evaluateExpression(ctx, expr, tgt, evaluateLocally);
  1262. if (ctx.getMatchExpr(expr, tgt))
  1263. return RuntimeAlias;
  1264. return NotFoundAlias;
  1265. }
  1266. //---------------------------------------------------------------------------
  1267. void CtxCollection::createFunctionStructure(HqlCppTranslator & translator, BuildCtx & ctx, bool canEvaluate, const char * serializeFunc)
  1268. {
  1269. clonectx.set(ctx);
  1270. clonectx.addGroup();
  1271. if (serializeFunc)
  1272. {
  1273. BuildCtx condctx(ctx);
  1274. OwnedHqlExpr cond = createVariable("in", makeBoolType());
  1275. IHqlStmt * stmt = condctx.addFilter(cond);
  1276. deserializectx.setown(new BuildCtx(condctx));
  1277. condctx.selectElse(stmt);
  1278. evalctx.setown(new BuildCtx(condctx));
  1279. //virtual void serializeCreateContext(MemoryBuffer & out)
  1280. serializectx.setown(new BuildCtx(declarectx));
  1281. StringBuffer s;
  1282. s.append("virtual void ").append(serializeFunc).append("(MemoryBuffer & out)");
  1283. serializectx->addQuotedCompoundOpt(s.str());
  1284. } else if (canEvaluate)
  1285. {
  1286. evalctx.setown(new BuildCtx(ctx));
  1287. evalctx->addGroup();
  1288. }
  1289. childctx.set(ctx);
  1290. }
  1291. //---------------------------------------------------------------------------
  1292. EvalContext::EvalContext(HqlCppTranslator & _translator, ParentExtract * _parentExtract, EvalContext * _parent)
  1293. : HqlExprAssociation(classMarkerExpr), translator(_translator)
  1294. {
  1295. parent = _parent;
  1296. parentExtract.set(_parentExtract);
  1297. }
  1298. IHqlExpression * EvalContext::createGraphLookup(unique_id_t id, bool isChild)
  1299. {
  1300. assertex(parent);
  1301. return parent->createGraphLookup(id, isChild);
  1302. }
  1303. bool EvalContext::needToEvaluateLocally(BuildCtx & ctx, IHqlExpression * expr)
  1304. {
  1305. return mustEvaluateInContext(ctx, expr);
  1306. }
  1307. bool EvalContext::evaluateInParent(BuildCtx & ctx, IHqlExpression * expr, bool hasOnStart)
  1308. {
  1309. if (!parent)
  1310. return false;
  1311. if (parent->isLibraryContext())
  1312. return (expr->getOperator() == no_libraryinput);
  1313. switch (expr->getOperator())
  1314. {
  1315. case no_libraryinput:
  1316. case no_rows:
  1317. case no_filepos:
  1318. case no_file_logicalname:
  1319. case no_counter:
  1320. case no_variable: // this really should happen
  1321. return true; // would have been bound if found
  1322. case no_id2blob:
  1323. return !queryBlobHelper(ctx, expr->queryChild(0));
  1324. case no_loopcounter:
  1325. return !translator.isCurrentActiveGraph(ctx, expr->queryChild(0));
  1326. case no_getgraphresult:
  1327. return !translator.isCurrentActiveGraph(ctx, expr->queryChild(1));
  1328. case no_getresult:
  1329. case no_workunit_dataset:
  1330. return translator.needToSerializeToSlave(expr);
  1331. case no_failcode:
  1332. case no_failmessage:
  1333. case no_fail:
  1334. return !ctx.queryMatchExpr(activeFailureMarkerExpr);
  1335. case no_xmltext:
  1336. case no_xmlunicode:
  1337. case no_xmlproject:
  1338. return !ctx.queryMatchExpr(xmlColumnProviderMarkerExpr);
  1339. case no_matched:
  1340. case no_matchtext:
  1341. case no_matchunicode:
  1342. case no_matchlength:
  1343. case no_matchposition:
  1344. case no_matchrow:
  1345. case no_matchutf8:
  1346. return !ctx.queryMatchExpr(activeNlpMarkerExpr);
  1347. case no_matchattr:
  1348. return !ctx.queryMatchExpr(activeNlpMarkerExpr) && !ctx.queryMatchExpr(activeProductionMarkerExpr);
  1349. }
  1350. //Some function of the above => eveluate where we are...
  1351. if (isContextDependent(expr))
  1352. return false;
  1353. if (isIndependentOfScope(expr))
  1354. return true;//isColocal();
  1355. //If can evaluate in parent's start context then always worth doing there.
  1356. if (parent->isRowInvariant(expr))
  1357. return true;
  1358. if (parentExtract && parentExtract->canEvaluate(expr))
  1359. {
  1360. if (!isColocal() || !hasOnStart)
  1361. return true;
  1362. if (translator.queryEvaluateCoLocalRowInvariantInExtract())
  1363. return true;
  1364. return false;
  1365. }
  1366. return false;
  1367. }
  1368. bool EvalContext::insideChildQuery() const
  1369. {
  1370. if (parentExtract)
  1371. return parentExtract->insideChildQuery();
  1372. if (parent)
  1373. return parent->insideChildQuery();
  1374. return false;
  1375. }
  1376. bool EvalContext::requiresOnStart() const
  1377. {
  1378. if (parentExtract)
  1379. return parentExtract->requiresOnStart();
  1380. if (parent)
  1381. return parent->requiresOnStart();
  1382. return false;
  1383. }
  1384. ActivityInstance * EvalContext::queryActivity()
  1385. {
  1386. if (parent)
  1387. return parent->queryActivity();
  1388. return NULL;
  1389. }
  1390. bool EvalContext::areGraphResultsAccessible(IHqlExpression * searchGraphId) const
  1391. {
  1392. if (parentExtract)
  1393. return parentExtract->areGraphResultsAccessible(searchGraphId);
  1394. if (parent)
  1395. return parent->areGraphResultsAccessible(searchGraphId);
  1396. return false;
  1397. }
  1398. IHqlExpression * HqlCppTranslator::doCreateGraphLookup(BuildCtx & declarectx, BuildCtx & resolvectx, unique_id_t id, const char * activity, bool isChild)
  1399. {
  1400. StringBuffer s, var;
  1401. if (isChild)
  1402. {
  1403. appendUniqueId(var.append("child"), id);
  1404. s.clear().append("Owned<IThorChildGraph> ").append(var).append(";");
  1405. declarectx.addQuoted(s);
  1406. s.clear().append(var).append(".setown(ctx->resolveChildQuery(").append(id).append(",").append(activity).append("));");
  1407. resolvectx.addQuoted(s);
  1408. }
  1409. else
  1410. {
  1411. //NB: resolveLocalQuery (unlike children) can't link otherwise you get a circular dependency.
  1412. appendUniqueId(var.append("graph"), id);
  1413. OwnedHqlExpr memberExpr = createQuoted(var, makeBoolType());
  1414. if (declarectx.queryMatchExpr(memberExpr))
  1415. return memberExpr.getClear();
  1416. s.clear().append("IEclGraphResults * ").append(var).append(";");
  1417. declarectx.addQuoted(s);
  1418. declarectx.associateExpr(memberExpr, memberExpr);
  1419. s.clear().append(var).append(" = ctx->resolveLocalQuery(").append(id).append(");");
  1420. resolvectx.addQuoted(s);
  1421. }
  1422. return createQuoted(var, makeBoolType());
  1423. }
  1424. //---------------------------------------------------------------------------
  1425. ClassEvalContext::ClassEvalContext(HqlCppTranslator & _translator, ParentExtract * _parentExtract, EvalContext * _parent, BuildCtx & createctx, BuildCtx & startctx)
  1426. : EvalContext(_translator, _parentExtract, _parent), onCreate(createctx), onStart(startctx)
  1427. {
  1428. }
  1429. IHqlExpression * ClassEvalContext::cloneExprInClass(CtxCollection & ctxs, IHqlExpression * expr)
  1430. {
  1431. if (!expr)
  1432. return NULL;
  1433. if (expr->getOperator() != no_pselect)
  1434. return LINK(expr);
  1435. LinkedHqlExpr value = expr;
  1436. Linked<ITypeInfo> type = expr->queryType();
  1437. assertex(hasModifier(type, typemod_member));
  1438. if (isTypePassedByAddress(type))
  1439. {
  1440. value.setown(getPointer(value));
  1441. type.setown(value->getType());
  1442. }
  1443. else
  1444. {
  1445. if (hasModifier(type, typemod_wrapper))
  1446. {
  1447. type.setown(removeModifier(type, typemod_wrapper));
  1448. value.setown(createValue(no_implicitcast, LINK(type), LINK(value)));
  1449. }
  1450. }
  1451. if (!hasModifier(type, typemod_member))
  1452. type.setown(makeModifier(value->getType(), typemod_member));
  1453. OwnedHqlExpr decl = ctxs.declarectx.getTempDeclare(type, NULL);
  1454. ctxs.clonectx.addAssign(decl, value);
  1455. return LINK(decl);
  1456. }
  1457. void ClassEvalContext::cloneAliasInClass(CtxCollection & ctxs, const CHqlBoundExpr & bound, CHqlBoundExpr & tgt)
  1458. {
  1459. // assertex(!bound.count);
  1460. ensureHelpersExist();
  1461. tgt.isAll.setown(cloneExprInClass(ctxs, bound.isAll));
  1462. tgt.count.setown(cloneExprInClass(ctxs, bound.count));
  1463. tgt.length.setown(cloneExprInClass(ctxs, bound.length));
  1464. tgt.expr.setown(cloneExprInClass(ctxs, bound.expr));
  1465. }
  1466. void ClassEvalContext::createMemberAlias(CtxCollection & ctxs, BuildCtx & ctx, IHqlExpression * value, CHqlBoundExpr & tgt)
  1467. {
  1468. if (ctxs.declarectx.getMatchExpr(value, tgt))
  1469. return;
  1470. //Should never be called for a nested class - that should be done in the context.
  1471. assertex(ctxs.evalctx != NULL);
  1472. translator.expandAliases(*ctxs.evalctx, value);
  1473. IAtom * serializeForm = internalAtom; // The format of serialized expressions in memory must match the internal serialization format
  1474. CHqlBoundTarget tempTarget;
  1475. if (translator.needToSerializeToSlave(value))
  1476. {
  1477. translator.buildTempExpr(*ctxs.evalctx, ctxs.declarectx, tempTarget, value, FormatNatural, false);
  1478. ensureSerialized(ctxs, tempTarget, serializeForm);
  1479. }
  1480. else
  1481. {
  1482. translator.buildTempExpr(ctxs.clonectx, ctxs.declarectx, tempTarget, value, FormatNatural, false);
  1483. }
  1484. tgt.setFromTarget(tempTarget);
  1485. ctxs.declarectx.associateExpr(value, tgt);
  1486. }
  1487. void ClassEvalContext::doCallNestedHelpers(const char * member, const char * activity, IHqlStmt * onCreateStmt, IHqlStmt * onStartStmt)
  1488. {
  1489. StringBuffer s;
  1490. onCreate.childctx.addQuoted(s.clear().append(member).append(".onCreate(ctx, ").append(activity).append(");"));
  1491. BuildCtx childctx(onStart.childctx);
  1492. if (!requiresOnStart())
  1493. childctx.addConditionalGroup(onStartStmt);
  1494. childctx.addQuoted(s.clear().append(member).append(".onStart();"));
  1495. }
  1496. void ClassEvalContext::ensureSerialized(CtxCollection & ctxs, const CHqlBoundTarget & target, IAtom * serializeForm)
  1497. {
  1498. if (ctxs.serializectx)
  1499. translator.ensureSerialized(target, *ctxs.serializectx, *ctxs.deserializectx, "*in", "out", serializeForm);
  1500. }
  1501. AliasKind ClassEvalContext::evaluateExpression(BuildCtx & ctx, IHqlExpression * value, CHqlBoundExpr & tgt, bool evaluateLocally)
  1502. {
  1503. if (onStart.declarectx.getMatchExpr(value, tgt))
  1504. {
  1505. if (onCreate.declarectx.queryMatchExpr(value))
  1506. return CreateTimeAlias;
  1507. return StartTimeAlias;
  1508. }
  1509. if (ctx.getMatchExpr(value, tgt))
  1510. return RuntimeAlias;
  1511. ///If contains a nasty like counter/matchtext then evaluate here for the moment...
  1512. if (!needToEvaluateLocally(ctx, value))
  1513. {
  1514. if (evaluateInParent(ctx, value, (onStart.evalctx != NULL)))
  1515. {
  1516. if (!parentExtract)
  1517. throwError(HQLERR_NoParentExtract);
  1518. CHqlBoundExpr bound;
  1519. AliasKind kind = parentExtract->evaluateExpression(ctx, value, bound, colocalMember, evaluateLocally);
  1520. switch (kind)
  1521. {
  1522. case RuntimeAlias:
  1523. tgt.set(bound);
  1524. break;
  1525. case CreateTimeAlias:
  1526. cloneAliasInClass(onCreate, bound, tgt);
  1527. onCreate.declarectx.associateExpr(value, tgt);
  1528. break;
  1529. case StartTimeAlias:
  1530. cloneAliasInClass(onStart, bound, tgt);
  1531. onStart.declarectx.associateExpr(value, tgt);
  1532. break;
  1533. }
  1534. return kind;
  1535. }
  1536. if (!evaluateLocally)
  1537. return NotFoundAlias;
  1538. if (!isContextDependentExceptGraph(value))
  1539. {
  1540. if (!isContextDependent(value) && !containsActiveDataset(value) && value->isIndependentOfScope())
  1541. {
  1542. createMemberAlias(onCreate, ctx, value, tgt);
  1543. return CreateTimeAlias;
  1544. }
  1545. bool evaluateInOnStart = false;
  1546. if (isRowInvariant(value) && onStart.evalctx)
  1547. {
  1548. evaluateInOnStart = true;
  1549. if (isGraphDependent(value))
  1550. {
  1551. //Need to find out which graph it is dependent on, and check that isn't defined locally
  1552. HqlExprCopyArray graphs;
  1553. gatherGraphReferences(graphs, value, true);
  1554. if (graphs.ordinality() != 0)
  1555. evaluateInOnStart = false;
  1556. }
  1557. }
  1558. if (evaluateInOnStart)
  1559. {
  1560. translator.traceExpression("alias", value);
  1561. createMemberAlias(onStart, ctx, value, tgt);
  1562. return StartTimeAlias;
  1563. }
  1564. }
  1565. }
  1566. if (!evaluateLocally)
  1567. return NotFoundAlias;
  1568. translator.expandAliases(ctx, value);
  1569. translator.buildTempExpr(ctx, value, tgt);
  1570. return RuntimeAlias;
  1571. }
  1572. void ClassEvalContext::tempCompatiablityEnsureSerialized(const CHqlBoundTarget & tgt)
  1573. {
  1574. IAtom * serializeForm = internalAtom; // The format of serialized expressions in memory must match the internal serialization format
  1575. ensureSerialized(onCreate, tgt, serializeForm);
  1576. }
  1577. bool ClassEvalContext::isRowInvariant(IHqlExpression * expr)
  1578. {
  1579. return translator.canEvaluateInContext(onStart.declarectx, expr);
  1580. }
  1581. //If this is called on something that is shared, then need to ensure it will be valid for all subsequent calls
  1582. //therefore, we can't test whether it is inside an onCreate function since that may not be true the first time, but may be subsequent times.
  1583. //and we need to generate the code where we guarantee it will have been executed before (almost) everything else.
  1584. //if not independent, then it can't be shared - so generate it after the code to extract temporary values
  1585. bool ClassEvalContext::getInvariantMemberContext(BuildCtx * ctx, BuildCtx * * declarectx, BuildCtx * * initctx, bool isIndependentMaybeShared, bool invariantEachStart)
  1586. {
  1587. if (invariantEachStart)
  1588. {
  1589. if (declarectx)
  1590. *declarectx = &onStart.declarectx;
  1591. if (initctx && (!ctx || isIndependentMaybeShared || !ctx->queryMatchExpr(insideOnStartMarker)))
  1592. {
  1593. ensureHelpersExist();
  1594. if (isIndependentMaybeShared)
  1595. *initctx = &onStart.clonectx; // before anything else happens.
  1596. else
  1597. *initctx = &onStart.childctx; // after global variables have been read or de-serialized.
  1598. }
  1599. }
  1600. else
  1601. {
  1602. if (declarectx)
  1603. *declarectx = &onCreate.declarectx;
  1604. if (initctx && (!ctx || isIndependentMaybeShared || !ctx->queryMatchExpr(insideOnCreateMarker)))
  1605. {
  1606. ensureHelpersExist();
  1607. if (isIndependentMaybeShared)
  1608. *initctx = &onCreate.clonectx; // before anything else happens.
  1609. else
  1610. *initctx = &onCreate.childctx; // after global variables have been read or de-serialized.
  1611. }
  1612. }
  1613. return true;
  1614. }
  1615. //---------------------------------------------------------------------------
  1616. GlobalClassEvalContext::GlobalClassEvalContext(HqlCppTranslator & _translator, ParentExtract * _parentExtract, EvalContext * _parent, BuildCtx & createctx, BuildCtx & startctx)
  1617. : ClassEvalContext(_translator, _parentExtract, _parent, createctx, startctx)
  1618. {
  1619. }
  1620. void GlobalClassEvalContext::ensureHelpersExist()
  1621. {
  1622. }
  1623. void GlobalClassEvalContext::callNestedHelpers(const char * member, IHqlStmt * onCreateStmt, IHqlStmt * onStartStmt)
  1624. {
  1625. doCallNestedHelpers(member, "this", onCreateStmt, onStartStmt);
  1626. }
  1627. //---------------------------------------------------------------------------
  1628. ActivityEvalContext::ActivityEvalContext(HqlCppTranslator & _translator, ActivityInstance * _activity, ParentExtract * _parentExtract, EvalContext * _parent, IHqlExpression * _colocal, BuildCtx & createctx, BuildCtx & startctx)
  1629. : ClassEvalContext(_translator, _parentExtract, _parent, createctx, startctx)
  1630. {
  1631. activity = _activity;
  1632. colocalMember.set(_colocal);
  1633. }
  1634. IHqlExpression * ActivityEvalContext::createGraphLookup(unique_id_t id, bool isChild)
  1635. {
  1636. return translator.doCreateGraphLookup(onCreate.declarectx, onCreate.childctx, id, "this", isChild);
  1637. }
  1638. void ActivityEvalContext::ensureHelpersExist()
  1639. {
  1640. }
  1641. void ActivityEvalContext::callNestedHelpers(const char * member, IHqlStmt * onCreateStmt, IHqlStmt * onStartStmt)
  1642. {
  1643. doCallNestedHelpers(member, "this", onCreateStmt, onStartStmt);
  1644. }
  1645. ActivityInstance * ActivityEvalContext::queryActivity()
  1646. {
  1647. return activity;
  1648. }
  1649. //---------------------------------------------------------------------------
  1650. NestedEvalContext::NestedEvalContext(HqlCppTranslator & _translator, const char * _memberName, ParentExtract * _parentExtract, EvalContext * _parent, IHqlExpression * _colocal, BuildCtx & createctx, BuildCtx & startctx)
  1651. : ClassEvalContext(_translator, _parentExtract, _parent, createctx, startctx)
  1652. {
  1653. colocalMember.set(_colocal);
  1654. helpersExist = false;
  1655. memberName.set(_memberName);
  1656. }
  1657. IHqlExpression * NestedEvalContext::createGraphLookup(unique_id_t id, bool isChild)
  1658. {
  1659. ensureHelpersExist();
  1660. return translator.doCreateGraphLookup(onCreate.declarectx, onCreate.childctx, id, "activity", isChild);
  1661. }
  1662. void NestedEvalContext::ensureHelpersExist()
  1663. {
  1664. assertex(parentExtract);
  1665. if (!helpersExist)
  1666. {
  1667. if (parent)
  1668. parent->ensureHelpersExist();
  1669. StringBuffer s;
  1670. ActivityInstance * rootActivity = queryActivity();
  1671. //void onStart(ICodeContext * _ctx, <ActivityClass> * _activity)
  1672. BuildCtx oncreatectx(onCreate.declarectx);
  1673. IHqlStmt * onCreateStmt = oncreatectx.addQuotedCompound(s.clear().append("inline void onCreate(ICodeContext * _ctx, ").append(rootActivity->className).append(" * _activity)"));
  1674. oncreatectx.addQuoted(s.clear().append("activity = _activity;"));
  1675. oncreatectx.addQuotedLiteral("ctx = _ctx;");
  1676. onCreate.declarectx.addQuotedLiteral("ICodeContext * ctx;");
  1677. onCreate.declarectx.addQuoted(s.clear().append(rootActivity->className).append(" * activity;"));
  1678. onCreate.declarectx.associateExpr(codeContextMarkerExpr, codeContextMarkerExpr);
  1679. onCreate.createFunctionStructure(translator, oncreatectx, false, NULL);
  1680. //void onStart(const byte * parentExtract)
  1681. BuildCtx onstartctx(onStart.declarectx);
  1682. IHqlStmt * onStartStmt = onstartctx.addQuotedCompound("inline void onStart()");
  1683. if (parentExtract)
  1684. parentExtract->beginChildActivity(onStart.declarectx, onstartctx, GraphCoLocal, colocalMember, true, parentExtract->canSerializeFields(), NULL);
  1685. onstartctx.associateExpr(insideOnStartMarker, NULL);
  1686. onStart.createFunctionStructure(translator, onstartctx, false, NULL);
  1687. parent->callNestedHelpers(memberName, onCreateStmt, onStartStmt);
  1688. if (!requiresOnStart())
  1689. onStartStmt->finishedFramework();
  1690. helpersExist = true;
  1691. }
  1692. }
  1693. void NestedEvalContext::initContext()
  1694. {
  1695. if (requiresOnStart())
  1696. ensureHelpersExist();
  1697. else if (parentExtract && parentExtract->canSerializeFields())
  1698. {
  1699. ensureHelpersExist();
  1700. parentExtract->associateCursors(onStart.declarectx, onStart.declarectx, GraphCoLocal);
  1701. }
  1702. }
  1703. bool NestedEvalContext::evaluateInParent(BuildCtx & ctx, IHqlExpression * expr, bool hasOnStart)
  1704. {
  1705. //This is a bit ugly. Latter condition is to cope with group aggregate callbacks
  1706. return parent->isRowInvariant(expr) || parentExtract->canEvaluate(expr);
  1707. }
  1708. void NestedEvalContext::callNestedHelpers(const char * member, IHqlStmt * onCreateStmt, IHqlStmt * onStartStmt)
  1709. {
  1710. doCallNestedHelpers(member, "activity", onCreateStmt, onStartStmt);
  1711. }
  1712. //---------------------------------------------------------------------------
  1713. MemberEvalContext::MemberEvalContext(HqlCppTranslator & _translator, ParentExtract * _parentExtract, EvalContext * _parent, BuildCtx & _ctx)
  1714. : EvalContext(_translator, _parentExtract, _parent), ctx(_ctx)
  1715. {
  1716. assertex(parent);
  1717. colocalMember.set(colocalSameClassPreserveExpr);
  1718. }
  1719. void MemberEvalContext::callNestedHelpers(const char * member, IHqlStmt * onCreateStmt, IHqlStmt * onStartStmt)
  1720. {
  1721. parent->callNestedHelpers(member, onCreateStmt, onStartStmt);
  1722. }
  1723. IHqlExpression * MemberEvalContext::createGraphLookup(unique_id_t id, bool isChild)
  1724. {
  1725. return parent->createGraphLookup(id, isChild);
  1726. }
  1727. AliasKind MemberEvalContext::evaluateExpression(BuildCtx & ctx, IHqlExpression * value, CHqlBoundExpr & tgt, bool evaluateLocally)
  1728. {
  1729. return parentExtract->evaluateExpression(ctx, value, tgt, colocalMember, evaluateLocally);
  1730. }
  1731. bool MemberEvalContext::isRowInvariant(IHqlExpression * expr)
  1732. {
  1733. return translator.canEvaluateInContext(ctx, expr);
  1734. }
  1735. void MemberEvalContext::ensureHelpersExist()
  1736. {
  1737. parent->ensureHelpersExist();
  1738. }
  1739. void MemberEvalContext::initContext()
  1740. {
  1741. parentExtract->associateCursors(ctx, ctx, GraphCoLocal);
  1742. }
  1743. bool MemberEvalContext::getInvariantMemberContext(BuildCtx * ctx, BuildCtx * * declarectx, BuildCtx * * initctx, bool isIndependentMaybeShared, bool invariantEachStart)
  1744. {
  1745. return parent->getInvariantMemberContext(ctx, declarectx, initctx, isIndependentMaybeShared, invariantEachStart);
  1746. }
  1747. void MemberEvalContext::tempCompatiablityEnsureSerialized(const CHqlBoundTarget & tgt)
  1748. {
  1749. parent->tempCompatiablityEnsureSerialized(tgt);
  1750. }
  1751. //---------------------------------------------------------------------------
  1752. void HqlCppTranslator::ensureContextAvailable(BuildCtx & ctx)
  1753. {
  1754. EvalContext * instance = queryEvalContext(ctx);
  1755. if (instance)
  1756. instance->ensureContextAvailable();
  1757. }
  1758. /*
  1759. The following is the structure that we are going to generate in the C++ for an activity
  1760. <<xx>> marks a context that is preserved
  1761. virtual void onCreate(ICodeContext * _ctx, IHThorArg * _colocal, MemoryBuffer * in) {
  1762. colocal = (c2*)_colocal;
  1763. ctx = _ctx;
  1764. <<onCreate.clonectx>>
  1765. // clone values from colocal-> into local variables.
  1766. if (in) {
  1767. //deserialize any values from
  1768. <<onCreate.deserializectx>>
  1769. }
  1770. else {
  1771. //evaluate any query-invariant values.
  1772. <<onCreate.evalctx>>
  1773. }
  1774. <<onCreate.childctx>>
  1775. // create graph lookups.
  1776. nestedObject.onCreate(ctx, this);
  1777. }
  1778. virtual void serializeCreateContext(MemoryBuffer & out) {
  1779. //serialize any query-invariant values
  1780. }
  1781. virtual void onStart(const byte * pe) {
  1782. extractVar = (byte *)pe;
  1783. <<onStart.clonectx>>
  1784. <<onStart.evalctx>>
  1785. <<onStart.childctx>>
  1786. nestedObject.onStart(pe);
  1787. }
  1788. NestedClass:
  1789. void onCreate(ICodeContext * _ctx, cH * _activity) {
  1790. activity = _activity;
  1791. ctx = _ctx;
  1792. <<onCreate.clonectx>>
  1793. <<onCreate.childctx>>
  1794. //clone values from parent class into this class
  1795. }
  1796. virtual void onStart(const byte * pe) {
  1797. extractVar = (byte *)pe; //This means that all values retrieved from the extract have the same format in the nested class
  1798. <<onStart.clonectx>>
  1799. <<onStart.childctx>>
  1800. }
  1801. virtual void onCreate(ICodeContext * _ctx, IHThorArg * _colocal) {
  1802. colocal = (c2*)_colocal;
  1803. ctx = _ctx;
  1804. <<onCreate.clonectx>>
  1805. // clone values from colocal-> into local variables.
  1806. <<onCreate.evalctx>>
  1807. // evaluate graph invariant expressions
  1808. //local graph lookups
  1809. commonCreate();
  1810. }
  1811. inline void commonCreate()
  1812. <<onCreate.childctx>>
  1813. // create child graph lookups.
  1814. // nestedObject.onCreate(ctx, this);
  1815. }
  1816. virtual void serializeContext(MemoryBuffer & out) {
  1817. //serialize any query-invariant values
  1818. }
  1819. virtual void onStart(const byte * pe) {
  1820. extractVar = (byte *)pe;
  1821. <<onStart.clonectx>>
  1822. <<onStart.evalctx>>
  1823. <<onStart.childctx>>
  1824. }
  1825. inline void commonOnStart()
  1826. {
  1827. nestedObject.onStart(pe);
  1828. }
  1829. virtual void onCreateStart(ICodeContext * _ctx, MemoryBuffer * in, const byte * pe)
  1830. {
  1831. ctx = _ctx;
  1832. colocal = NULL;
  1833. <<onCreate.deserialize>>
  1834. commonCreate();
  1835. extractVar = (byte *)pe;
  1836. commonOnStart();
  1837. }
  1838. xxx.clonectx - The point where members are cloned from colocal parent activities and parent classes.
  1839. xxx.evalctx - The point where anything that needs to be evaluated is placed.
  1840. Notes on onCreate/onStart
  1841. a) onStart needs to be created/called for child activities (and their nested classes) in order to bind the variables correctly.
  1842. b) onCreate needs to be called for nested classes in order to access grand parent colocal extracts.
  1843. c) If a function is created it needs to be called from parent (which stops us using compoundOpt unless the base class contains a default implementation)
  1844. d) We don't want to call onCReate() + onStart all the time because it generates lots of extra code.
  1845. =>
  1846. 1) For any child class (activity or nested) we always generate + call them even if they don't have any extra code.
  1847. 2) For nested classes of top level activities the functions are generated and called when something is added that needs it.
  1848. Notes
  1849. * Serialization + evaluation of expressions
  1850. a) if an activity is not executed remotely there is no need to serialize on Create
  1851. => serialize remote roxie activites
  1852. => serialize remote thor child activities.
  1853. => optionally serialize all root thor activities depending on whether Jake makes use of it.
  1854. => add a debug flag to force serialization for the purpose of testing.
  1855. b) table-invariant values.
  1856. => evaluated once in the onCreate() of the activity. Not retrieved from a parent activity.
  1857. c) row invariant values
  1858. => if a value can be evaluated in a parent extract context then it is evaluated there, and
  1859. then cloned/serialized into the current context.
  1860. d) since row invariant values are calulated in the parent context there is nothing to be gained from serializing the
  1861. start context. => Remove the code + methods from the eclagent helper definition.
  1862. e) Nested class values.
  1863. These are always evaluated in the parent context, and then cloned into the current class.
  1864. - The extracts from all parent colocal activities are cloned into the child activity
  1865. - All members from a parent activity are copied into members in the current activity + accessed from there.
  1866. [This is partly to simplify the access expressions from colocal children]
  1867. Evaluation of expressions:
  1868. o Table-invariant
  1869. - Do in the onCreate of the activity.
  1870. - Could also serialize from parent to child queries, so only evaluated once.
  1871. = Trade off between re-eveluation and cloning/extra data serialized to child query.
  1872. o Other expressions
  1873. - Each expression has a minimal level it can be evaluated at.
  1874. - main decision is whether to evaluate in the parent's build ctx, or in a local onStart()
  1875. = If there is no onStart, then always evaluate in parent build ctx.
  1876. = If there is an onStart, then you are trading off the cost of serializing the values required to compute
  1877. the expression, and possibly multiple evaluations, against the cost of serializing the expression.
  1878. - For non-local always evaluate in the parent build ctx.
  1879. - For colocal, it is debatable - I'll add as a flag.
  1880. o non-local
  1881. o COUNTER, MATCHTEXT, XMLTEXT, FAILCODE/FAILMESSAGE
  1882. - These are all context dependent in different ways.
  1883. - They all need to be evaluated in the correct context (not too deep or too shallow)
  1884. - Simplest is to evaluate any expression that contains them at the deepest level, and actual expressions evaluated
  1885. at higher levels.
  1886. Activity
  1887. Invariant: evaluate in onCreateContext
  1888. Nested
  1889. Evaluate in parent. If
  1890. General
  1891. * see codegen.txt for more details.
  1892. */