fvtransform.cpp 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960
  1. /*##############################################################################
  2. Copyright (C) 2011 HPCC Systems.
  3. All rights reserved. This program is free software: you can redistribute it and/or modify
  4. it under the terms of the GNU Affero General Public License as
  5. published by the Free Software Foundation, either version 3 of the
  6. License, or (at your option) any later version.
  7. This program is distributed in the hope that it will be useful,
  8. but WITHOUT ANY WARRANTY; without even the implied warranty of
  9. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  10. GNU Affero General Public License for more details.
  11. You should have received a copy of the GNU Affero General Public License
  12. along with this program. If not, see <http://www.gnu.org/licenses/>.
  13. ############################################################################## */
  14. #include "jliball.hpp"
  15. #include "eclrtl.hpp"
  16. #include "fileview.hpp"
  17. #include "fverror.hpp"
  18. #include "fvrelate.ipp"
  19. #include "deftype.hpp"
  20. #include "fvresultset.ipp"
  21. #include "thorplugin.hpp"
  22. #include "eclrtl_imp.hpp"
  23. #include "hqlfold.hpp"
  24. #include "hqlvalid.hpp"
  25. #include "hqlrepository.hpp"
  26. static ViewTransformerRegistry * theTransformerRegistry;
  27. static ITypeInfo * stringType;
  28. static ITypeInfo * utf8Type;
  29. static ITypeInfo * unicodeType;
  30. static _ATOM addAtom;
  31. MODULE_INIT(INIT_PRIORITY_STANDARD)
  32. {
  33. addAtom = createIdentifierAtom("add");
  34. stringType = makeStringType(UNKNOWN_LENGTH, NULL, NULL);
  35. utf8Type = makeUtf8Type(UNKNOWN_LENGTH, NULL);
  36. unicodeType = makeUnicodeType(UNKNOWN_LENGTH, NULL);
  37. theTransformerRegistry = new ViewTransformerRegistry;
  38. theTransformerRegistry->addTransformer(new ViewFailTransformer);
  39. theTransformerRegistry->addTransformer(new ViewAddTransformer);
  40. return true;
  41. }
  42. MODULE_EXIT()
  43. {
  44. delete theTransformerRegistry;
  45. unicodeType->Release();
  46. utf8Type->Release();
  47. stringType->Release();
  48. }
  49. //---------------------------------------------------------------------------
  50. void CardinalityElement::init(unsigned len, const char * _text)
  51. {
  52. if (len == 0)
  53. {
  54. text.set("1");
  55. min = 1;
  56. max = 1;
  57. return;
  58. }
  59. text.set(_text, len);
  60. const char * dotdot = strstr(text, "..");
  61. if (dotdot)
  62. {
  63. min = rtlStrToUInt4(dotdot-text, text);
  64. if (stricmp(dotdot+2, "M") == 0)
  65. max = Unbounded;
  66. else
  67. max = rtlVStrToUInt4(dotdot+2);
  68. }
  69. else
  70. {
  71. if (stricmp(text, "M") == 0)
  72. {
  73. min = 0;
  74. max = Unbounded;
  75. }
  76. else
  77. {
  78. min = rtlVStrToUInt4(text);
  79. max = min;
  80. }
  81. }
  82. }
  83. void CardinalityMapping::init(const char * text)
  84. {
  85. const char * colon = strchr(text, ':');
  86. if (colon)
  87. {
  88. primary.init(colon-text, text);
  89. secondary.init(strlen(colon+1), colon+1);
  90. }
  91. else
  92. {
  93. //May as well try and make some sense of it
  94. primary.init(0, "");
  95. secondary.init(strlen(text), text);
  96. }
  97. }
  98. void getInvertedCardinality(StringBuffer & out, const char * cardinality)
  99. {
  100. if (cardinality)
  101. {
  102. CardinalityMapping mapping(cardinality);
  103. out.append(mapping.secondary.text).append(":").append(mapping.primary.text);
  104. }
  105. }
  106. //---------------------------------------------------------------------------
  107. ViewFieldTransformer * ViewFieldTransformer::bind(const HqlExprArray & args)
  108. {
  109. return LINK(this);
  110. }
  111. void ViewFieldTransformer::transform(MemoryAttr & utfTarget, const MemoryAttr & utfSrc)
  112. {
  113. //NB: The system utf8 functions typically take a length, whilst MemoryAttr provide a size.
  114. unsigned lenTarget;
  115. char * target;
  116. const char * source = static_cast<const char *>(utfSrc.get());
  117. unsigned lenSource = rtlUtf8Length(utfSrc.length(), source);
  118. transform(lenTarget, target, lenSource, source);
  119. unsigned sizeTarget = rtlUtf8Size(lenTarget, target);
  120. utfTarget.setOwn(sizeTarget, target);
  121. }
  122. ViewFailTransformer::ViewFailTransformer() : ViewFieldTransformer(failAtom)
  123. {
  124. }
  125. void ViewFailTransformer::transform(unsigned & lenTarget, char * & target, unsigned lenSource, const char * source)
  126. {
  127. throwError(FVERR_FailTransformation);
  128. lenTarget = 0;
  129. target = NULL;
  130. }
  131. ViewAddTransformer::ViewAddTransformer() : ViewFieldTransformer(addAtom)
  132. {
  133. }
  134. ViewAddTransformer::ViewAddTransformer(const HqlExprArray & _args) : ViewFieldTransformer(addAtom)
  135. {
  136. appendArray(args, _args);
  137. }
  138. ViewFieldTransformer * ViewAddTransformer::bind(const HqlExprArray & args)
  139. {
  140. return new ViewAddTransformer(args);
  141. }
  142. void ViewAddTransformer::transform(unsigned & lenTarget, char * & target, unsigned lenSource, const char * source)
  143. {
  144. unsigned __int64 value = rtlUtf8ToInt(lenSource, source);
  145. if (args.ordinality())
  146. value += args.item(0).queryValue()->getIntValue();
  147. rtlInt8ToStrX(lenTarget, target, value);
  148. }
  149. //---------------------------------------------------------------------------
  150. void ViewFieldUtf8Transformer::transform(unsigned & lenTarget, char * & target, unsigned lenSource, const char * source)
  151. {
  152. (*function)(lenTarget, target, lenSource, source);
  153. }
  154. void ViewFieldUnicodeTransformer::transform(unsigned & lenTarget, char * & target, unsigned lenSource, const char * source)
  155. {
  156. unsigned lenUnicodeSrc;
  157. unsigned lenUnicodeTarget;
  158. rtlDataAttr unicodeSrc;
  159. rtlDataAttr unicodeTarget;
  160. rtlUtf8ToUnicodeX(lenUnicodeSrc, unicodeSrc.refustr(), lenSource, source);
  161. (*function)(lenUnicodeTarget, unicodeTarget.refustr(), lenUnicodeSrc, unicodeSrc.getustr());
  162. rtlUnicodeToUtf8X(lenTarget, target, lenUnicodeTarget, unicodeTarget.getustr());
  163. }
  164. void ViewFieldStringTransformer::transform(unsigned & lenTarget, char * & target, unsigned lenSource, const char * source)
  165. {
  166. unsigned lenStringSrc;
  167. unsigned lenStringTarget;
  168. rtlDataAttr stringSrc;
  169. rtlDataAttr stringTarget;
  170. rtlUtf8ToStrX(lenStringSrc, stringSrc.refstr(), lenSource, source);
  171. (*function)(lenStringTarget, stringTarget.refstr(), lenStringSrc, stringSrc.getstr());
  172. rtlStrToUtf8X(lenTarget, target, lenStringTarget, stringTarget.getstr());
  173. }
  174. //---------------------------------------------------------------------------
  175. ViewFieldTransformer * ViewFieldECLTransformer::bind(const HqlExprArray & args)
  176. {
  177. if (args.ordinality() == 0)
  178. return LINK(this);
  179. return new ViewFieldBoundECLTransformer(this, args);
  180. }
  181. void ViewFieldECLTransformer::transform(unsigned & lenTarget, char * & target, unsigned lenSource, const char * source, const HqlExprArray & extraArgs)
  182. {
  183. Owned<ITypeInfo> sourceType = makeUtf8Type(lenSource, 0);
  184. IValue * sourceValue = createUtf8Value(source, LINK(sourceType));
  185. OwnedHqlExpr sourceExpr = createConstant(sourceValue);
  186. HqlExprArray actuals;
  187. actuals.append(*LINK(sourceExpr));
  188. appendArray(actuals, extraArgs);
  189. ThrowingErrorReceiver errors;
  190. OwnedHqlExpr call = createBoundFunction(&errors, function, actuals, NULL, true);
  191. OwnedHqlExpr castValue = ensureExprType(call, utf8Type);
  192. OwnedHqlExpr folded = quickFoldExpression(castValue, NULL, 0);
  193. IValue * foldedValue = folded->queryValue();
  194. assertex(foldedValue);
  195. unsigned len = foldedValue->queryType()->getStringLen();
  196. const char * data = static_cast<const char *>(foldedValue->queryValue());
  197. unsigned size = rtlUtf8Size(len, data);
  198. lenTarget = len;
  199. target = (char *)rtlMalloc(size);
  200. memcpy(target, data, size);
  201. }
  202. void ViewFieldECLTransformer::transform(unsigned & lenTarget, char * & target, unsigned lenSource, const char * source)
  203. {
  204. HqlExprArray extraArgs;
  205. transform(lenTarget, target, lenSource, source, extraArgs);
  206. }
  207. void ViewFieldBoundECLTransformer::transform(unsigned & lenTarget, char * & target, unsigned lenSource, const char * source)
  208. {
  209. transformer->transform(lenTarget, target, lenSource, source, args);
  210. }
  211. //---------------------------------------------------------------------------
  212. void ViewTransformerRegistry::addTransformer(ViewFieldTransformer * ownedTransformer)
  213. {
  214. transformers.append(*ownedTransformer);
  215. }
  216. void ViewTransformerRegistry::addFieldUtf8Transformer(const char * name, utf8FieldTransformerFunction func)
  217. {
  218. transformers.append(* new ViewFieldUtf8Transformer(createIdentifierAtom(name), func));
  219. }
  220. void ViewTransformerRegistry::addFieldStringTransformer(const char * name, stringFieldTransformerFunction func)
  221. {
  222. transformers.append(* new ViewFieldStringTransformer(createIdentifierAtom(name), func));
  223. }
  224. void ViewTransformerRegistry::addFieldUnicodeTransformer(const char * name, unicodeFieldTransformerFunction func)
  225. {
  226. transformers.append(* new ViewFieldUnicodeTransformer(createIdentifierAtom(name), func));
  227. }
  228. void ViewTransformerRegistry::addPlugins(const char * name)
  229. {
  230. loadedPlugins.setown(new SafePluginMap(&pluginCtx, true));
  231. loadedPlugins->loadFromList(name);
  232. ThrowingErrorReceiver errors;
  233. dataServer.setown(createSourceFileEclRepository(&errors, name, NULL, NULL, 0));
  234. HqlScopeArray scopes;
  235. HqlParseContext parseCtx(dataServer, NULL);
  236. HqlLookupContext ctx(parseCtx, &errors);
  237. dataServer->getRootScopes(scopes, ctx);
  238. ForEachItemIn(i, scopes)
  239. {
  240. IHqlScope * scope = &scopes.item(i);
  241. HqlExprArray symbols;
  242. scope->getSymbols(symbols);
  243. ForEachItemIn(j, symbols)
  244. {
  245. IHqlExpression & cur = symbols.item(j);
  246. if (cur.getOperator() == no_service)
  247. addServiceDefinition(&cur);
  248. }
  249. }
  250. }
  251. static bool matchesSimpleTransformer(IHqlExpression * definition, ITypeInfo * type)
  252. {
  253. if (definition->queryBody()->queryType() != type)
  254. return false;
  255. if (definition->numChildren() != 2)
  256. return false;
  257. if (definition->queryChild(1)->queryType() != type)
  258. return false;
  259. return true;
  260. }
  261. void * ViewTransformerRegistry::resolveExternal(IHqlExpression * funcdef)
  262. {
  263. IHqlExpression *body = funcdef->queryChild(0);
  264. StringBuffer entry;
  265. StringBuffer lib;
  266. getProperty(body, entrypointAtom, entry);
  267. getProperty(body, libraryAtom, lib);
  268. if (!lib.length())
  269. getProperty(body, pluginAtom, lib);
  270. ensureFileExtension(lib, SharedObjectExtension);
  271. Owned<ILoadedDllEntry> match = loadedPlugins->getPluginDll(lib.toCharArray(), NULL, false); // MORE - shouldn't it check the version????
  272. if (!match)
  273. return NULL;
  274. return GetSharedProcedure(match->getInstance(), entry.toCharArray());
  275. }
  276. ViewFieldTransformer * ViewTransformerRegistry::createTransformer(IHqlExpression * funcdef)
  277. {
  278. IHqlExpression *body = funcdef->queryChild(0);
  279. if(!body)
  280. return NULL;
  281. StringBuffer entry;
  282. StringBuffer lib;
  283. getProperty(body, entrypointAtom, entry);
  284. getProperty(body, libraryAtom, lib);
  285. if (!lib.length())
  286. getProperty(body, pluginAtom, lib);
  287. if ((entry.length() == 0) || (lib.length() == 0))
  288. return NULL;
  289. if(!body->hasProperty(pureAtom) && !body->hasProperty(templateAtom))
  290. return NULL;
  291. if(!body->hasProperty(cAtom))
  292. return NULL;
  293. if(body->hasProperty(gctxmethodAtom) || body->hasProperty(ctxmethodAtom) || body->hasProperty(omethodAtom))
  294. return NULL;
  295. if(body->hasProperty(contextAtom) || body->hasProperty(globalContextAtom))
  296. return NULL;
  297. //Special case string->string mapping (e.g., uppercase)
  298. if (matchesSimpleTransformer(funcdef, stringType))
  299. {
  300. stringFieldTransformerFunction resolved = (stringFieldTransformerFunction)resolveExternal(funcdef);
  301. if (resolved)
  302. return new ViewFieldStringTransformer(funcdef->queryName(), resolved);
  303. }
  304. //Special case string->string mapping (e.g., uppercase)
  305. if (matchesSimpleTransformer(funcdef, unicodeType))
  306. {
  307. unicodeFieldTransformerFunction resolved = (unicodeFieldTransformerFunction)resolveExternal(funcdef);
  308. if (resolved)
  309. return new ViewFieldUnicodeTransformer(funcdef->queryName(), resolved);
  310. }
  311. //MORE: special case string->string etc.
  312. return new ViewFieldECLTransformer(funcdef);
  313. }
  314. void ViewTransformerRegistry::addServiceDefinition(IHqlExpression * service)
  315. {
  316. Owned<ViewServiceEntry> entry = new ViewServiceEntry;
  317. entry->name.set(service->queryName()->str());
  318. HqlExprArray symbols;
  319. service->queryScope()->getSymbols(symbols);
  320. ForEachItemIn(i, symbols)
  321. {
  322. IHqlExpression & cur = symbols.item(i);
  323. if (cur.getOperator() == no_funcdef && cur.queryChild(0)->getOperator() == no_external)
  324. {
  325. ViewFieldTransformer * transformer = createTransformer(&cur);
  326. if (transformer)
  327. entry->transformers.append(*transformer);
  328. }
  329. }
  330. plugins.append(*entry.getClear());
  331. }
  332. ViewFieldTransformer * find(const ViewFieldTransformerArray & transformers, const char * name, const HqlExprArray & args)
  333. {
  334. if (!name)
  335. return NULL;
  336. _ATOM search = createIdentifierAtom(name);
  337. ForEachItemIn(i, transformers)
  338. {
  339. ViewFieldTransformer & cur = transformers.item(i);
  340. if (cur.matches(search))
  341. return cur.bind(args);
  342. }
  343. return NULL;
  344. }
  345. ViewFieldTransformer * ViewTransformerRegistry::resolve(const char * name, const HqlExprArray & args)
  346. {
  347. return find(transformers, name, args);
  348. }
  349. ViewFieldTransformer * ViewTransformerRegistry::resolve(const char * servicename, const char * functionName, const HqlExprArray & args)
  350. {
  351. if (!servicename)
  352. return NULL;
  353. ForEachItemIn(i, plugins)
  354. {
  355. ViewServiceEntry & cur = plugins.item(i);
  356. if (stricmp(servicename, cur.name) == 0)
  357. return find(cur.transformers, functionName, args);
  358. }
  359. return NULL;
  360. }
  361. IViewTransformerRegistry & queryTransformerRegistry()
  362. {
  363. return *theTransformerRegistry;
  364. }
  365. //---------------------------------------------------------------------------
  366. bool containsFail(const ViewFieldTransformerArray & transforms)
  367. {
  368. ForEachItemIn(i, transforms)
  369. {
  370. if (transforms.item(i).matches(failAtom))
  371. return true;
  372. }
  373. return false;
  374. }
  375. void translateValue(MemoryAttr & result, const MemoryAttr & filterValue, const ViewFieldTransformerArray & transforms)
  376. {
  377. unsigned numTransforms = transforms.ordinality();
  378. if (numTransforms)
  379. {
  380. MemoryAttr tempValue[2];
  381. unsigned whichTarget = 0;
  382. const MemoryAttr * source = &filterValue;
  383. for (unsigned i=0; i < numTransforms-1; i++)
  384. {
  385. MemoryAttr * target = &tempValue[whichTarget];
  386. transforms.item(i).transform(*target, *source);
  387. source = target;
  388. whichTarget = 1-whichTarget;
  389. }
  390. transforms.item(numTransforms-1).transform(result, *source);
  391. }
  392. else
  393. result.set(filterValue.length(), filterValue.get());
  394. }
  395. ViewJoinColumn::ViewJoinColumn(unsigned _whichColumn, const ViewFieldTransformerArray & _getTransforms, const ViewFieldTransformerArray & _setTransforms)
  396. {
  397. whichColumn = _whichColumn;
  398. appendArray(getTransforms, _getTransforms);
  399. appendArray(setTransforms, _setTransforms);
  400. getContainsFail = containsFail(getTransforms);
  401. setContainsFail = containsFail(setTransforms);
  402. }
  403. void ViewJoinColumn::addFilter(IFilteredResultSet * resultSet, const MemoryAttr & value)
  404. {
  405. const MemoryAttr * source = &value;
  406. MemoryAttr tempValue;
  407. if (setTransforms.ordinality())
  408. {
  409. translateValue(tempValue, value, setTransforms);
  410. source = &tempValue;
  411. }
  412. resultSet->addFilter(whichColumn, source->length(), (const char *)source->get());
  413. }
  414. void ViewJoinColumn::clearFilter(IFilteredResultSet * resultSet)
  415. {
  416. resultSet->clearFilter(whichColumn);
  417. }
  418. void ViewJoinColumn::getValue(MemoryAttr & value, IResultSetCursor * cursor)
  419. {
  420. if (getTransforms.ordinality())
  421. {
  422. MemoryAttr rowValue;
  423. MemoryAttr2IStringVal adaptor(rowValue);
  424. cursor->getDisplayText(adaptor, whichColumn);
  425. translateValue(value, rowValue, getTransforms);
  426. }
  427. else
  428. {
  429. MemoryAttr2IStringVal adaptor(value);
  430. cursor->getDisplayText(adaptor, whichColumn);
  431. }
  432. }
  433. //---------------------------------------------------------------------------
  434. struct TextReference
  435. {
  436. TextReference() { len =0; text = NULL; }
  437. TextReference(size32_t _len, const char * _text) : len(_len), text(_text) {}
  438. inline bool eq(const char * search) const { return (len == strlen(search)) && (memicmp(text, search, len) == 0); }
  439. inline void get(StringAttr & target) const { target.set(text, len); }
  440. void set(size32_t _len, const char * _text) { len = _len; text = _text; }
  441. IHqlExpression * createIntConstant();
  442. IHqlExpression * createStringConstant();
  443. size32_t len;
  444. const char * text;
  445. };
  446. IHqlExpression * TextReference::createIntConstant()
  447. {
  448. return createConstant(rtlStrToInt8(len, text));
  449. }
  450. IHqlExpression * TextReference::createStringConstant()
  451. {
  452. return createConstant(createUnicodeValue(text+1, len-2, "", true, true));
  453. }
  454. class MappingParser
  455. {
  456. enum { TokEof=256, TokId, TokInt, TokString };
  457. public:
  458. MappingParser(const IResultSetMetaData & _fieldMeta, bool _datasetSelectorAllowed) : fieldMeta(_fieldMeta), datasetSelectorAllowed(_datasetSelectorAllowed)
  459. {
  460. tokenType = TokEof;
  461. lenInput = 0;
  462. input = NULL;
  463. offset = 0;
  464. }
  465. void parseColumnMappingList(FieldTransformInfoArray & results, unsigned len, const char * text);
  466. protected:
  467. unsigned lexToken();
  468. void assertToken(int expected);
  469. void getTokenText(StringAttr & target) { curToken.get(target); }
  470. void parseAttribute(FieldTransformInfo & output);
  471. void parseColumn(FieldTransformInfo & output);
  472. void parseColumnMapping(FieldTransformInfo & output);
  473. void parseConstantList(HqlExprArray & args);
  474. void parseTransformList(ViewFieldTransformerArray & transforms);
  475. protected:
  476. const IResultSetMetaData & fieldMeta;
  477. bool datasetSelectorAllowed;
  478. unsigned tokenType;
  479. TextReference curToken;
  480. unsigned offset;
  481. unsigned lenInput;
  482. const char * input;
  483. };
  484. inline bool isLeadingIdentChar(byte next)
  485. {
  486. return isalpha(next) || (next == '_') || (next == '$');
  487. }
  488. inline bool isTrailingIdentChar(byte next)
  489. {
  490. return isalnum(next) || (next == '_') || (next == '$');
  491. }
  492. //MORE: This should really use the hqllexer - especially if it gets any more complex!
  493. unsigned MappingParser::lexToken()
  494. {
  495. const byte * buffer = (const byte *)input;
  496. unsigned cur = offset;
  497. while ((cur < lenInput) && isspace(buffer[cur]))
  498. cur++;
  499. if (cur < lenInput)
  500. {
  501. byte next = buffer[cur];
  502. if (isLeadingIdentChar(next))
  503. {
  504. cur++;
  505. while ((cur < lenInput) && (isTrailingIdentChar(buffer[cur])))
  506. cur++;
  507. tokenType = TokId;
  508. }
  509. else if (isdigit(next) ||
  510. ((next == '-') && (cur+1 < lenInput) && isdigit(buffer[cur+1])))
  511. {
  512. cur++;
  513. while ((cur < lenInput) && (isdigit(buffer[cur])))
  514. cur++;
  515. tokenType = TokInt;
  516. }
  517. else if (next == '\'')
  518. {
  519. cur++;
  520. while (cur < lenInput)
  521. {
  522. byte next = buffer[cur];
  523. if (next == '\'')
  524. break;
  525. else if (next == '\\')
  526. {
  527. if (cur+1 < lenInput)
  528. cur++;
  529. }
  530. cur++;
  531. }
  532. if (cur == lenInput)
  533. throwError2(FVERR_BadStringTermination, cur-offset, input+offset);
  534. cur++;
  535. tokenType = TokString;
  536. }
  537. else
  538. {
  539. tokenType = next;
  540. cur++;
  541. }
  542. }
  543. else
  544. {
  545. tokenType = TokEof;
  546. }
  547. curToken.set(cur-offset, input+offset);
  548. offset = cur;
  549. return tokenType;
  550. }
  551. void MappingParser::assertToken(int expected)
  552. {
  553. if (tokenType != expected)
  554. {
  555. StringBuffer id;
  556. switch (expected)
  557. {
  558. case TokId:
  559. id.append("identifier");
  560. break;
  561. default:
  562. id.append((char)expected);
  563. break;
  564. }
  565. unsigned len = lenInput-offset;
  566. if (len>10) len = 10;
  567. throwError3(FVERR_ExpectedX, id.str(), len, input+offset);
  568. }
  569. }
  570. void MappingParser::parseColumn(FieldTransformInfo & output)
  571. {
  572. output.datasetColumn = NotFound;
  573. output.column = NotFound;
  574. unsigned firstFieldIndex = 0;
  575. const IResultSetMetaData * curMeta = &fieldMeta;
  576. loop
  577. {
  578. StringAttr fieldName;
  579. assertToken(TokId);
  580. getTokenText(fieldName);
  581. lexToken();
  582. //Cheat and cast the meta so the field lookup can be done more efficiently
  583. const CResultSetMetaData & castFieldMeta = static_cast<const CResultSetMetaData &>(*curMeta);
  584. unsigned matchColumn = castFieldMeta.queryColumnIndex(firstFieldIndex, fieldName);
  585. if (matchColumn == NotFound)
  586. throwError1(FVERR_UnrecognisedFieldX, fieldName.get());
  587. DisplayType kind = fieldMeta.getColumnDisplayType(matchColumn);
  588. switch (kind)
  589. {
  590. case TypeBoolean:
  591. case TypeInteger:
  592. case TypeUnsignedInteger:
  593. case TypeReal:
  594. case TypeString:
  595. case TypeData:
  596. case TypeUnicode:
  597. case TypeUnknown:
  598. case TypeSet:
  599. output.column = matchColumn;
  600. break;
  601. case TypeBeginRecord:
  602. //Restrict the search fields to the contents of the record.
  603. firstFieldIndex = matchColumn+1;
  604. break;
  605. case TypeDataset:
  606. {
  607. if (!datasetSelectorAllowed)
  608. throwError1(FVERR_CannotSelectFromDatasetX, fieldName.get());
  609. if (output.datasetColumn != NotFound)
  610. throwError1(FVERR_CannotSelectManyFromDatasetX, fieldName.get());
  611. firstFieldIndex = 0;
  612. curMeta = curMeta->getChildMeta(matchColumn);
  613. output.datasetColumn = matchColumn;
  614. break;
  615. }
  616. default:
  617. throwUnexpected();
  618. }
  619. if (output.column != NotFound)
  620. return;
  621. assertToken('.');
  622. lexToken();
  623. }
  624. }
  625. void MappingParser::parseConstantList(HqlExprArray & args)
  626. {
  627. loop
  628. {
  629. switch (tokenType)
  630. {
  631. case TokInt:
  632. args.append(*curToken.createIntConstant());
  633. break;
  634. case TokString:
  635. args.append(*curToken.createStringConstant());
  636. break;
  637. default:
  638. unsigned len = lenInput - offset > 10 ? 10 : lenInput - offset;
  639. throwError3(FVERR_ExpectedX, "int or string constant", len, input+offset);
  640. }
  641. lexToken();
  642. if (tokenType != ',')
  643. return;
  644. lexToken();
  645. }
  646. }
  647. void MappingParser::parseTransformList(ViewFieldTransformerArray & transforms)
  648. {
  649. loop
  650. {
  651. assertToken(TokId);
  652. StringAttr mappingName, childName, grandName;
  653. curToken.get(mappingName);
  654. lexToken();
  655. if (tokenType == '.')
  656. {
  657. lexToken();
  658. assertToken(TokId);
  659. curToken.get(childName);
  660. lexToken();
  661. }
  662. if (tokenType == '.')
  663. {
  664. lexToken();
  665. assertToken(TokId);
  666. curToken.get(grandName);
  667. lexToken();
  668. }
  669. HqlExprArray args;
  670. if (tokenType == '(')
  671. {
  672. lexToken();
  673. if (tokenType != ')')
  674. parseConstantList(args);
  675. assertToken(')');
  676. lexToken();
  677. }
  678. ViewFieldTransformer * transform;
  679. if (childName)
  680. {
  681. transform = theTransformerRegistry->resolve(mappingName, childName, args);
  682. if (!transform)
  683. {
  684. //Maybe they specified the module name - should provide a 3 valued lookup
  685. transform = theTransformerRegistry->resolve(childName, grandName, args);
  686. }
  687. if (!transform)
  688. throwError2(FVERR_UnrecognisedMappingFunctionXY, mappingName.get(), childName.get());
  689. }
  690. else
  691. {
  692. transform = theTransformerRegistry->resolve(mappingName, args);
  693. if (!transform)
  694. throwError1(FVERR_UnrecognisedMappingFunctionX, mappingName.get());
  695. }
  696. transforms.append(*transform);
  697. if (tokenType != ',')
  698. break;
  699. lexToken();
  700. }
  701. }
  702. void MappingParser::parseAttribute(FieldTransformInfo & output)
  703. {
  704. assertToken(TokId);
  705. if (curToken.eq("get"))
  706. {
  707. lexToken();
  708. assertToken('(');
  709. lexToken();
  710. parseTransformList(output.getTransforms);
  711. assertToken(')');
  712. lexToken();
  713. }
  714. else if (curToken.eq("set"))
  715. {
  716. lexToken();
  717. assertToken('(');
  718. lexToken();
  719. parseTransformList(output.setTransforms);
  720. assertToken(')');
  721. lexToken();
  722. }
  723. else if (curToken.eq("displayname"))
  724. {
  725. lexToken();
  726. assertToken('(');
  727. lexToken();
  728. assertToken(TokId); // could allow a string I guess
  729. curToken.get(output.naturalName);
  730. lexToken();
  731. assertToken(')');
  732. lexToken();
  733. }
  734. else
  735. {
  736. unsigned len = lenInput-offset;
  737. if (len>10) len = 10;
  738. throwError3(FVERR_ExpectedX, "Definition name", len, input+offset);
  739. }
  740. }
  741. void MappingParser::parseColumnMapping(FieldTransformInfo & output)
  742. {
  743. parseColumn(output);
  744. int endToken = '}';
  745. //be flexible and allow () or {}?
  746. if (tokenType == '(')
  747. endToken = ')';
  748. else if (tokenType != '{')
  749. return;
  750. lexToken();
  751. if (tokenType != endToken)
  752. {
  753. loop
  754. {
  755. parseAttribute(output);
  756. if (tokenType != ',')
  757. break;
  758. lexToken();
  759. }
  760. }
  761. assertToken(endToken);
  762. lexToken();
  763. }
  764. void MappingParser::parseColumnMappingList(FieldTransformInfoArray & results, unsigned len, const char * text)
  765. {
  766. lenInput = len;
  767. input = text;
  768. offset = 0;
  769. lexToken();
  770. if (tokenType == TokEof)
  771. return;
  772. loop
  773. {
  774. FieldTransformInfo * next = new FieldTransformInfo;
  775. results.append(*next);
  776. parseColumnMapping(*next);
  777. if (tokenType == TokEof)
  778. break;
  779. assertToken(',');
  780. lexToken();
  781. }
  782. }
  783. void parseColumnMappingList(FieldTransformInfoArray & results,
  784. const IResultSetMetaData & fieldMeta,
  785. bool isDatasetAllowed, // if non null dataset.x is allowed, and column returned via pointer
  786. const char * text)
  787. {
  788. MappingParser parser(fieldMeta, isDatasetAllowed);
  789. parser.parseColumnMappingList(results, strlen(text), text);
  790. }
  791. void parseFileColumnMapping(FieldTransformInfoArray & results, const char * text, const IResultSetMetaData & fieldMeta)
  792. {
  793. MappingParser parser(fieldMeta, false);
  794. parser.parseColumnMappingList(results, strlen(text), text);
  795. }
  796. static void test()
  797. {
  798. HqlExprArray args;
  799. MemoryAttr source;
  800. source.set(26,"Gavin H\303\243lliday !!\316\261\316\221\307\272!!");
  801. {
  802. Owned<ViewFieldTransformer> transform = theTransformerRegistry->resolve("stringlib","StringToUpperCase",args);
  803. MemoryAttr target;
  804. transform->transform(target, source);
  805. }
  806. {
  807. Owned<ViewFieldTransformer> transform = theTransformerRegistry->resolve("unicodelib","UnicodeToUpperCase",args);
  808. MemoryAttr target;
  809. transform->transform(target, source);
  810. }
  811. source.get();
  812. }