rtldynfield.cpp 58 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463
  1. /*##############################################################################
  2. HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. ############################################################################## */
  13. #include "platform.h"
  14. #include <math.h>
  15. #include <stdio.h>
  16. #include "jmisc.hpp"
  17. #include "jlib.hpp"
  18. #include "eclhelper.hpp"
  19. #include "eclrtl_imp.hpp"
  20. #include "rtldynfield.hpp"
  21. #include "rtlrecord.hpp"
  22. #include "rtlembed.hpp"
  23. //#define TRACE_TRANSLATION
  24. #define VALIDATE_TYPEINFO_HASHES
  25. #define RTLTYPEINFO_FORMAT_1 80 // In case we ever want to support more than one format
  26. //---------------------------------------------------------------------------------------------------------------------
  27. const RtlTypeInfo *FieldTypeInfoStruct::createRtlTypeInfo(IThorIndexCallback *_callback) const
  28. {
  29. const RtlTypeInfo *ret = nullptr;
  30. switch (fieldType & RFTMkind)
  31. {
  32. case type_boolean:
  33. ret = new RtlBoolTypeInfo(fieldType, length);
  34. break;
  35. case type_keyedint:
  36. ret = new RtlKeyedIntTypeInfo(fieldType, length, childType);
  37. break;
  38. case type_blob: // MORE - will need its own type (see code below)
  39. case type_int:
  40. ret = new RtlIntTypeInfo(fieldType, length);
  41. break;
  42. #if 0 // Later when implemented
  43. case type_blob:
  44. ret = new RtlBlobTypeInfo(fieldType, length, childType, _callback);
  45. break;
  46. #endif
  47. case type_filepos:
  48. #if __BYTE_ORDER == __LITTLE_ENDIAN
  49. ret = new RtlSwapIntTypeInfo(fieldType, length);
  50. #else
  51. ret = new RtlIntTypeInfo(fieldType, length);
  52. #endif
  53. break;
  54. case type_real:
  55. ret = new RtlRealTypeInfo(fieldType, length);
  56. break;
  57. case type_decimal:
  58. ret = new RtlDecimalTypeInfo(fieldType, length);
  59. break;
  60. case type_string:
  61. ret = new RtlStringTypeInfo(fieldType, length);
  62. break;
  63. case type_bitfield:
  64. ret = new RtlBitfieldTypeInfo(fieldType, length);
  65. break;
  66. case type_varstring:
  67. ret = new RtlVarStringTypeInfo(fieldType, length);
  68. break;
  69. case type_data:
  70. ret = new RtlDataTypeInfo(fieldType, length);
  71. break;
  72. case type_table:
  73. assert(childType);
  74. ret = new RtlDatasetTypeInfo(fieldType, length, childType);
  75. break;
  76. case type_set:
  77. assert(childType);
  78. ret = new RtlSetTypeInfo(fieldType, length, childType);
  79. break;
  80. case type_row:
  81. assert(childType);
  82. ret = new RtlRowTypeInfo(fieldType, length, childType);
  83. break;
  84. case type_swapint:
  85. ret = new RtlSwapIntTypeInfo(fieldType, length);
  86. break;
  87. case type_packedint:
  88. ret = new RtlPackedIntTypeInfo(fieldType, length);
  89. break;
  90. case type_qstring:
  91. ret = new RtlQStringTypeInfo(fieldType, length);
  92. break;
  93. case type_unicode:
  94. ret = new RtlUnicodeTypeInfo(fieldType, length, locale);
  95. break;
  96. case type_varunicode:
  97. ret = new RtlVarUnicodeTypeInfo(fieldType, length, locale);
  98. break;
  99. case type_utf8:
  100. ret = new RtlUtf8TypeInfo(fieldType, length, locale);
  101. break;
  102. case type_record:
  103. ret = new RtlRecordTypeInfo(fieldType, length, fieldsArray);
  104. break;
  105. case type_ifblock:
  106. ret = new RtlDynamicIfBlockTypeInfo(fieldType, length, fieldsArray);
  107. break;
  108. default:
  109. throwUnexpected();
  110. }
  111. return ret;
  112. };
  113. typedef MapBetween<const RtlTypeInfo *, const RtlTypeInfo *, StringAttr, const char *> TypeNameMap;
  114. typedef MapBetween<const RtlTypeInfo *, const RtlTypeInfo *, unsigned, unsigned> TypeNumMap;
  115. /**
  116. * class CRtlFieldTypeSerializer
  117. *
  118. * Serializer class for creating json representation of a RtlTypeInfo structure.
  119. *
  120. */
  121. class CRtlFieldTypeSerializer
  122. {
  123. public:
  124. /**
  125. * Serialize a RtlTypeInfo structure to JSON
  126. *
  127. * @param out Buffer for resulting serialized string
  128. * @param type RtlTypeInfo structure to be serialized
  129. * @return Referenced to supplied buffer
  130. */
  131. static StringBuffer &serialize(StringBuffer &out, const RtlTypeInfo *type)
  132. {
  133. CRtlFieldTypeSerializer s(out, type);
  134. s.doSerialize();
  135. return out;
  136. }
  137. private:
  138. CRtlFieldTypeSerializer(StringBuffer &_out, const RtlTypeInfo *_base)
  139. : json(_out), base(_base)
  140. {
  141. }
  142. void doSerialize()
  143. {
  144. json.append("{");
  145. serializeType(base);
  146. json.append("\n}");
  147. }
  148. void serializeType(const RtlTypeInfo *type)
  149. {
  150. if (!serialized(type))
  151. {
  152. // Make sure all child types are serialized first
  153. const RtlTypeInfo *child = type->queryChildType();
  154. if (child)
  155. serializeType(child);
  156. const RtlFieldInfo * const * fields = type->queryFields();
  157. if (fields)
  158. {
  159. for (;;)
  160. {
  161. const RtlFieldInfo * child = *fields;
  162. if (!child)
  163. break;
  164. serializeType(child->type);
  165. fields++;
  166. }
  167. }
  168. // Now serialize this one
  169. if (type != base)
  170. {
  171. VStringBuffer newName("ty%d", ++nextTypeName);
  172. types.setValue(type, newName.str());
  173. startField(newName.str());
  174. serializeMe(type);
  175. closeCurly();
  176. }
  177. else
  178. serializeMe(type);
  179. }
  180. }
  181. void serializeMe(const RtlTypeInfo *type)
  182. {
  183. if (!type->canSerialize())
  184. throw makeStringException(MSGAUD_user, 1000, "This type structure cannot be serialized");
  185. addPropHex("fieldType", type->fieldType);
  186. addProp("length", type->length);
  187. addPropNonEmpty("locale", type->queryLocale());
  188. const RtlTypeInfo *child = type->queryChildType();
  189. if (child)
  190. addPropType("child", child);
  191. const RtlFieldInfo * const * fields = type->queryFields();
  192. if (fields)
  193. {
  194. startFields();
  195. for (;;)
  196. {
  197. const RtlFieldInfo * child = *fields;
  198. if (!child)
  199. break;
  200. newline();
  201. openCurly();
  202. addProp("name", child->name);
  203. addPropType("type", child->type);
  204. addProp("xpath", child->xpath);
  205. if (child->flags)
  206. addPropHex("flags", child->flags);
  207. // initializer is tricky - it's not (in general) a null-terminated string but the actual length is not easily available
  208. if (child->initializer)
  209. {
  210. addProp("init", child->type->size(child->initializer, nullptr), child->initializer);
  211. }
  212. closeCurly();
  213. fields++;
  214. }
  215. endFields();
  216. }
  217. }
  218. bool serialized(const RtlTypeInfo *type)
  219. {
  220. return types.find(type) != nullptr;
  221. }
  222. void startField(const char *name)
  223. {
  224. newline().appendf("\"%s\": ", name);
  225. openCurly();
  226. }
  227. void addProp(const char *propName, const char *propVal)
  228. {
  229. if (propVal)
  230. {
  231. newline();
  232. encodeJSON(json.append("\""), propName).append("\": ");
  233. encodeJSON(json.append("\""), propVal).append("\"");
  234. }
  235. }
  236. void addProp(const char *propName, size32_t propLen, const byte *propVal)
  237. {
  238. if (propVal)
  239. {
  240. newline();
  241. encodeJSON(json.append("\""), propName).append("\": \"");
  242. JBASE64_Encode(propVal, propLen, json, false);
  243. json.append("\"");
  244. }
  245. }
  246. void addPropNonEmpty(const char *propName, const char *propVal)
  247. {
  248. if (propVal && *propVal)
  249. addProp(propName, propVal);
  250. }
  251. void addProp(const char *propName, unsigned propVal)
  252. {
  253. newline().appendf("\"%s\": %u", propName, propVal);
  254. }
  255. void addPropHex(const char *propName, unsigned propVal)
  256. {
  257. newline().appendf("\"%s\": %u", propName, propVal); // Nice idea but json does not support hex constants :(
  258. }
  259. void addPropType(const char *propName, const RtlTypeInfo *type)
  260. {
  261. addProp(propName, queryTypeName(type));
  262. }
  263. const char *queryTypeName(const RtlTypeInfo *type)
  264. {
  265. StringAttr *typeName = types.getValue(type);
  266. assertex(typeName);
  267. return typeName->get();
  268. }
  269. void startFields()
  270. {
  271. newline().appendf("\"fields\": ");
  272. openCurly('[');
  273. }
  274. void endFields()
  275. {
  276. closeCurly(']');
  277. }
  278. StringBuffer &newline()
  279. {
  280. if (commaPending)
  281. json.append(',');
  282. json.appendf("\n%*s", indent, "");
  283. commaPending = true;
  284. return json;
  285. }
  286. void closeCurly(char brace = '}')
  287. {
  288. indent--;
  289. json.appendf("\n%*s%c", indent, "", brace);
  290. commaPending = true;
  291. }
  292. void openCurly(char brace = '{')
  293. {
  294. json.append(brace);
  295. indent++;
  296. commaPending = false;
  297. }
  298. TypeNameMap types;
  299. StringBuffer &json;
  300. const RtlTypeInfo *base = nullptr;
  301. unsigned indent = 1;
  302. unsigned nextTypeName = 0;
  303. bool commaPending = false;
  304. };
  305. class CRtlFieldTypeBinSerializer
  306. {
  307. public:
  308. /**
  309. * Serialize a RtlTypeInfo structure to binary
  310. *
  311. * @param out Buffer for resulting serialized string
  312. * @param type RtlTypeInfo structure to be serialized
  313. * @return Referenced to supplied buffer
  314. */
  315. static MemoryBuffer &serialize(MemoryBuffer &out, const RtlTypeInfo *type)
  316. {
  317. int oldEnd = out.setEndian(__LITTLE_ENDIAN);
  318. CRtlFieldTypeBinSerializer s(out);
  319. byte format = RTLTYPEINFO_FORMAT_1;
  320. out.append(format);
  321. DelayedMarker<hash64_t> hash(out);
  322. DelayedSizeMarker size(out);
  323. size32_t pos = out.length();
  324. s.serializeType(type);
  325. size.write();
  326. hash.write(rtlHash64Data(size.size(), out.toByteArray()+pos, 0));
  327. out.setEndian(oldEnd);
  328. return out;
  329. }
  330. private:
  331. CRtlFieldTypeBinSerializer(MemoryBuffer &_out)
  332. : out(_out)
  333. {
  334. }
  335. void serializeType(const RtlTypeInfo *type)
  336. {
  337. if (!serialized(type))
  338. {
  339. // Make sure all child types are serialized first
  340. const RtlTypeInfo *child = type->queryChildType();
  341. if (child)
  342. serializeType(child);
  343. const RtlFieldInfo * const * fields = type->queryFields();
  344. if (fields)
  345. {
  346. for (unsigned idx = 0;;idx++)
  347. {
  348. const RtlFieldInfo * child = fields[idx];
  349. if (!child)
  350. break;
  351. serializeType(child->type);
  352. }
  353. }
  354. // Now serialize this one
  355. types.setValue(type, nextTypeNum++);
  356. serializeMe(type);
  357. }
  358. }
  359. void serializeMe(const RtlTypeInfo *type)
  360. {
  361. if (!type->canSerialize())
  362. throw makeStringException(MSGAUD_user, 1000, "This type structure cannot be serialized");
  363. unsigned fieldType = type->fieldType;
  364. const char *locale = type->queryLocale();
  365. if (locale && *locale)
  366. fieldType |= RFTMhasLocale;
  367. const RtlTypeInfo *child = type->queryChildType();
  368. if (child)
  369. fieldType |= RFTMhasChildType;
  370. const RtlFieldInfo * const * fields = type->queryFields();
  371. if (fields)
  372. fieldType |= RFTMhasFields;
  373. out.append(fieldType);
  374. out.appendPacked(type->length);
  375. if (fieldType & RFTMhasLocale)
  376. out.append(locale);
  377. if (child)
  378. out.appendPacked(queryTypeIdx(child));
  379. if (fields)
  380. {
  381. unsigned count = countFields(fields);
  382. out.appendPacked(count);
  383. for (;;)
  384. {
  385. const RtlFieldInfo * child = *fields;
  386. if (!child)
  387. break;
  388. out.append(child->name);
  389. out.appendPacked(queryTypeIdx(child->type));
  390. unsigned flags = child->flags;
  391. if (child->xpath)
  392. flags |= RFTMhasXpath;
  393. if (child->initializer)
  394. flags |= RFTMhasInitializer;
  395. out.append(flags);
  396. if (child->xpath)
  397. out.append(child->xpath);
  398. // initializer is tricky - it's not (in general) a null-terminated string but the actual length is not easily available
  399. if (child->initializer)
  400. {
  401. unsigned initLength = child->type->size(child->initializer, nullptr);
  402. out.appendPacked(initLength).append(initLength, child->initializer);
  403. }
  404. fields++;
  405. }
  406. }
  407. }
  408. bool serialized(const RtlTypeInfo *type)
  409. {
  410. return types.find(type) != nullptr;
  411. }
  412. unsigned queryTypeIdx(const RtlTypeInfo *type)
  413. {
  414. unsigned *typeNum = types.getValue(type);
  415. assertex(typeNum);
  416. return *typeNum;
  417. }
  418. TypeNumMap types;
  419. MemoryBuffer &out;
  420. unsigned nextTypeNum = 0;
  421. };
  422. /**
  423. * class CRtlFieldTypeDeserializer
  424. *
  425. * Deserializer class for creating a RtlTypeInfo structure from json representation.
  426. *
  427. * Note that the resulting RtlTypeInfo structures are owned by this object and will be
  428. * destroyed when this object is destroyed.
  429. *
  430. */
  431. class CRtlFieldTypeDeserializer : public CInterfaceOf<IRtlFieldTypeDeserializer>
  432. {
  433. public:
  434. /**
  435. * CRtlFieldTypeDeserializer constructor
  436. *
  437. * @param _callback Supplies a callback to be used for blobs/filepositions.
  438. */
  439. CRtlFieldTypeDeserializer(IThorIndexCallback *_callback)
  440. : callback(_callback)
  441. {
  442. }
  443. /**
  444. * CRtlFieldTypeDeserializer destructor
  445. * <p>
  446. * Releases all RtlTypeInfo and related structures created by this deserializer
  447. */
  448. ~CRtlFieldTypeDeserializer()
  449. {
  450. // Need some care - all the RtlTypeInfo objects I created need to be destroyed, together with anything else I had to create
  451. // Strings (other than the init strings) are preserved in the AtomTable
  452. HashIterator allTypes(types);
  453. ForEach(allTypes)
  454. {
  455. const RtlTypeInfo **type = types.mapToValue(&allTypes.query());
  456. cleanupType(*type);
  457. }
  458. cleanupType(base);
  459. }
  460. /**
  461. * Obtain the deserialized type information
  462. * <p>
  463. * Note that the RtlTypeInfo objects are not link-counted, so the lifetime of these objects
  464. * is determined by the lifetime of the deserializer. They will be released once the deserializer
  465. * that created them is deleted.
  466. * <p>
  467. * Do not call more than once.
  468. *
  469. * @param _json JSON text to be deserialized, as created by CRtlFieldTypeSerializer
  470. * @return Deserialized type object
  471. */
  472. virtual const RtlTypeInfo *deserialize(const char *json) override
  473. {
  474. assertex(!base);
  475. Owned<IPropertyTree> jsonTree = createPTreeFromJSONString(json);
  476. base = deserializeType(jsonTree, jsonTree);
  477. return base;
  478. }
  479. /**
  480. * Obtain the deserialized type information
  481. * <p>
  482. * Note that the RtlTypeInfo objects are not link-counted, so the lifetime of these objects
  483. * is determined by the lifetime of the deserializer. They will be released once the deserializer
  484. * that created them is deleted.
  485. * <p>
  486. * Do not call more than once.
  487. *
  488. * @param _jsonTree JSON property tree to be deserialized, as created by CRtlFieldTypeSerializer
  489. * @return Deserialized type object
  490. */
  491. virtual const RtlTypeInfo *deserialize(IPropertyTree &jsonTree) override
  492. {
  493. assertex(!base);
  494. base = deserializeType(&jsonTree, &jsonTree);
  495. return base;
  496. }
  497. /**
  498. * Obtain the deserialized type information
  499. * <p>
  500. * Note that the RtlTypeInfo objects are not link-counted, so the lifetime of these objects
  501. * is determined by the lifetime of the deserializer. They will be released once the deserializer
  502. * that created them is deleted.
  503. * <p>
  504. * Do not call more than once.
  505. *
  506. * @param buf Binary serialized typeinfo to be deserialized, as created by CRtlFieldTypeSerializer
  507. * @return Deserialized type object
  508. */
  509. virtual const RtlTypeInfo *deserialize(MemoryBuffer &buf) override
  510. {
  511. assertex(!base);
  512. unsigned nextTypeNum = 0;
  513. int oldEndian = buf.setEndian(__LITTLE_ENDIAN);
  514. try
  515. {
  516. byte format;
  517. buf.read(format);
  518. if (format != RTLTYPEINFO_FORMAT_1)
  519. throw MakeStringException(0, "Invalid type info (%d) in CRtlFieldTypeDeserializer::deserialize", format);
  520. hash64_t hash;
  521. buf.read(hash);
  522. size32_t size;
  523. buf.read(size);
  524. #ifdef VALIDATE_TYPEINFO_HASHES
  525. hash64_t expected = rtlHash64Data(size, buf.readDirect(0), 0);
  526. if (expected != hash)
  527. throw MakeStringException(0, "Invalid type info hash in CRtlFieldTypeDeserializer::deserialize");
  528. #endif
  529. size32_t endpos = buf.getPos() + size;
  530. while (buf.getPos() < endpos)
  531. {
  532. if (base)
  533. {
  534. addType(base, nextTypeNum++);
  535. base = nullptr; // in case of exceptions...
  536. }
  537. base = deserializeType(buf);
  538. }
  539. if (buf.getPos()!=endpos)
  540. throw MakeStringException(0, "Invalid type info (incorrect size data) in CRtlFieldTypeDeserializer::deserialize");
  541. buf.setEndian(oldEndian);
  542. return base;
  543. }
  544. catch(...)
  545. {
  546. buf.setEndian(oldEndian);
  547. throw;
  548. }
  549. }
  550. virtual const RtlTypeInfo *addType(FieldTypeInfoStruct &info, const ITypeInfo *type) override
  551. {
  552. VStringBuffer name("%p", type);
  553. const RtlTypeInfo ** found = types.getValue(name);
  554. if (found)
  555. return *found;
  556. info.locale = keep(info.locale);
  557. const RtlTypeInfo * ret = info.createRtlTypeInfo(callback);
  558. types.setValue(name, ret);
  559. return ret;
  560. }
  561. virtual const RtlTypeInfo *lookupType(const ITypeInfo *type) const override
  562. {
  563. VStringBuffer name("%p", type);
  564. const RtlTypeInfo ** found = types.getValue(name);
  565. if (found)
  566. return *found;
  567. return nullptr;
  568. }
  569. virtual const RtlFieldInfo *addFieldInfo(const char *fieldName, const char *xpath, const RtlTypeInfo *type, unsigned flags, const char *init) override
  570. {
  571. // MORE - we could hang onto this for cleanup, rather than assuming that we keep it via a later addType() call?
  572. return new RtlFieldStrInfo(keep(fieldName), keep(xpath), type, flags, init);
  573. }
  574. private:
  575. KeptAtomTable atoms; // Used to ensure proper lifetime of strings used in type structures
  576. MapStringTo<const RtlTypeInfo *> types; // Ensures structures only generated once
  577. const RtlTypeInfo *base = nullptr; // Holds the resulting type
  578. IThorIndexCallback *callback = nullptr;
  579. void cleanupType(const RtlTypeInfo *type)
  580. {
  581. if (type)
  582. {
  583. // Releases all memory for a single RtlTypeInfo object
  584. const RtlFieldInfo * const * fields = type->queryFields();
  585. if (fields)
  586. {
  587. const RtlFieldInfo * const * cur = fields;
  588. for (;;)
  589. {
  590. const RtlFieldInfo * child = *cur;
  591. if (!child)
  592. break;
  593. // We don't need to delete other strings - they are owned by atom table.
  594. // But the initializer is decoded and thus owned by me
  595. delete child->initializer;
  596. delete child;
  597. cur++;
  598. }
  599. delete [] fields;
  600. }
  601. type->doDelete();
  602. }
  603. }
  604. const RtlTypeInfo *lookupType(const char *name, IPropertyTree *all)
  605. {
  606. const RtlTypeInfo ** found = types.getValue(name);
  607. if (found)
  608. return *found;
  609. const RtlTypeInfo *type = deserializeType(all->queryPropTree(name), all);
  610. types.setValue(name, type);
  611. return type;
  612. }
  613. const RtlTypeInfo *lookupType(unsigned idx)
  614. {
  615. // Could keep an expanding array of types instead - but the hash table is already there for json support...
  616. VStringBuffer key("%u", idx);
  617. const RtlTypeInfo ** found = types.getValue(key);
  618. if (found)
  619. return *found;
  620. throw makeStringException(-1, "Invalid serialized type information");
  621. }
  622. void addType(const RtlTypeInfo *type, unsigned idx)
  623. {
  624. VStringBuffer key("%u", idx);
  625. assert(types.getValue(key)==nullptr);
  626. types.setValue(key, type);
  627. }
  628. const char *keep(const char *string)
  629. {
  630. if (string)
  631. return str(atoms.addAtom(string));
  632. else
  633. return nullptr;
  634. }
  635. const RtlTypeInfo *deserializeType(IPropertyTree *type, IPropertyTree *all)
  636. {
  637. FieldTypeInfoStruct info;
  638. info.fieldType = type->getPropInt("fieldType");
  639. info.length = type->getPropInt("length");
  640. info.locale = keep(type->queryProp("locale"));
  641. const char *child = type->queryProp("child");
  642. if (child)
  643. info.childType = lookupType(child, all);
  644. if ((info.fieldType & RFTMkind) == type_record)
  645. {
  646. unsigned numFields = type->getCount("fields");
  647. info.fieldsArray = new const RtlFieldInfo * [numFields+1];
  648. info.fieldsArray[numFields] = nullptr;
  649. Owned<IPropertyTreeIterator> fields = type->getElements("fields");
  650. unsigned n = 0;
  651. ForEach(*fields)
  652. {
  653. IPropertyTree &field = fields->query();
  654. const char *fieldTypeName = field.queryProp("type");
  655. const char *fieldName = keep(field.queryProp("name"));
  656. const char *fieldXpath = keep(field.queryProp("xpath"));
  657. unsigned flags = field.getPropInt("flags");
  658. const char *fieldInit = field.queryProp("init");
  659. if (fieldInit)
  660. {
  661. StringBuffer decoded;
  662. JBASE64_Decode(fieldInit, decoded);
  663. fieldInit = decoded.detach(); // NOTE - this gets freed in cleanupType()
  664. }
  665. info.fieldsArray[n] = new RtlFieldStrInfo(fieldName, fieldXpath, lookupType(fieldTypeName, all), flags, fieldInit);
  666. n++;
  667. }
  668. }
  669. return info.createRtlTypeInfo(callback);
  670. }
  671. const RtlTypeInfo *deserializeType(MemoryBuffer &type)
  672. {
  673. FieldTypeInfoStruct info;
  674. type.read(info.fieldType);
  675. type.readPacked(info.length);
  676. if (info.fieldType & RFTMhasLocale)
  677. {
  678. const char *locale;
  679. type.read(locale);
  680. info.locale = keep(locale);
  681. }
  682. if (info.fieldType & RFTMhasChildType)
  683. {
  684. unsigned childIdx;
  685. type.readPacked(childIdx);
  686. info.childType = lookupType(childIdx);
  687. }
  688. if (info.fieldType & RFTMhasFields)
  689. {
  690. unsigned numFields;
  691. type.readPacked(numFields);
  692. info.fieldsArray = new const RtlFieldInfo * [numFields+1];
  693. info.fieldsArray[numFields] = nullptr;
  694. for (int n = 0; n < numFields; n++)
  695. {
  696. const char *fieldName;
  697. type.read(fieldName);
  698. unsigned fieldType;
  699. type.readPacked(fieldType);
  700. unsigned fieldFlags;
  701. type.read(fieldFlags);
  702. const char *xpath = nullptr;
  703. if (fieldFlags & RFTMhasXpath)
  704. type.read(xpath);
  705. void *init = nullptr;
  706. if (fieldFlags & RFTMhasInitializer)
  707. {
  708. unsigned initLength;
  709. type.readPacked(initLength);
  710. init = malloc(initLength);
  711. memcpy(init, type.readDirect(initLength), initLength);
  712. }
  713. fieldFlags &= ~RFTMserializerFlags;
  714. info.fieldsArray[n] = new RtlFieldStrInfo(keep(fieldName), keep(xpath), lookupType(fieldType), fieldFlags, (const char *) init);
  715. }
  716. }
  717. info.fieldType &= ~RFTMserializerFlags;
  718. return info.createRtlTypeInfo(callback);
  719. }
  720. void patchIndexFilePos()
  721. {
  722. if (callback && (base->fieldType & RFTMkind) == type_record)
  723. {
  724. // Yukky hack time
  725. // Assumes that the fieldinfo is not shared...
  726. // But that is also assumed by the code that cleans them up.
  727. const RtlFieldInfo * const *fields = base->queryFields();
  728. for(;;)
  729. {
  730. const RtlFieldInfo *field = *fields++;
  731. if (!field)
  732. break;
  733. if (field->type->getType() == type_blob)
  734. {
  735. static_cast<RtlBlobTypeInfo *>(const_cast<RtlTypeInfo *>(field->type))->setCallback(callback);
  736. }
  737. }
  738. }
  739. }
  740. };
  741. extern ECLRTL_API IRtlFieldTypeDeserializer *createRtlFieldTypeDeserializer(IThorIndexCallback *callback)
  742. {
  743. return new CRtlFieldTypeDeserializer(callback);
  744. }
  745. extern ECLRTL_API StringBuffer &dumpTypeInfo(StringBuffer &ret, const RtlTypeInfo *t)
  746. {
  747. return CRtlFieldTypeSerializer::serialize(ret, t);
  748. }
  749. extern ECLRTL_API MemoryBuffer &dumpTypeInfo(MemoryBuffer &ret, const RtlTypeInfo *t)
  750. {
  751. return CRtlFieldTypeBinSerializer::serialize(ret, t);
  752. }
  753. extern ECLRTL_API void serializeRecordType(size32_t & __lenResult, void * & __result, IOutputMetaData & metaVal)
  754. {
  755. MemoryBuffer ret;
  756. CRtlFieldTypeBinSerializer::serialize(ret, metaVal.queryTypeInfo());
  757. __lenResult = ret.length();
  758. __result = ret.detach();
  759. }
  760. extern ECLRTL_API void dumpRecordType(size32_t & __lenResult,char * & __result,IOutputMetaData &metaVal)
  761. {
  762. StringBuffer ret;
  763. CRtlFieldTypeSerializer::serialize(ret, metaVal.queryTypeInfo());
  764. #ifdef _DEBUG
  765. StringBuffer ret2;
  766. CRtlFieldTypeDeserializer deserializer(nullptr);
  767. CRtlFieldTypeSerializer::serialize(ret2, deserializer.deserialize(ret));
  768. assert(streq(ret, ret2));
  769. MemoryBuffer out;
  770. CRtlFieldTypeBinSerializer::serialize(out, metaVal.queryTypeInfo());
  771. CRtlFieldTypeDeserializer bindeserializer(nullptr);
  772. CRtlFieldTypeSerializer::serialize(ret2.clear(), bindeserializer.deserialize(out));
  773. assert(streq(ret, ret2));
  774. #endif
  775. __lenResult = ret.length();
  776. __result = ret.detach();
  777. }
  778. extern ECLRTL_API void getFieldVal(size32_t & __lenResult,char * & __result, int column, IOutputMetaData & metaVal, const byte *row)
  779. {
  780. __lenResult = 0;
  781. __result = nullptr;
  782. if (column >= 0)
  783. {
  784. const RtlRecord &r = metaVal.queryRecordAccessor(true);
  785. unsigned numOffsets = r.getNumVarFields() + 1;
  786. size_t * variableOffsets = (size_t *)alloca(numOffsets * sizeof(size_t));
  787. RtlRow offsetCalculator(r, row, numOffsets, variableOffsets);
  788. offsetCalculator.getUtf8(__lenResult, __result, column);
  789. }
  790. }
  791. extern ECLRTL_API int getFieldNum(const char *fieldName, IOutputMetaData & metaVal)
  792. {
  793. const RtlRecord r = metaVal.queryRecordAccessor(true);
  794. return r.getFieldNum(fieldName);
  795. }
  796. enum FieldMatchType {
  797. // On a field, exactly one of the below is set, but translator returns a bitmap indicating
  798. // which were required (and we can restrict translation to allow some types but not others)
  799. match_perfect = 0x00, // exact type match - use memcpy
  800. match_link = 0x01, // copy a nested dataset by linking
  801. match_move = 0x02, // at least one field has moved (set on translator)
  802. match_remove = 0x04, // at least one field has been removed (set on translator)
  803. match_truncate = 0x08, // dest is truncated copy of source - use memcpy
  804. match_extend = 0x10, // dest is padded version of source - use memcpy and memset
  805. match_typecast = 0x20, // type has changed - cast required
  806. match_none = 0x40, // No matching field in source - use null value
  807. match_recurse = 0x80, // Use recursive translator for child records/datasets
  808. match_fail = 0x100, // no translation possible
  809. // This flag may be set in conjunction with the others
  810. match_inifblock = 0x200, // matching to a field in an ifblock - may not be present
  811. };
  812. StringBuffer &describeFlags(StringBuffer &out, FieldMatchType flags)
  813. {
  814. if (flags == match_perfect)
  815. return out.append("perfect");
  816. unsigned origlen = out.length();
  817. if (flags & match_link) out.append("|link");
  818. if (flags & match_move) out.append("|move");
  819. if (flags & match_remove) out.append("|remove");
  820. if (flags & match_truncate) out.append("|truncate");
  821. if (flags & match_extend) out.append("|extend");
  822. if (flags & match_typecast) out.append("|typecast");
  823. if (flags & match_none) out.append("|none");
  824. if (flags & match_recurse) out.append("|recurse");
  825. if (flags & match_inifblock) out.append("|ifblock");
  826. if (flags & match_fail) out.append("|fail");
  827. assertex(out.length() > origlen);
  828. return out.remove(origlen, 1);
  829. }
  830. inline constexpr FieldMatchType operator|(FieldMatchType a, FieldMatchType b) { return (FieldMatchType)((int)a | (int)b); }
  831. inline FieldMatchType &operator|=(FieldMatchType &a, FieldMatchType b) { return (FieldMatchType &) ((int &)a |= (int)b); }
  832. class GeneralRecordTranslator : public CInterfaceOf<IDynamicTransform>
  833. {
  834. public:
  835. GeneralRecordTranslator(const RtlRecord &_destRecInfo, const RtlRecord &_srcRecInfo)
  836. : destRecInfo(_destRecInfo), sourceRecInfo(_srcRecInfo)
  837. {
  838. matchInfo = new MatchInfo[destRecInfo.getNumFields()];
  839. createMatchInfo();
  840. }
  841. ~GeneralRecordTranslator()
  842. {
  843. delete [] matchInfo;
  844. }
  845. virtual void describe() const override
  846. {
  847. doDescribe(0);
  848. }
  849. virtual size32_t translate(ARowBuilder &builder, const byte *sourceRec) const override
  850. {
  851. dbgassertex(canTranslate());
  852. return doTranslate(builder, 0, sourceRec);
  853. }
  854. virtual bool canTranslate() const override
  855. {
  856. return (matchFlags & match_fail) == 0;
  857. }
  858. virtual bool needsTranslate() const override
  859. {
  860. return (matchFlags & ~match_link) != 0;
  861. }
  862. private:
  863. void doDescribe(unsigned indent) const
  864. {
  865. for (unsigned idx = 0; idx < destRecInfo.getNumFields(); idx++)
  866. {
  867. const char *source = destRecInfo.queryName(idx);
  868. const MatchInfo &match = matchInfo[idx];
  869. if (match.matchType == match_none)
  870. DBGLOG("%*sNo match for field %s - default value will be used", indent, "", source);
  871. else
  872. {
  873. StringBuffer matchStr;
  874. DBGLOG("%*sMatch (%s) to field %d for field %s (%x)", indent, "", describeFlags(matchStr, match.matchType).str(), match.matchIdx, source, destRecInfo.queryType(idx)->fieldType);
  875. if (match.subTrans)
  876. match.subTrans->doDescribe(indent+2);
  877. }
  878. }
  879. if (!canTranslate())
  880. DBGLOG("%*sTranslation is NOT possible", indent, "");
  881. else if (needsTranslate())
  882. {
  883. StringBuffer matchStr;
  884. DBGLOG("%*sTranslation is possible (%s)", indent, "", describeFlags(matchStr, matchFlags).str());
  885. }
  886. else
  887. DBGLOG("%*sTranslation is not necessary", indent, "");
  888. }
  889. size32_t doTranslate(ARowBuilder &builder, size32_t offset, const byte *sourceRec) const
  890. {
  891. unsigned numOffsets = sourceRecInfo.getNumVarFields() + 1;
  892. size_t * variableOffsets = (size_t *)alloca(numOffsets * sizeof(size_t));
  893. byte * destConditions = (byte *)alloca(destRecInfo.getNumIfBlocks() * sizeof(byte));
  894. RtlRow sourceRow(sourceRecInfo, sourceRec, numOffsets, variableOffsets);
  895. size32_t estimate = destRecInfo.getFixedSize();
  896. if (!estimate)
  897. {
  898. estimate = estimateNewSize(sourceRow);
  899. builder.ensureCapacity(offset+estimate, "record");
  900. }
  901. size32_t origOffset = offset;
  902. for (unsigned idx = 0; idx < destRecInfo.getNumFields(); idx++)
  903. {
  904. const RtlFieldInfo *field = destRecInfo.queryField(idx);
  905. if (field->omitable() && destRecInfo.excluded(field, builder.getSelf(), destConditions))
  906. continue;
  907. const RtlTypeInfo *type = field->type;
  908. const MatchInfo &match = matchInfo[idx];
  909. if (match.matchType == match_none || match.matchType==match_fail)
  910. offset = type->buildNull(builder, offset, field);
  911. else
  912. {
  913. unsigned matchField = match.matchIdx;
  914. const RtlTypeInfo *sourceType = sourceRecInfo.queryType(matchField);
  915. size_t sourceOffset = sourceRow.getOffset(matchField);
  916. const byte *source = sourceRec + sourceOffset;
  917. size_t copySize = sourceRow.getSize(matchField);
  918. if (copySize == 0 && (match.matchType & match_inifblock)) // Field is missing because of an ifblock - use default value
  919. {
  920. offset = type->buildNull(builder, offset, field);
  921. }
  922. else
  923. {
  924. switch (match.matchType & ~match_inifblock)
  925. {
  926. case match_perfect:
  927. {
  928. // Look ahead for other perfect matches and combine the copies
  929. while (idx < destRecInfo.getNumFields()-1)
  930. {
  931. const MatchInfo &nextMatch = matchInfo[idx+1];
  932. if (nextMatch.matchType == match_perfect && nextMatch.matchIdx == matchField+1)
  933. {
  934. idx++;
  935. matchField++;
  936. }
  937. else
  938. break;
  939. }
  940. size_t copySize = sourceRow.getOffset(matchField+1) - sourceOffset;
  941. builder.ensureCapacity(offset+copySize, field->name);
  942. memcpy(builder.getSelf()+offset, source, copySize);
  943. offset += copySize;
  944. break;
  945. }
  946. case match_truncate:
  947. {
  948. assert(type->isFixedSize());
  949. size32_t copySize = type->getMinSize();
  950. builder.ensureCapacity(offset+copySize, field->name);
  951. memcpy(builder.getSelf()+offset, source, copySize);
  952. offset += copySize;
  953. break;
  954. }
  955. case match_extend:
  956. {
  957. assert(type->isFixedSize());
  958. size32_t destSize = type->getMinSize();
  959. builder.ensureCapacity(offset+destSize, field->name);
  960. memcpy(builder.getSelf()+offset, source, copySize);
  961. offset += copySize;
  962. unsigned fillSize = destSize - copySize;
  963. memset(builder.getSelf()+offset, match.fillChar, fillSize);
  964. offset += fillSize;
  965. break;
  966. }
  967. case match_typecast:
  968. offset = translateScalar(builder, offset, field, type, sourceType, source);
  969. break;
  970. case match_link:
  971. {
  972. // a 32-bit record count, and a (linked) pointer to an array of record pointers
  973. byte *dest = builder.ensureCapacity(offset+sizeof(size32_t)+sizeof(const byte **), field->name)+offset;
  974. *(size32_t *)dest = *(size32_t *)source;
  975. *(const byte ***)(dest + sizeof(size32_t)) = rtlLinkRowset(*(const byte ***)(source + sizeof(size32_t)));
  976. offset += sizeof(size32_t)+sizeof(const byte **);
  977. break;
  978. }
  979. case match_recurse:
  980. if (type->getType()==type_record)
  981. offset = match.subTrans->doTranslate(builder, offset, source);
  982. else if (type->isLinkCounted())
  983. {
  984. // a 32-bit record count, and a pointer to an array of record pointers
  985. IEngineRowAllocator *childAllocator = builder.queryAllocator()->createChildRowAllocator(type->queryChildType());
  986. assertex(childAllocator); // May not be available when using serialized types (but unlikely to want to create linkcounted children remotely either)
  987. size32_t sizeInBytes = sizeof(size32_t) + sizeof(void *);
  988. builder.ensureCapacity(offset+sizeInBytes, field->name);
  989. size32_t numRows = 0;
  990. const byte **childRows = nullptr;
  991. if (sourceType->isLinkCounted())
  992. {
  993. // a 32-bit count, then a pointer to the source rows
  994. size32_t childCount = *(size32_t *) source;
  995. source += sizeof(size32_t);
  996. const byte ** sourceRows = *(const byte***) source;
  997. for (size32_t childRow = 0; childRow < childCount; childRow++)
  998. {
  999. RtlDynamicRowBuilder childBuilder(*childAllocator);
  1000. size32_t childLen = match.subTrans->doTranslate(childBuilder, 0, sourceRows[childRow]);
  1001. childRows = childAllocator->appendRowOwn(childRows, ++numRows, (void *) childBuilder.finalizeRowClear(childLen));
  1002. }
  1003. }
  1004. else
  1005. {
  1006. // a 32-bit size, then rows inline
  1007. size32_t childSize = *(size32_t *) source;
  1008. source += sizeof(size32_t);
  1009. const byte *initialSource = source;
  1010. while ((size_t)(source - initialSource) < childSize)
  1011. {
  1012. RtlDynamicRowBuilder childBuilder(*childAllocator);
  1013. size32_t childLen = match.subTrans->doTranslate(childBuilder, 0, source);
  1014. childRows = childAllocator->appendRowOwn(childRows, ++numRows, (void *) childBuilder.finalizeRowClear(childLen));
  1015. source += sourceType->queryChildType()->size(source, nullptr); // MORE - shame to repeat a calculation that the translate above almost certainly just did
  1016. }
  1017. }
  1018. // Go back in and patch the count, remembering it may have moved
  1019. rtlWriteInt4(builder.getSelf()+offset, numRows);
  1020. * ( const void * * ) (builder.getSelf()+offset+sizeof(size32_t)) = childRows;
  1021. offset += sizeInBytes;
  1022. }
  1023. else
  1024. {
  1025. size32_t countOffset = offset;
  1026. byte *dest = builder.ensureCapacity(offset+sizeof(size32_t), field->name)+offset;
  1027. offset += sizeof(size32_t);
  1028. size32_t initialOffset = offset;
  1029. *(size32_t *)dest = 0; // patched below when true figure known
  1030. if (sourceType->isLinkCounted())
  1031. {
  1032. // a 32-bit count, then a pointer to the source rows
  1033. size32_t childCount = *(size32_t *) source;
  1034. source += sizeof(size32_t);
  1035. const byte ** sourceRows = *(const byte***) source;
  1036. for (size32_t childRow = 0; childRow < childCount; childRow++)
  1037. {
  1038. offset = match.subTrans->doTranslate(builder, offset, sourceRows[childRow]);
  1039. }
  1040. }
  1041. else
  1042. {
  1043. // a 32-bit size, then rows inline
  1044. size32_t childSize = *(size32_t *) source;
  1045. source += sizeof(size32_t);
  1046. const byte *initialSource = source;
  1047. while ((size_t)(source - initialSource) < childSize)
  1048. {
  1049. offset = match.subTrans->doTranslate(builder, offset, source);
  1050. source += sourceType->queryChildType()->size(source, nullptr); // MORE - shame to repeat a calculation that the translate above almost certainly just did
  1051. }
  1052. }
  1053. dest = builder.getSelf() + countOffset; // Note - may have been moved by reallocs since last calculated
  1054. *(size32_t *)dest = offset - initialOffset;
  1055. }
  1056. break;
  1057. default:
  1058. throwUnexpected();
  1059. }
  1060. }
  1061. }
  1062. }
  1063. if (estimate && offset-origOffset != estimate)
  1064. {
  1065. // Note - ifblocks make this assertion invalid. We do not account for potentially omitted fields
  1066. // when estimating target record size.
  1067. if (!destRecInfo.getNumIfBlocks())
  1068. assert(offset-origOffset > estimate); // Estimate is always supposed to be conservative
  1069. #ifdef TRACE_TRANSLATION
  1070. DBGLOG("Wrote %u bytes to record (estimate was %u)\n", offset-origOffset, estimate);
  1071. #endif
  1072. }
  1073. return offset;
  1074. }
  1075. inline FieldMatchType match() const
  1076. {
  1077. return matchFlags;
  1078. }
  1079. const RtlRecord &destRecInfo;
  1080. const RtlRecord &sourceRecInfo;
  1081. unsigned fixedDelta = 0; // total size of all fixed-size source fields that are not matched
  1082. UnsignedArray unmatched; // List of all variable-size source fields that are unmatched
  1083. FieldMatchType matchFlags = match_perfect;
  1084. struct MatchInfo
  1085. {
  1086. unsigned matchIdx = 0;
  1087. FieldMatchType matchType = match_fail;
  1088. char fillChar = 0;
  1089. GeneralRecordTranslator *subTrans = nullptr;
  1090. ~MatchInfo()
  1091. {
  1092. delete subTrans;
  1093. }
  1094. } *matchInfo;
  1095. static size32_t translateScalar(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, const RtlTypeInfo *destType, const RtlTypeInfo *sourceType, const byte *source)
  1096. {
  1097. // This code COULD move into rtlfield.cpp?
  1098. switch(destType->getType())
  1099. {
  1100. case type_boolean:
  1101. case type_int:
  1102. case type_swapint:
  1103. case type_packedint:
  1104. case type_filepos:
  1105. offset = destType->buildInt(builder, offset, field, sourceType->getInt(source));
  1106. break;
  1107. case type_real:
  1108. offset = destType->buildReal(builder, offset, field, sourceType->getReal(source));
  1109. break;
  1110. case type_decimal: // Go via string - not common enough to special-case
  1111. case type_data:
  1112. case type_string:
  1113. case type_varstring:
  1114. case type_qstring:
  1115. {
  1116. size32_t size;
  1117. rtlDataAttr text;
  1118. sourceType->getString(size, text.refstr(), source);
  1119. offset = destType->buildString(builder, offset, field, size, text.getstr());
  1120. break;
  1121. }
  1122. case type_unicode:
  1123. case type_varunicode:
  1124. case type_utf8:
  1125. {
  1126. size32_t utf8chars;
  1127. rtlDataAttr utf8Text;
  1128. sourceType->getUtf8(utf8chars, utf8Text.refstr(), source);
  1129. offset = destType->buildUtf8(builder, offset, field, utf8chars, utf8Text.getstr());
  1130. break;
  1131. }
  1132. case type_set:
  1133. {
  1134. bool isAll = *(bool *) source;
  1135. source+= sizeof(bool);
  1136. byte *dest = builder.ensureCapacity(offset+sizeof(bool)+sizeof(size32_t), field->name)+offset;
  1137. *(size32_t *) (dest + sizeof(bool)) = 0; // Patch later when size known
  1138. offset += sizeof(bool) + sizeof(size32_t);
  1139. if (isAll)
  1140. {
  1141. *(bool*) dest = true;
  1142. }
  1143. else
  1144. {
  1145. *(bool*) dest = false;
  1146. size32_t sizeOffset = offset - sizeof(size32_t); // Where we need to patch
  1147. size32_t childSize = *(size32_t *)source;
  1148. source += sizeof(size32_t);
  1149. const byte *initialSource = source;
  1150. size32_t initialOffset = offset;
  1151. const RtlTypeInfo *destChildType = destType->queryChildType();
  1152. const RtlTypeInfo *sourceChildType = sourceType->queryChildType();
  1153. while ((size_t)(source - initialSource) < childSize)
  1154. {
  1155. offset = translateScalar(builder, offset, field, destChildType, sourceChildType, source);
  1156. source += sourceChildType->size(source, nullptr); // MORE - shame to repeat a calculation that the translate above almost certainly just did
  1157. }
  1158. dest = builder.getSelf() + sizeOffset; // Note - man have been moved by reallocs since last calculated
  1159. *(size32_t *)dest = offset - initialOffset;
  1160. }
  1161. break;
  1162. }
  1163. default:
  1164. throwUnexpected();
  1165. }
  1166. return offset;
  1167. }
  1168. size32_t estimateNewSize(const RtlRow &sourceRow) const
  1169. {
  1170. //DBGLOG("Source record size is %d", (int) sourceRow.getRecordSize());
  1171. size32_t expectedSize = sourceRow.getRecordSize() - fixedDelta;
  1172. //DBGLOG("Source record size without omitted fixed size fields is %d", expectedSize);
  1173. ForEachItemIn(i, unmatched)
  1174. {
  1175. unsigned fieldNo = unmatched.item(i);
  1176. expectedSize -= sourceRow.getSize(fieldNo);
  1177. //DBGLOG("Reducing estimated size by %d to %d for omitted field %d (%s)", (int) sourceRow.getSize(fieldNo), expectedSize, fieldNo, sourceRecInfo.queryName(fieldNo));
  1178. }
  1179. if (matchFlags & ~(match_perfect|match_link|match_none|match_extend|match_truncate))
  1180. {
  1181. for (unsigned idx = 0; idx < destRecInfo.getNumFields(); idx++)
  1182. {
  1183. const MatchInfo &match = matchInfo[idx];
  1184. const RtlTypeInfo *type = destRecInfo.queryType(idx);
  1185. unsigned matchField = match.matchIdx;
  1186. switch (match.matchType)
  1187. {
  1188. case match_perfect:
  1189. case match_link:
  1190. case match_none:
  1191. case match_extend:
  1192. case match_truncate:
  1193. // These ones were already included in fixedDelta
  1194. break;
  1195. default:
  1196. // This errs on the side of small - i.e. it assumes that all typecasts end up at minimum size
  1197. // We could do better in some cases e.g. variable string <-> variable unicode we can assume factor of 2,
  1198. // uft8 <-> string we could calculate here - but unlikely to be worth the effort.
  1199. // But it's fine for fixed size output fields, including truncate/extend
  1200. // We could also precalculate the expected delta if all omitted fields are fixed size - but not sure how likely/worthwhile that is.
  1201. expectedSize += type->getMinSize() - sourceRow.getSize(matchField);
  1202. //DBGLOG("Adjusting estimated size by (%d - %d) to %d for translated field %d (%s)", (int) sourceRow.getSize(matchField), type->getMinSize(), expectedSize, matchField, sourceRecInfo.queryName(matchField));
  1203. break;
  1204. }
  1205. }
  1206. }
  1207. return expectedSize;
  1208. }
  1209. void createMatchInfo()
  1210. {
  1211. for (unsigned idx = 0; idx < destRecInfo.getNumFields(); idx++)
  1212. {
  1213. const RtlFieldInfo *field = destRecInfo.queryField(idx);
  1214. const RtlTypeInfo *type = field->type;
  1215. MatchInfo &info = matchInfo[idx];
  1216. info.matchIdx = sourceRecInfo.getFieldNum(destRecInfo.queryName(idx));
  1217. if (info.matchIdx == -1)
  1218. {
  1219. info.matchType = match_none;
  1220. size32_t defaultSize = field->initializer ? type->size(field->initializer, nullptr) : type->getMinSize();
  1221. fixedDelta -= defaultSize;
  1222. //DBGLOG("Decreasing fixedDelta size by %d to %d for defaulted field %d (%s)", defaultSize, fixedDelta, idx, destRecInfo.queryName(idx));
  1223. }
  1224. else
  1225. {
  1226. const RtlTypeInfo *sourceType = sourceRecInfo.queryType(info.matchIdx);
  1227. if (!type->isScalar() || !sourceType->isScalar())
  1228. {
  1229. if (type->getType() != sourceType->getType())
  1230. info.matchType = match_fail; // No translation from one non-scalar type to another
  1231. else
  1232. {
  1233. switch (type->getType())
  1234. {
  1235. case type_set:
  1236. if (type->queryChildType()->fieldType==sourceType->queryChildType()->fieldType &&
  1237. type->queryChildType()->length==sourceType->queryChildType()->length)
  1238. info.matchType = match_perfect;
  1239. else
  1240. info.matchType = match_typecast;
  1241. break;
  1242. case type_row: // These are not expected I think...
  1243. throwUnexpected();
  1244. case type_ifblock:
  1245. case type_record:
  1246. case type_table:
  1247. {
  1248. const RtlRecord *subDest = destRecInfo.queryNested(idx);
  1249. const RtlRecord *subSrc = sourceRecInfo.queryNested(info.matchIdx);
  1250. info.subTrans = new GeneralRecordTranslator(*subDest, *subSrc);
  1251. if (!info.subTrans->needsTranslate())
  1252. {
  1253. // Child does not require translation, but check linkcount mode matches too!
  1254. if (type->isLinkCounted())
  1255. if (sourceType->isLinkCounted())
  1256. info.matchType = match_link;
  1257. else
  1258. info.matchType = match_recurse;
  1259. else
  1260. if (sourceType->isLinkCounted())
  1261. info.matchType = match_recurse;
  1262. else
  1263. info.matchType = match_perfect;
  1264. if (info.matchType != match_recurse)
  1265. {
  1266. delete info.subTrans;
  1267. info.subTrans = nullptr;
  1268. }
  1269. }
  1270. else if (info.subTrans->canTranslate())
  1271. {
  1272. info.matchType = match_recurse;
  1273. matchFlags |= info.subTrans->matchFlags;
  1274. }
  1275. else
  1276. info.matchType = match_fail;
  1277. break;
  1278. }
  1279. default:
  1280. info.matchType = match_fail;
  1281. break;
  1282. }
  1283. }
  1284. }
  1285. else if (type->fieldType==sourceType->fieldType)
  1286. {
  1287. if (type->length==sourceType->length)
  1288. {
  1289. info.matchType = match_perfect;
  1290. }
  1291. else
  1292. {
  1293. assert(type->isFixedSize()); // Both variable size would have matched length above
  1294. info.matchType = match_typecast;
  1295. if (type->length < sourceType->length)
  1296. {
  1297. if (type->canTruncate())
  1298. {
  1299. info.matchType = match_truncate;
  1300. fixedDelta += sourceType->getMinSize()-type->getMinSize();
  1301. //DBGLOG("Increasing fixedDelta size by %d to %d for truncated field %d (%s)", sourceType->getMinSize()-type->getMinSize(), fixedDelta, idx, destRecInfo.queryName(idx));
  1302. }
  1303. }
  1304. else
  1305. {
  1306. if (type->canExtend(info.fillChar))
  1307. {
  1308. info.matchType = match_extend;
  1309. fixedDelta += sourceType->getMinSize()-type->getMinSize();
  1310. //DBGLOG("Decreasing fixedDelta size by %d to %d for truncated field %d (%s)", type->getMinSize()-sourceType->getMinSize(), fixedDelta, idx, destRecInfo.queryName(idx));
  1311. }
  1312. }
  1313. }
  1314. }
  1315. else
  1316. info.matchType = match_typecast;
  1317. if (sourceRecInfo.queryField(info.matchIdx)->flags & RFTMinifblock)
  1318. info.matchType |= match_inifblock; // Avoids incorrect commoning up of adjacent matches
  1319. // MORE - could note the highest interesting fieldnumber in the source and not bother filling in offsets after that
  1320. // Not sure it would help much though - usually need to know the total record size anyway in real life
  1321. if (idx != info.matchIdx)
  1322. matchFlags |= match_move;
  1323. }
  1324. matchFlags |= info.matchType;
  1325. }
  1326. if (sourceRecInfo.getNumFields() > destRecInfo.getNumFields())
  1327. matchFlags |= match_remove;
  1328. if (matchFlags && !destRecInfo.getFixedSize())
  1329. {
  1330. for (unsigned idx = 0; idx < sourceRecInfo.getNumFields(); idx++)
  1331. {
  1332. const RtlFieldInfo *field = sourceRecInfo.queryField(idx);
  1333. const RtlTypeInfo *type = field->type;
  1334. if (destRecInfo.getFieldNum(field->name) == (unsigned) -1)
  1335. {
  1336. // unmatched field
  1337. if (type->isFixedSize())
  1338. {
  1339. //DBGLOG("Reducing estimated size by %d for (fixed size) omitted field %s", (int) type->getMinSize(), field->name);
  1340. fixedDelta += type->getMinSize();
  1341. }
  1342. else
  1343. unmatched.append(idx);
  1344. }
  1345. }
  1346. //DBGLOG("Source record contains %d bytes of omitted fixed size fields", fixedDelta);
  1347. }
  1348. }
  1349. };
  1350. extern ECLRTL_API const IDynamicTransform *createRecordTranslator(const RtlRecord &_destRecInfo, const RtlRecord &_srcRecInfo)
  1351. {
  1352. return new GeneralRecordTranslator(_destRecInfo, _srcRecInfo);
  1353. }
  1354. class TranslatedRowStream : public CInterfaceOf<IRowStream>
  1355. {
  1356. public:
  1357. TranslatedRowStream(IRowStream *_inputStream, IEngineRowAllocator *_resultAllocator, const RtlRecord &outputRecord, const RtlRecord &inputRecord)
  1358. : inputStream(_inputStream), resultAllocator(_resultAllocator), translator(new GeneralRecordTranslator(outputRecord, inputRecord))
  1359. {
  1360. translator->describe();
  1361. }
  1362. virtual const void *nextRow()
  1363. {
  1364. if (eof)
  1365. return NULL;
  1366. const void *inRow = inputStream->nextRow();
  1367. if (!inRow)
  1368. {
  1369. if (eogSeen)
  1370. eof = true;
  1371. else
  1372. eogSeen = true;
  1373. return nullptr;
  1374. }
  1375. else
  1376. eogSeen = false;
  1377. RtlDynamicRowBuilder rowBuilder(resultAllocator);
  1378. size32_t len = translator->translate(rowBuilder, (const byte *) inRow);
  1379. rtlReleaseRow(inRow);
  1380. return rowBuilder.finalizeRowClear(len);
  1381. }
  1382. virtual void stop() override
  1383. {
  1384. resultAllocator.clear();
  1385. }
  1386. bool canTranslate() const
  1387. {
  1388. return translator->canTranslate();
  1389. }
  1390. bool needsTranslate() const
  1391. {
  1392. return translator->needsTranslate();
  1393. }
  1394. protected:
  1395. Linked<IRowStream> inputStream;
  1396. Linked<IEngineRowAllocator> resultAllocator;
  1397. Owned<const IDynamicTransform> translator;
  1398. unsigned numOffsets = 0;
  1399. size_t * variableOffsets = nullptr;
  1400. bool eof = false;
  1401. bool eogSeen = false;
  1402. };
  1403. extern ECLRTL_API IRowStream * transformRecord(IEngineRowAllocator * resultAllocator,IOutputMetaData & metaInput,IRowStream * input)
  1404. {
  1405. if (resultAllocator->queryOutputMeta()==&metaInput)
  1406. return LINK(input);
  1407. Owned<TranslatedRowStream> stream = new TranslatedRowStream(input, resultAllocator,
  1408. resultAllocator->queryOutputMeta()->queryRecordAccessor(true),
  1409. metaInput.queryRecordAccessor(true));
  1410. if (!stream->needsTranslate())
  1411. return LINK(input);
  1412. else if (!stream->canTranslate())
  1413. rtlFail(0, "Cannot translate record stream");
  1414. else
  1415. return stream.getClear();
  1416. }