rtldynfield.cpp 85 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090
  1. /*##############################################################################
  2. HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. ############################################################################## */
  13. #include "platform.h"
  14. #include <math.h>
  15. #include <stdio.h>
  16. #include "jmisc.hpp"
  17. #include "jlib.hpp"
  18. #include "eclhelper.hpp"
  19. #include "eclrtl_imp.hpp"
  20. #include "rtldynfield.hpp"
  21. #include "rtlrecord.hpp"
  22. #include "rtlembed.hpp"
  23. #include "rtlnewkey.hpp"
  24. //#define TRACE_TRANSLATION
  25. #define VALIDATE_TYPEINFO_HASHES
  26. #define RTLTYPEINFO_FORMAT_1 81 // In case we ever want to support more than one format or change how it is stored
  27. //---------------------------------------------------------------------------------------------------------------------
  28. extern ECLRTL_API RecordTranslationMode getTranslationMode(const char *val)
  29. {
  30. if (isEmptyString(val) || strToBool(val) || strieq(val, "payload"))
  31. return RecordTranslationMode::Payload;
  32. else if (strieq(val, "alwaysDisk") || strieq(val, "disk"))
  33. return RecordTranslationMode::AlwaysDisk;
  34. else if (strieq(val, "alwaysECL") || strieq(val, "ecl"))
  35. return RecordTranslationMode::AlwaysECL;
  36. else
  37. return RecordTranslationMode::None;
  38. }
  39. extern ECLRTL_API const char *getTranslationModeText(RecordTranslationMode val)
  40. {
  41. switch (val)
  42. {
  43. case RecordTranslationMode::AlwaysDisk: return "alwaysDisk";
  44. case RecordTranslationMode::AlwaysECL: return "alwaysECL";
  45. case RecordTranslationMode::Payload: return "payload";
  46. case RecordTranslationMode::None: return "off";
  47. }
  48. throwUnexpected();
  49. }
  50. //---------------------------------------------------------------------------------------------------------------------
  51. const RtlTypeInfo *FieldTypeInfoStruct::createRtlTypeInfo() const
  52. {
  53. const RtlTypeInfo *ret = nullptr;
  54. switch (fieldType & RFTMkind)
  55. {
  56. case type_boolean:
  57. ret = new RtlBoolTypeInfo(fieldType, length);
  58. break;
  59. case type_keyedint:
  60. ret = new RtlKeyedIntTypeInfo(fieldType, length, childType);
  61. break;
  62. case type_int:
  63. ret = new RtlIntTypeInfo(fieldType, length);
  64. break;
  65. case type_blob:
  66. ret = new RtlBlobTypeInfo(fieldType, length, childType);
  67. break;
  68. case type_filepos:
  69. #if __BYTE_ORDER == __LITTLE_ENDIAN
  70. ret = new RtlSwapIntTypeInfo(fieldType, length);
  71. #else
  72. ret = new RtlIntTypeInfo(fieldType, length);
  73. #endif
  74. break;
  75. case type_real:
  76. ret = new RtlRealTypeInfo(fieldType, length);
  77. break;
  78. case type_decimal:
  79. ret = new RtlDecimalTypeInfo(fieldType, length);
  80. break;
  81. case type_string:
  82. ret = new RtlStringTypeInfo(fieldType, length);
  83. break;
  84. case type_bitfield:
  85. ret = new RtlBitfieldTypeInfo(fieldType, length);
  86. break;
  87. case type_varstring:
  88. ret = new RtlVarStringTypeInfo(fieldType, length);
  89. break;
  90. case type_data:
  91. ret = new RtlDataTypeInfo(fieldType, length);
  92. break;
  93. case type_table:
  94. assert(childType);
  95. ret = new RtlDatasetTypeInfo(fieldType, length, childType);
  96. break;
  97. case type_dictionary:
  98. assert(childType);
  99. ret = new RtlDictionaryTypeInfo(fieldType, length, childType);
  100. break;
  101. case type_set:
  102. assert(childType);
  103. ret = new RtlSetTypeInfo(fieldType, length, childType);
  104. break;
  105. case type_row:
  106. assert(childType);
  107. ret = new RtlRowTypeInfo(fieldType, length, childType);
  108. break;
  109. case type_swapint:
  110. ret = new RtlSwapIntTypeInfo(fieldType, length);
  111. break;
  112. case type_packedint:
  113. ret = new RtlPackedIntTypeInfo(fieldType, length);
  114. break;
  115. case type_qstring:
  116. ret = new RtlQStringTypeInfo(fieldType, length);
  117. break;
  118. case type_unicode:
  119. ret = new RtlUnicodeTypeInfo(fieldType, length, locale);
  120. break;
  121. case type_varunicode:
  122. ret = new RtlVarUnicodeTypeInfo(fieldType, length, locale);
  123. break;
  124. case type_utf8:
  125. ret = new RtlUtf8TypeInfo(fieldType, length, locale);
  126. break;
  127. case type_record:
  128. ret = new RtlRecordTypeInfo(fieldType, length, fieldsArray);
  129. break;
  130. case type_ifblock:
  131. ret = new RtlDynamicIfBlockTypeInfo(fieldType, length, fieldsArray, nullptr, filter);
  132. break;
  133. case type_alien:
  134. assert(childType);
  135. ret = new RtlAlienTypeInfo(fieldType, length, childType);
  136. break;
  137. default:
  138. throwUnexpected();
  139. }
  140. return ret;
  141. };
  142. typedef MapBetween<const RtlTypeInfo *, const RtlTypeInfo *, StringAttr, const char *> TypeNameMap;
  143. typedef MapBetween<const RtlTypeInfo *, const RtlTypeInfo *, unsigned, unsigned> TypeNumMap;
  144. /**
  145. * class CRtlFieldTypeSerializer
  146. *
  147. * Serializer class for creating json representation of a RtlTypeInfo structure.
  148. *
  149. */
  150. class CRtlFieldTypeSerializer
  151. {
  152. public:
  153. /**
  154. * Serialize a RtlTypeInfo structure to JSON
  155. *
  156. * @param out Buffer for resulting serialized string
  157. * @param type RtlTypeInfo structure to be serialized
  158. * @return Referenced to supplied buffer
  159. */
  160. static StringBuffer &serialize(StringBuffer &out, const RtlTypeInfo *type)
  161. {
  162. CRtlFieldTypeSerializer s(out, type);
  163. s.doSerialize();
  164. return out;
  165. }
  166. private:
  167. CRtlFieldTypeSerializer(StringBuffer &_out, const RtlTypeInfo *_base)
  168. : json(_out), base(_base)
  169. {
  170. }
  171. void doSerialize()
  172. {
  173. json.append("{");
  174. serializeType(base);
  175. json.append("\n}");
  176. }
  177. void serializeType(const RtlTypeInfo *type)
  178. {
  179. if (!serialized(type))
  180. {
  181. // Make sure all child types are serialized first
  182. const RtlTypeInfo *childType = type->queryChildType();
  183. if (childType)
  184. serializeType(childType);
  185. const RtlFieldInfo * const * fields = type->queryFields();
  186. if (fields)
  187. {
  188. for (;;)
  189. {
  190. const RtlFieldInfo * child = *fields;
  191. if (!child)
  192. break;
  193. serializeType(child->type);
  194. fields++;
  195. }
  196. }
  197. // Now serialize this one
  198. if (type != base)
  199. {
  200. VStringBuffer newName("ty%d", ++nextTypeName);
  201. types.setValue(type, newName.str());
  202. startField(newName.str());
  203. serializeMe(type);
  204. closeCurly();
  205. }
  206. else
  207. serializeMe(type);
  208. }
  209. }
  210. void serializeMe(const RtlTypeInfo *type)
  211. {
  212. if (!type->canSerialize())
  213. throw makeStringException(MSGAUD_user, 1000, "This type structure cannot be serialized");
  214. addPropHex("fieldType", type->fieldType);
  215. addProp("length", type->length);
  216. addPropNonEmpty("locale", type->queryLocale());
  217. const RtlTypeInfo *childType = type->queryChildType();
  218. if (childType)
  219. addPropType("child", childType);
  220. const IFieldFilter * filter = type->queryFilter();
  221. if (filter)
  222. {
  223. StringBuffer filterText;
  224. filter->serialize(filterText);
  225. addPropType("filterType", &filter->queryType());
  226. addProp("filter", filterText);
  227. }
  228. const RtlFieldInfo * const * fields = type->queryFields();
  229. if (fields)
  230. {
  231. startFields();
  232. for (;;)
  233. {
  234. const RtlFieldInfo * child = *fields;
  235. if (!child)
  236. break;
  237. newline();
  238. openCurly();
  239. addProp("name", child->name);
  240. addPropType("type", child->type);
  241. addProp("xpath", child->xpath);
  242. if (child->flags)
  243. addPropHex("flags", child->flags);
  244. // initializer is tricky - it's not (in general) a null-terminated string but the actual length is not easily available
  245. if (child->initializer)
  246. {
  247. if (isVirtualInitializer(child->initializer))
  248. addProp("vinit", getVirtualInitializer(child->initializer));
  249. else
  250. addProp("init", child->type->size((const byte *) child->initializer, nullptr), (const byte *) child->initializer);
  251. }
  252. closeCurly();
  253. fields++;
  254. }
  255. endFields();
  256. }
  257. }
  258. bool serialized(const RtlTypeInfo *type)
  259. {
  260. return types.find(type) != nullptr;
  261. }
  262. void startField(const char *name)
  263. {
  264. newline().appendf("\"%s\": ", name);
  265. openCurly();
  266. }
  267. void addProp(const char *propName, const char *propVal)
  268. {
  269. if (propVal)
  270. {
  271. newline();
  272. encodeJSON(json.append("\""), propName).append("\": ");
  273. encodeJSON(json.append("\""), propVal).append("\"");
  274. }
  275. }
  276. void addProp(const char *propName, size32_t propLen, const byte *propVal)
  277. {
  278. if (propVal)
  279. {
  280. newline();
  281. encodeJSON(json.append("\""), propName).append("\": \"");
  282. JBASE64_Encode(propVal, propLen, json, false);
  283. json.append("\"");
  284. }
  285. }
  286. void addPropNonEmpty(const char *propName, const char *propVal)
  287. {
  288. if (propVal && *propVal)
  289. addProp(propName, propVal);
  290. }
  291. void addProp(const char *propName, unsigned propVal)
  292. {
  293. newline().appendf("\"%s\": %u", propName, propVal);
  294. }
  295. void addPropHex(const char *propName, unsigned propVal)
  296. {
  297. newline().appendf("\"%s\": %u", propName, propVal); // Nice idea but json does not support hex constants :(
  298. }
  299. void addPropType(const char *propName, const RtlTypeInfo *type)
  300. {
  301. addProp(propName, queryTypeName(type));
  302. }
  303. const char *queryTypeName(const RtlTypeInfo *type)
  304. {
  305. StringAttr *typeName = types.getValue(type);
  306. assertex(typeName);
  307. return typeName->get();
  308. }
  309. void startFields()
  310. {
  311. newline().appendf("\"fields\": ");
  312. openCurly('[');
  313. }
  314. void endFields()
  315. {
  316. closeCurly(']');
  317. }
  318. StringBuffer &newline()
  319. {
  320. if (commaPending)
  321. json.append(',');
  322. json.appendf("\n%*s", indent, "");
  323. commaPending = true;
  324. return json;
  325. }
  326. void closeCurly(char brace = '}')
  327. {
  328. indent--;
  329. json.appendf("\n%*s%c", indent, "", brace);
  330. commaPending = true;
  331. }
  332. void openCurly(char brace = '{')
  333. {
  334. json.append(brace);
  335. indent++;
  336. commaPending = false;
  337. }
  338. TypeNameMap types;
  339. StringBuffer &json;
  340. const RtlTypeInfo *base = nullptr;
  341. unsigned indent = 1;
  342. unsigned nextTypeName = 0;
  343. bool commaPending = false;
  344. };
  345. class CRtlFieldTypeBinSerializer
  346. {
  347. public:
  348. /**
  349. * Serialize a RtlTypeInfo structure to binary
  350. *
  351. * @param out Buffer for resulting serialized string
  352. * @param type RtlTypeInfo structure to be serialized
  353. * @return Referenced to supplied buffer
  354. */
  355. static MemoryBuffer &serialize(MemoryBuffer &out, const RtlTypeInfo *type)
  356. {
  357. int oldEnd = out.setEndian(__LITTLE_ENDIAN);
  358. CRtlFieldTypeBinSerializer s(out);
  359. byte format = RTLTYPEINFO_FORMAT_1;
  360. out.append(format);
  361. DelayedMarker<hash64_t> hash(out);
  362. DelayedSizeMarker size(out);
  363. size32_t pos = out.length();
  364. s.serializeType(type);
  365. size.write();
  366. hash.write(rtlHash64Data(size.size(), out.toByteArray()+pos, 0));
  367. out.setEndian(oldEnd);
  368. return out;
  369. }
  370. private:
  371. CRtlFieldTypeBinSerializer(MemoryBuffer &_out)
  372. : out(_out)
  373. {
  374. }
  375. void serializeType(const RtlTypeInfo *type)
  376. {
  377. if (!serialized(type))
  378. {
  379. // Make sure all child types are serialized first
  380. const RtlTypeInfo *child = type->queryChildType();
  381. if (child)
  382. serializeType(child);
  383. const RtlFieldInfo * const * fields = type->queryFields();
  384. if (fields)
  385. {
  386. for (unsigned idx = 0;;idx++)
  387. {
  388. const RtlFieldInfo * child = fields[idx];
  389. if (!child)
  390. break;
  391. serializeType(child->type);
  392. }
  393. }
  394. // Now serialize this one
  395. types.setValue(type, nextTypeNum++);
  396. serializeMe(type);
  397. }
  398. }
  399. void serializeMe(const RtlTypeInfo *type)
  400. {
  401. if (!type->canSerialize())
  402. throw makeStringException(MSGAUD_user, 1000, "This type structure cannot be serialized");
  403. unsigned fieldType = type->fieldType;
  404. const char *locale = type->queryLocale();
  405. if (locale && *locale)
  406. fieldType |= RFTMhasLocale;
  407. const RtlTypeInfo *child = type->queryChildType();
  408. if (child)
  409. fieldType |= RFTMhasChildType;
  410. const RtlFieldInfo * const * fields = type->queryFields();
  411. if (fields)
  412. fieldType |= RFTMhasFields;
  413. out.append(fieldType);
  414. out.appendPacked(type->length);
  415. if (fieldType & RFTMhasLocale)
  416. out.append(locale);
  417. if (child)
  418. out.appendPacked(queryTypeIdx(child));
  419. const IFieldFilter * filter = type->queryFilter();
  420. if (filter)
  421. {
  422. out.appendPacked(queryTypeIdx(&filter->queryType()));
  423. filter->serialize(out);
  424. }
  425. if (fields)
  426. {
  427. unsigned count = countFields(fields);
  428. out.appendPacked(count);
  429. for (;;)
  430. {
  431. const RtlFieldInfo * child = *fields;
  432. if (!child)
  433. break;
  434. out.append(child->name);
  435. out.appendPacked(queryTypeIdx(child->type));
  436. unsigned flags = child->flags;
  437. if (child->xpath)
  438. flags |= RFTMhasXpath;
  439. if (child->initializer)
  440. {
  441. if (isVirtualInitializer(child->initializer))
  442. flags |= RFTMhasVirtualInitializer;
  443. else
  444. flags |= RFTMhasInitializer;
  445. }
  446. out.append(flags);
  447. if (child->xpath)
  448. out.append(child->xpath);
  449. // initializer is tricky - it's not (in general) a null-terminated string but the actual length is not easily available
  450. if (flags & RFTMhasInitializer)
  451. {
  452. unsigned initLength = child->type->size((const byte *) child->initializer, nullptr);
  453. out.appendPacked(initLength).append(initLength, (const byte *) child->initializer);
  454. }
  455. else if (flags &RFTMhasVirtualInitializer)
  456. out.append(getVirtualInitializer(child->initializer));
  457. fields++;
  458. }
  459. }
  460. }
  461. bool serialized(const RtlTypeInfo *type)
  462. {
  463. return types.find(type) != nullptr;
  464. }
  465. unsigned queryTypeIdx(const RtlTypeInfo *type)
  466. {
  467. unsigned *typeNum = types.getValue(type);
  468. assertex(typeNum);
  469. return *typeNum;
  470. }
  471. TypeNumMap types;
  472. MemoryBuffer &out;
  473. unsigned nextTypeNum = 0;
  474. };
  475. /**
  476. * class CRtlFieldTypeDeserializer
  477. *
  478. * Deserializer class for creating a RtlTypeInfo structure from json representation.
  479. *
  480. * Note that the resulting RtlTypeInfo structures are owned by this object and will be
  481. * destroyed when this object is destroyed.
  482. *
  483. */
  484. class CRtlFieldTypeDeserializer : public CInterfaceOf<IRtlFieldTypeDeserializer>
  485. {
  486. public:
  487. /**
  488. * CRtlFieldTypeDeserializer constructor
  489. *
  490. * @param _callback Supplies a callback to be used for blobs/filepositions.
  491. */
  492. CRtlFieldTypeDeserializer()
  493. {
  494. }
  495. /**
  496. * CRtlFieldTypeDeserializer destructor
  497. * <p>
  498. * Releases all RtlTypeInfo and related structures created by this deserializer
  499. */
  500. ~CRtlFieldTypeDeserializer()
  501. {
  502. // Need some care - all the RtlTypeInfo objects I created need to be destroyed, together with anything else I had to create
  503. // Strings (other than the init strings) are preserved in the AtomTable
  504. // First allow the types to clean up any critical cached information, then delete them in a second pass
  505. HashIterator allTypes(types);
  506. ForEach(allTypes)
  507. {
  508. const RtlTypeInfo **type = types.mapToValue(&allTypes.query());
  509. cleanupType(*type);
  510. }
  511. cleanupType(base);
  512. ForEach(allTypes)
  513. {
  514. const RtlTypeInfo **type = types.mapToValue(&allTypes.query());
  515. deleteType(*type);
  516. }
  517. deleteType(base);
  518. }
  519. /**
  520. * Obtain the deserialized type information
  521. * <p>
  522. * Note that the RtlTypeInfo objects are not link-counted, so the lifetime of these objects
  523. * is determined by the lifetime of the deserializer. They will be released once the deserializer
  524. * that created them is deleted.
  525. * <p>
  526. * Do not call more than once.
  527. *
  528. * @param _json JSON text to be deserialized, as created by CRtlFieldTypeSerializer
  529. * @return Deserialized type object
  530. */
  531. virtual const RtlTypeInfo *deserialize(const char *json) override
  532. {
  533. assertex(!base);
  534. Owned<IPropertyTree> jsonTree = createPTreeFromJSONString(json);
  535. base = deserializeType(jsonTree, jsonTree);
  536. return base;
  537. }
  538. /**
  539. * Obtain the deserialized type information
  540. * <p>
  541. * Note that the RtlTypeInfo objects are not link-counted, so the lifetime of these objects
  542. * is determined by the lifetime of the deserializer. They will be released once the deserializer
  543. * that created them is deleted.
  544. * <p>
  545. * Do not call more than once.
  546. *
  547. * @param _jsonTree JSON property tree to be deserialized, as created by CRtlFieldTypeSerializer
  548. * @return Deserialized type object
  549. */
  550. virtual const RtlTypeInfo *deserialize(IPropertyTree &jsonTree) override
  551. {
  552. assertex(!base);
  553. base = deserializeType(&jsonTree, &jsonTree);
  554. return base;
  555. }
  556. /**
  557. * Obtain the deserialized type information
  558. * <p>
  559. * Note that the RtlTypeInfo objects are not link-counted, so the lifetime of these objects
  560. * is determined by the lifetime of the deserializer. They will be released once the deserializer
  561. * that created them is deleted.
  562. * <p>
  563. * Do not call more than once.
  564. *
  565. * @param buf Binary serialized typeinfo to be deserialized, as created by CRtlFieldTypeSerializer
  566. * @return Deserialized type object
  567. */
  568. virtual const RtlTypeInfo *deserialize(MemoryBuffer &buf) override
  569. {
  570. assertex(!base);
  571. unsigned nextTypeNum = 0;
  572. int oldEndian = buf.setEndian(__LITTLE_ENDIAN);
  573. try
  574. {
  575. byte format;
  576. buf.read(format);
  577. if (format != RTLTYPEINFO_FORMAT_1)
  578. throw MakeStringException(0, "Invalid type info (%d) in CRtlFieldTypeDeserializer::deserialize", format);
  579. hash64_t hash;
  580. buf.read(hash);
  581. size32_t size;
  582. buf.read(size);
  583. #ifdef VALIDATE_TYPEINFO_HASHES
  584. hash64_t expected = rtlHash64Data(size, buf.readDirect(0), 0);
  585. if (expected != hash)
  586. throw MakeStringException(0, "Invalid type info hash in CRtlFieldTypeDeserializer::deserialize");
  587. #endif
  588. size32_t endpos = buf.getPos() + size;
  589. while (buf.getPos() < endpos)
  590. {
  591. if (base)
  592. {
  593. addType(base, nextTypeNum++);
  594. base = nullptr; // in case of exceptions...
  595. }
  596. base = deserializeType(buf);
  597. }
  598. if (buf.getPos()!=endpos)
  599. throw MakeStringException(0, "Invalid type info (incorrect size data) in CRtlFieldTypeDeserializer::deserialize");
  600. buf.setEndian(oldEndian);
  601. return base;
  602. }
  603. catch(...)
  604. {
  605. buf.setEndian(oldEndian);
  606. throw;
  607. }
  608. }
  609. virtual const RtlTypeInfo *addType(FieldTypeInfoStruct &info, const IInterface *typeOrIfblock) override
  610. {
  611. VStringBuffer name("%p", typeOrIfblock);
  612. const RtlTypeInfo ** found = types.getValue(name);
  613. if (found)
  614. return *found;
  615. savedTypes.append(LINK(typeOrIfblock));
  616. info.locale = keep(info.locale);
  617. const RtlTypeInfo * ret = info.createRtlTypeInfo();
  618. types.setValue(name, ret);
  619. unsigned baseType = (info.fieldType & RFTMkind);
  620. if (baseType == type_record)
  621. patchIfBlockParentRow(ret, static_cast<const RtlRecordTypeInfo *>(ret));
  622. return ret;
  623. }
  624. virtual const RtlTypeInfo *lookupType(const IInterface * typeOrIfBlock) const override
  625. {
  626. VStringBuffer name("%p", typeOrIfBlock);
  627. const RtlTypeInfo ** found = types.getValue(name);
  628. if (found)
  629. return *found;
  630. return nullptr;
  631. }
  632. virtual const RtlFieldInfo *addFieldInfo(const char *fieldName, const char *xpath, const RtlTypeInfo *type, unsigned flags, const char *init) override
  633. {
  634. // MORE - we could hang onto this for cleanup, rather than assuming that we keep it via a later addType() call?
  635. return new RtlFieldStrInfo(keep(fieldName), keep(xpath), type, flags, init);
  636. }
  637. private:
  638. KeptAtomTable atoms; // Used to ensure proper lifetime of strings used in type structures
  639. MapStringTo<const RtlTypeInfo *> types; // Ensures structures only generated once
  640. const RtlTypeInfo *base = nullptr; // Holds the resulting type
  641. IConstPointerArray savedTypes; // ensure types remain alive for subsequent lookups
  642. void deleteType(const RtlTypeInfo *type)
  643. {
  644. if (type)
  645. {
  646. // Releases all memory for a single RtlTypeInfo object
  647. const RtlFieldInfo * const * fields = type->queryFields();
  648. if (fields)
  649. {
  650. const RtlFieldInfo * const * cur = fields;
  651. for (;;)
  652. {
  653. const RtlFieldInfo * child = *cur;
  654. if (!child)
  655. break;
  656. // We don't need to delete other strings - they are owned by atom table.
  657. // But the initializer is decoded and thus owned by me
  658. if (!isVirtualInitializer(child->initializer))
  659. free((void *)child->initializer);
  660. delete child;
  661. cur++;
  662. }
  663. delete [] fields;
  664. }
  665. type->doDelete();
  666. }
  667. }
  668. void cleanupType(const RtlTypeInfo *type)
  669. {
  670. if (type)
  671. type->doCleanup();
  672. }
  673. const RtlTypeInfo *lookupType(const char *name, IPropertyTree *all)
  674. {
  675. const RtlTypeInfo ** found = types.getValue(name);
  676. if (found)
  677. return *found;
  678. const RtlTypeInfo *type = deserializeType(all->queryPropTree(name), all);
  679. types.setValue(name, type);
  680. return type;
  681. }
  682. const RtlTypeInfo *lookupType(unsigned idx)
  683. {
  684. // Could keep an expanding array of types instead - but the hash table is already there for json support...
  685. VStringBuffer key("%u", idx);
  686. const RtlTypeInfo ** found = types.getValue(key);
  687. if (found)
  688. return *found;
  689. throw makeStringException(-1, "Invalid serialized type information");
  690. }
  691. void addType(const RtlTypeInfo *type, unsigned idx)
  692. {
  693. VStringBuffer key("%u", idx);
  694. assert(types.getValue(key)==nullptr);
  695. types.setValue(key, type);
  696. }
  697. const char *keep(const char *string)
  698. {
  699. if (string)
  700. return str(atoms.addAtom(string));
  701. else
  702. return nullptr;
  703. }
  704. const RtlTypeInfo *deserializeType(IPropertyTree *type, IPropertyTree *all)
  705. {
  706. FieldTypeInfoStruct info;
  707. info.fieldType = type->getPropInt("fieldType");
  708. info.length = type->getPropInt("length");
  709. info.locale = keep(type->queryProp("locale"));
  710. const char *child = type->queryProp("child");
  711. if (child)
  712. info.childType = lookupType(child, all);
  713. unsigned baseType = (info.fieldType & RFTMkind);
  714. if ((baseType == type_record) || (baseType == type_ifblock))
  715. {
  716. unsigned numFields = type->getCount("fields");
  717. info.fieldsArray = new const RtlFieldInfo * [numFields+1];
  718. info.fieldsArray[numFields] = nullptr;
  719. Owned<IPropertyTreeIterator> fields = type->getElements("fields");
  720. unsigned n = 0;
  721. ForEach(*fields)
  722. {
  723. IPropertyTree &field = fields->query();
  724. const char *fieldTypeName = field.queryProp("type");
  725. const char *fieldName = keep(field.queryProp("name"));
  726. const char *fieldXpath = keep(field.queryProp("xpath"));
  727. unsigned flags = field.getPropInt("flags");
  728. const char *fieldInit = field.queryProp("init");
  729. const char *fieldVInit = field.queryProp("vinit");
  730. if (fieldInit)
  731. {
  732. StringBuffer decoded;
  733. JBASE64_Decode(fieldInit, decoded);
  734. fieldInit = decoded.detach(); // NOTE - this gets freed in cleanupType()
  735. }
  736. else if (fieldVInit)
  737. {
  738. fieldInit = (const char *)(memsize_t)atoi(fieldVInit);
  739. }
  740. info.fieldsArray[n] = new RtlFieldStrInfo(fieldName, fieldXpath, lookupType(fieldTypeName, all), flags, fieldInit);
  741. n++;
  742. }
  743. }
  744. if (baseType == type_ifblock)
  745. {
  746. //Filter field needs to be deserialized and the type resolved separately outside the deserialize call
  747. //because there isn't a RtlTypeInfo available to resolve the field (since we are currently deserializing it!)
  748. const char * filterText = type->queryProp("filter");
  749. StringBuffer fieldIdText;
  750. readFieldFromFieldFilter(fieldIdText, filterText);
  751. unsigned fieldId = atoi(fieldIdText);
  752. const RtlTypeInfo * fieldType = lookupType(type->queryProp("filterType"), all);
  753. info.filter = deserializeFieldFilter(fieldId, *fieldType, filterText);
  754. }
  755. const RtlTypeInfo * result = info.createRtlTypeInfo();
  756. if (baseType == type_record)
  757. patchIfBlockParentRow(result, static_cast<const RtlRecordTypeInfo *>(result));
  758. return result;
  759. }
  760. const RtlTypeInfo *deserializeType(MemoryBuffer &type)
  761. {
  762. FieldTypeInfoStruct info;
  763. type.read(info.fieldType);
  764. type.readPacked(info.length);
  765. if (info.fieldType & RFTMhasLocale)
  766. {
  767. const char *locale;
  768. type.read(locale);
  769. info.locale = keep(locale);
  770. }
  771. if (info.fieldType & RFTMhasChildType)
  772. {
  773. unsigned childIdx;
  774. type.readPacked(childIdx);
  775. info.childType = lookupType(childIdx);
  776. }
  777. unsigned baseType = (info.fieldType & RFTMkind);
  778. if (baseType == type_ifblock)
  779. {
  780. unsigned childIdx;
  781. type.readPacked(childIdx);
  782. const RtlTypeInfo * fieldType = lookupType(childIdx);
  783. unsigned fieldId;
  784. type.readPacked(fieldId);
  785. info.filter = deserializeFieldFilter(fieldId, *fieldType, type);
  786. }
  787. if (info.fieldType & RFTMhasFields)
  788. {
  789. unsigned numFields;
  790. type.readPacked(numFields);
  791. info.fieldsArray = new const RtlFieldInfo * [numFields+1];
  792. info.fieldsArray[numFields] = nullptr;
  793. for (unsigned n = 0; n < numFields; n++)
  794. {
  795. const char *fieldName;
  796. type.read(fieldName);
  797. if (fieldName[0] == '\0')
  798. fieldName = nullptr;
  799. unsigned fieldType;
  800. type.readPacked(fieldType);
  801. unsigned fieldFlags;
  802. type.read(fieldFlags);
  803. const char *xpath = nullptr;
  804. if (fieldFlags & RFTMhasXpath)
  805. type.read(xpath);
  806. void *init = nullptr;
  807. if (fieldFlags & RFTMhasInitializer)
  808. {
  809. unsigned initLength;
  810. type.readPacked(initLength);
  811. init = malloc(initLength);
  812. memcpy(init, type.readDirect(initLength), initLength);
  813. }
  814. else if (fieldFlags & RFTMhasVirtualInitializer)
  815. {
  816. byte virtualKind;
  817. type.read(virtualKind);
  818. init = (void *)(memsize_t)virtualKind;
  819. }
  820. fieldFlags &= ~RFTMserializerFlags;
  821. info.fieldsArray[n] = new RtlFieldStrInfo(keep(fieldName), keep(xpath), lookupType(fieldType), fieldFlags, (const char *) init);
  822. }
  823. }
  824. info.fieldType &= ~RFTMserializerFlags;
  825. const RtlTypeInfo * result = info.createRtlTypeInfo();
  826. if (baseType == type_record)
  827. patchIfBlockParentRow(result, static_cast<const RtlRecordTypeInfo *>(result));
  828. return result;
  829. }
  830. void patchIfBlockParentRow(const RtlTypeInfo * fieldType, const RtlRecordTypeInfo * parentRow)
  831. {
  832. const RtlFieldInfo * const * fields = fieldType->queryFields();
  833. for (;*fields;fields++)
  834. {
  835. const RtlFieldInfo * cur = *fields;
  836. if (!cur)
  837. break;
  838. const RtlTypeInfo * curType = cur->type;
  839. if ((curType->fieldType & RFTMkind) == type_ifblock)
  840. {
  841. const RtlDynamicIfBlockTypeInfo * constifblock = static_cast<const RtlDynamicIfBlockTypeInfo *>(curType);
  842. RtlDynamicIfBlockTypeInfo * ifblock = const_cast<RtlDynamicIfBlockTypeInfo *>(constifblock);
  843. ifblock->setParent(parentRow);
  844. patchIfBlockParentRow(curType, parentRow);
  845. }
  846. }
  847. }
  848. };
  849. extern ECLRTL_API IRtlFieldTypeDeserializer *createRtlFieldTypeDeserializer()
  850. {
  851. return new CRtlFieldTypeDeserializer();
  852. }
  853. extern ECLRTL_API StringBuffer &dumpTypeInfo(StringBuffer &ret, const RtlTypeInfo *t)
  854. {
  855. return CRtlFieldTypeSerializer::serialize(ret, t);
  856. }
  857. extern ECLRTL_API bool dumpTypeInfo(MemoryBuffer &ret, const RtlTypeInfo *t)
  858. {
  859. try
  860. {
  861. CRtlFieldTypeBinSerializer::serialize(ret, t);
  862. return true;
  863. }
  864. catch (IException *E)
  865. {
  866. EXCLOG(E);
  867. E->Release();
  868. return false;
  869. }
  870. }
  871. extern ECLRTL_API void serializeRecordType(size32_t & __lenResult, void * & __result, IOutputMetaData & metaVal)
  872. {
  873. MemoryBuffer ret;
  874. try
  875. {
  876. CRtlFieldTypeBinSerializer::serialize(ret, metaVal.queryTypeInfo());
  877. }
  878. catch (IException * e)
  879. {
  880. ret.clear();
  881. e->Release();
  882. }
  883. __lenResult = ret.length();
  884. __result = ret.detach();
  885. }
  886. extern ECLRTL_API void dumpRecordType(size32_t & __lenResult,char * & __result,IOutputMetaData &metaVal)
  887. {
  888. StringBuffer ret;
  889. try
  890. {
  891. CRtlFieldTypeSerializer::serialize(ret, metaVal.queryTypeInfo());
  892. #ifdef _DEBUG
  893. StringBuffer ret2;
  894. CRtlFieldTypeDeserializer deserializer;
  895. CRtlFieldTypeSerializer::serialize(ret2, deserializer.deserialize(ret));
  896. assert(streq(ret, ret2));
  897. MemoryBuffer out;
  898. CRtlFieldTypeBinSerializer::serialize(out, metaVal.queryTypeInfo());
  899. CRtlFieldTypeDeserializer bindeserializer;
  900. CRtlFieldTypeSerializer::serialize(ret2.clear(), bindeserializer.deserialize(out));
  901. assert(streq(ret, ret2));
  902. #endif
  903. }
  904. catch (IException * e)
  905. {
  906. e->errorMessage(ret.clear());
  907. e->Release();
  908. }
  909. __lenResult = ret.length();
  910. __result = ret.detach();
  911. }
  912. extern ECLRTL_API void getFieldVal(size32_t & __lenResult,char * & __result, int column, IOutputMetaData & metaVal, const byte *row)
  913. {
  914. __lenResult = 0;
  915. __result = nullptr;
  916. if (column >= 0)
  917. {
  918. const RtlRecord &r = metaVal.queryRecordAccessor(true);
  919. if ((unsigned) column < r.getNumFields())
  920. {
  921. unsigned numOffsets = r.getNumVarFields() + 1;
  922. size_t * variableOffsets = (size_t *)alloca(numOffsets * sizeof(size_t));
  923. RtlRow offsetCalculator(r, row, numOffsets, variableOffsets);
  924. offsetCalculator.getUtf8(__lenResult, __result, column);
  925. }
  926. }
  927. }
  928. extern ECLRTL_API int getFieldNum(const char *fieldName, IOutputMetaData & metaVal)
  929. {
  930. const RtlRecord r = metaVal.queryRecordAccessor(true);
  931. return r.getFieldNum(fieldName);
  932. }
  933. enum FieldMatchType {
  934. // On a field, exactly one of the below is set, but translator returns a bitmap indicating
  935. // which were required (and we can restrict translation to allow some types but not others)
  936. match_perfect = 0x00, // exact type match - use memcpy
  937. match_link = 0x01, // copy a nested dataset by linking
  938. match_move = 0x02, // at least one field has moved (set on translator)
  939. match_remove = 0x04, // at least one field has been removed (set on translator)
  940. match_truncate = 0x08, // dest is truncated copy of source - use memcpy
  941. match_extend = 0x10, // dest is padded version of source - use memcpy and memset
  942. match_typecast = 0x20, // type has changed - cast required
  943. match_none = 0x40, // No matching field in source - use null value
  944. match_recurse = 0x80, // Use recursive translator for child records/datasets
  945. match_fail = 0x100, // no translation possible
  946. match_keychange = 0x200, // at least one affected field not marked as payload (set on translator)
  947. match_virtual = 0x800, // at least one affected field is a virtual field (set on translator)
  948. // This flag may be set in conjunction with the others
  949. match_inifblock = 0x400, // matching to a field in an ifblock - may not be present
  950. match_deblob = 0x1000, // source needs fetching from a blob prior to translation
  951. match_dynamic = 0x2000, // source needs fetching from dynamic source (callback)
  952. };
  953. StringBuffer &describeFlags(StringBuffer &out, FieldMatchType flags)
  954. {
  955. if (flags == match_perfect)
  956. return out.append("perfect");
  957. unsigned origlen = out.length();
  958. if (flags & match_link) out.append("|link");
  959. if (flags & match_move) out.append("|move");
  960. if (flags & match_remove) out.append("|remove");
  961. if (flags & match_truncate) out.append("|truncate");
  962. if (flags & match_extend) out.append("|extend");
  963. if (flags & match_typecast) out.append("|typecast");
  964. if (flags & match_none) out.append("|none");
  965. if (flags & match_recurse) out.append("|recurse");
  966. if (flags & match_inifblock) out.append("|ifblock");
  967. if (flags & match_keychange) out.append("|keychange");
  968. if (flags & match_fail) out.append("|fail");
  969. if (flags & match_virtual) out.append("|virtual");
  970. if (flags & match_deblob) out.append("|blob");
  971. if (flags & match_dynamic) out.append("|dynamic");
  972. assertex(out.length() > origlen);
  973. return out.remove(origlen, 1);
  974. }
  975. inline constexpr FieldMatchType operator|(FieldMatchType a, FieldMatchType b) { return (FieldMatchType)((int)a | (int)b); }
  976. inline FieldMatchType &operator|=(FieldMatchType &a, FieldMatchType b) { return (FieldMatchType &) ((int &)a |= (int)b); }
  977. class GeneralRecordTranslator : public CInterfaceOf<IDynamicTransform>
  978. {
  979. public:
  980. GeneralRecordTranslator(const RtlRecord &_destRecInfo, const RtlRecord &_srcRecInfo, bool _binarySource, type_vals _callbackRawType = type_any)
  981. : destRecInfo(_destRecInfo), sourceRecInfo(_srcRecInfo), binarySource(_binarySource), callbackRawType(_callbackRawType)
  982. {
  983. matchInfo = new MatchInfo[destRecInfo.getNumFields()];
  984. createMatchInfo();
  985. #ifdef _DEBUG
  986. //describe();
  987. #endif
  988. }
  989. ~GeneralRecordTranslator()
  990. {
  991. delete [] matchInfo;
  992. }
  993. // IDynamicTransform impl.
  994. virtual void describe() const override
  995. {
  996. doDescribe(0);
  997. }
  998. virtual size32_t translate(ARowBuilder &builder, IVirtualFieldCallback & callback, const byte *sourceRec) const override
  999. {
  1000. assertex(binarySource);
  1001. return doTranslate(builder, callback, 0, sourceRec);
  1002. }
  1003. virtual size32_t translate(ARowBuilder &builder, IVirtualFieldCallback & callback, const RtlRow &sourceRow) const override
  1004. {
  1005. assertex(binarySource);
  1006. sourceRow.lazyCalcOffsets(-1); // MORE - could save the max one we actually need...
  1007. return doTranslateOpaqueType(builder, callback, 0, &sourceRow);
  1008. }
  1009. virtual size32_t translate(ARowBuilder &builder, IVirtualFieldCallback & callback, const IDynamicFieldValueFetcher & fetcher) const override
  1010. {
  1011. assertex(!binarySource);
  1012. return doTranslateOpaqueType(builder, callback, 0, &fetcher);
  1013. }
  1014. virtual bool canTranslate() const override
  1015. {
  1016. return (matchFlags & match_fail) == 0;
  1017. }
  1018. virtual bool needsTranslate() const override
  1019. {
  1020. return !binarySource || (matchFlags & ~(match_link|match_inifblock)) != 0;
  1021. }
  1022. virtual bool needsNonVirtualTranslate() const override
  1023. {
  1024. return (matchFlags & ~(match_link|match_virtual|match_keychange|match_inifblock)) != 0;
  1025. }
  1026. virtual bool keyedTranslated() const override
  1027. {
  1028. return (matchFlags & match_keychange) != 0;
  1029. }
  1030. private:
  1031. void doDescribe(unsigned indent) const
  1032. {
  1033. unsigned perfect=0;
  1034. unsigned reported=0;
  1035. for (unsigned idx = 0; idx < destRecInfo.getNumFields(); idx++)
  1036. {
  1037. const char *source = destRecInfo.queryName(idx);
  1038. const MatchInfo &match = matchInfo[idx];
  1039. if (match.matchType == match_none)
  1040. DBGLOG("%*sNo match for field %s - default value will be used", indent, "", source);
  1041. else if (match.matchType == match_virtual)
  1042. DBGLOG("%*sUse virtual value for field %s", indent, "", source);
  1043. else
  1044. {
  1045. if (match.matchType != match_perfect)
  1046. {
  1047. reported++;
  1048. StringBuffer matchStr;
  1049. DBGLOG("%*sMatch (%s) to field %d for field %s (typecode %x)", indent, "", describeFlags(matchStr, match.matchType).str(), match.matchIdx, source, destRecInfo.queryType(idx)->fieldType);
  1050. if (match.subTrans)
  1051. match.subTrans->doDescribe(indent+2);
  1052. }
  1053. else
  1054. perfect++;
  1055. }
  1056. }
  1057. if (allUnmatched.ordinality())
  1058. {
  1059. VStringBuffer msg("%*sDropped field", indent, "");
  1060. if (allUnmatched.ordinality()>1)
  1061. msg.append('s');
  1062. for (unsigned idx = 0; idx < allUnmatched.ordinality() && idx < 5; idx++)
  1063. {
  1064. if (idx)
  1065. msg.append(',');
  1066. msg.appendf(" %s", sourceRecInfo.queryName(allUnmatched.item(idx)));
  1067. }
  1068. if (allUnmatched.ordinality() > 5)
  1069. msg.appendf(" and %u other fields", allUnmatched.ordinality() - 5);
  1070. DBGLOG("%s", msg.str());
  1071. }
  1072. if (!canTranslate())
  1073. DBGLOG("%*sTranslation is NOT possible", indent, "");
  1074. else if (needsTranslate())
  1075. {
  1076. StringBuffer matchStr;
  1077. if (perfect)
  1078. DBGLOG("%u %sfield%s matched perfectly", perfect, reported ? "other " : "", perfect==1 ? "" : "s");
  1079. DBGLOG("%*sTranslation is possible (%s)", indent, "", describeFlags(matchStr, matchFlags).str());
  1080. }
  1081. else
  1082. DBGLOG("%*sTranslation is not necessary", indent, "");
  1083. }
  1084. size32_t doTranslate(ARowBuilder &builder, IVirtualFieldCallback & callback, size32_t offset, const byte *sourceRec) const
  1085. {
  1086. unsigned numOffsets = sourceRecInfo.getNumVarFields() + 1;
  1087. size_t * variableOffsets = (size_t *)alloca(numOffsets * sizeof(size_t));
  1088. RtlRow sourceRow(sourceRecInfo, sourceRec, numOffsets, variableOffsets); // MORE - could save the max source offset we actually need, and only set up that many...
  1089. return doTranslateOpaqueType(builder, callback, offset, &sourceRow);
  1090. }
  1091. size32_t doTranslateOpaqueType(ARowBuilder &builder, IVirtualFieldCallback & callback, size32_t offset, const void *sourceRow) const
  1092. {
  1093. dbgassertex(canTranslate());
  1094. byte * destConditions = (byte *)alloca(destRecInfo.getNumIfBlocks() * sizeof(byte));
  1095. memset(destConditions, 2, destRecInfo.getNumIfBlocks() * sizeof(byte));
  1096. size32_t estimate = destRecInfo.getFixedSize();
  1097. bool hasBlobs = false;
  1098. if (!estimate)
  1099. {
  1100. if (binarySource)
  1101. estimate = estimateNewSize(*(const RtlRow *)sourceRow);
  1102. else
  1103. estimate = destRecInfo.getMinRecordSize();
  1104. builder.ensureCapacity(offset+estimate, "record");
  1105. }
  1106. size32_t origOffset = offset;
  1107. for (unsigned idx = 0; idx < destRecInfo.getNumFields(); idx++)
  1108. {
  1109. const RtlFieldInfo *field = destRecInfo.queryField(idx);
  1110. if (field->omitable() && destRecInfo.excluded(field, builder.getSelf(), destConditions))
  1111. continue;
  1112. const RtlTypeInfo *type = field->type;
  1113. const MatchInfo &match = matchInfo[idx];
  1114. if (match.matchType == match_none || match.matchType==match_fail)
  1115. {
  1116. offset = type->buildNull(builder, offset, field);
  1117. }
  1118. else if (match.matchType == match_virtual)
  1119. {
  1120. switch (getVirtualInitializer(field->initializer))
  1121. {
  1122. case FVirtualFilePosition:
  1123. offset = type->buildInt(builder, offset, field, callback.getFilePosition(sourceRow));
  1124. break;
  1125. case FVirtualLocalFilePosition:
  1126. offset = type->buildInt(builder, offset, field, callback.getLocalFilePosition(sourceRow));
  1127. break;
  1128. case FVirtualFilename:
  1129. {
  1130. const char * filename = callback.queryLogicalFilename(sourceRow);
  1131. offset = type->buildString(builder, offset, field, strlen(filename), filename);
  1132. break;
  1133. }
  1134. default:
  1135. throwUnexpected();
  1136. }
  1137. }
  1138. else
  1139. {
  1140. unsigned matchField = match.matchIdx;
  1141. const RtlTypeInfo *sourceType = sourceRecInfo.queryType(matchField);
  1142. size_t sourceOffset = 0;
  1143. const byte *source = nullptr;
  1144. size_t copySize = 0;
  1145. if (binarySource)
  1146. {
  1147. const RtlRow &rtlRow = *(const RtlRow *)sourceRow;
  1148. sourceOffset = rtlRow.getOffset(matchField);
  1149. source = rtlRow.queryRow() + sourceOffset;
  1150. copySize = rtlRow.getSize(matchField);
  1151. }
  1152. if (match.matchType & match_deblob)
  1153. {
  1154. offset_t blobId = sourceType->getInt(source);
  1155. sourceType = sourceType->queryChildType();
  1156. sourceOffset = 0;
  1157. source = callback.lookupBlob(blobId);
  1158. copySize = sourceType->size(source, source);
  1159. hasBlobs = true;
  1160. }
  1161. if (copySize == 0 && (match.matchType & match_inifblock)) // Field is missing because of an ifblock - use default value
  1162. {
  1163. offset = type->buildNull(builder, offset, field);
  1164. }
  1165. else
  1166. {
  1167. switch (match.matchType & ~(match_inifblock|match_deblob))
  1168. {
  1169. case match_perfect:
  1170. {
  1171. // Look ahead for other perfect matches and combine the copies
  1172. if (!(match.matchType & match_deblob))
  1173. {
  1174. while (idx < destRecInfo.getNumFields()-1)
  1175. {
  1176. const MatchInfo &nextMatch = matchInfo[idx+1];
  1177. if (nextMatch.matchType == match_perfect && nextMatch.matchIdx == matchField+1)
  1178. {
  1179. idx++;
  1180. matchField++;
  1181. }
  1182. else
  1183. break;
  1184. }
  1185. copySize = ((const RtlRow *)sourceRow)->getOffset(matchField+1) - sourceOffset;
  1186. }
  1187. builder.ensureCapacity(offset+copySize, field->name);
  1188. memcpy(builder.getSelf()+offset, source, copySize);
  1189. offset += copySize;
  1190. break;
  1191. }
  1192. case match_truncate:
  1193. {
  1194. assert(type->isFixedSize());
  1195. copySize = type->getMinSize();
  1196. builder.ensureCapacity(offset+copySize, field->name);
  1197. memcpy(builder.getSelf()+offset, source, copySize);
  1198. offset += copySize;
  1199. break;
  1200. }
  1201. case match_extend:
  1202. {
  1203. assert(type->isFixedSize());
  1204. size32_t destSize = type->getMinSize();
  1205. builder.ensureCapacity(offset+destSize, field->name);
  1206. memcpy(builder.getSelf()+offset, source, copySize);
  1207. offset += copySize;
  1208. unsigned fillSize = destSize - copySize;
  1209. memset(builder.getSelf()+offset, match.fillChar, fillSize);
  1210. offset += fillSize;
  1211. break;
  1212. }
  1213. case match_typecast:
  1214. offset = translateScalar(builder, offset, field, *type, *sourceType, source);
  1215. break;
  1216. case match_typecast|match_dynamic:
  1217. {
  1218. const IDynamicFieldValueFetcher &callbackRowHandler = *(const IDynamicFieldValueFetcher *)sourceRow;
  1219. source = callbackRowHandler.queryValue(matchField, copySize);
  1220. if (callbackRawType == type_string)
  1221. offset = translateScalarFromString(builder, offset, field, *type, *sourceType, (const char *)source, (size_t)copySize);
  1222. else
  1223. offset = translateScalarFromUtf8(builder, offset, field, *type, *sourceType, (const char *)source, (size_t)copySize);
  1224. break;
  1225. }
  1226. case match_link:
  1227. {
  1228. // a 32-bit record count, and a (linked) pointer to an array of record pointers
  1229. byte *dest = builder.ensureCapacity(offset+sizeof(size32_t)+sizeof(const byte **), field->name)+offset;
  1230. *(size32_t *)dest = *(size32_t *)source;
  1231. *(const byte ***)(dest + sizeof(size32_t)) = rtlLinkRowset(*(const byte ***)(source + sizeof(size32_t)));
  1232. offset += sizeof(size32_t)+sizeof(const byte **);
  1233. break;
  1234. }
  1235. case match_recurse|match_dynamic:
  1236. {
  1237. const IDynamicFieldValueFetcher &callbackRowHandler = *(const IDynamicFieldValueFetcher *)sourceRow;
  1238. Owned<IDynamicRowIterator> iterator = callbackRowHandler.getNestedIterator(matchField);
  1239. if (type->getType()==type_record)
  1240. {
  1241. IDynamicFieldValueFetcher &fieldFetcher = iterator->query();
  1242. offset = match.subTrans->doTranslateOpaqueType(builder, callback, offset, &fieldFetcher);
  1243. }
  1244. else if (type->isLinkCounted())
  1245. {
  1246. // a 32-bit record count, and a pointer to an array of record pointers
  1247. IEngineRowAllocator *childAllocator = builder.queryAllocator()->createChildRowAllocator(type->queryChildType());
  1248. assertex(childAllocator); // May not be available when using serialized types (but unlikely to want to create linkcounted children remotely either)
  1249. size32_t sizeInBytes = sizeof(size32_t) + sizeof(void *);
  1250. builder.ensureCapacity(offset+sizeInBytes, field->name);
  1251. size32_t numRows = 0;
  1252. const byte **childRows = nullptr;
  1253. ForEach(*iterator)
  1254. {
  1255. IDynamicFieldValueFetcher &fieldFetcher = iterator->query();
  1256. RtlDynamicRowBuilder childBuilder(*childAllocator);
  1257. size32_t childLen = match.subTrans->doTranslateOpaqueType(childBuilder, callback, 0, &fieldFetcher);
  1258. childRows = childAllocator->appendRowOwn(childRows, ++numRows, (void *) childBuilder.finalizeRowClear(childLen));
  1259. }
  1260. if (type->getType() == type_dictionary)
  1261. {
  1262. const RtlTypeInfo * childType = type->queryChildType();
  1263. assertex(childType && childType->getType() == type_record);
  1264. CHThorHashLookupInfo lookupHelper(static_cast<const RtlRecordTypeInfo &>(*childType));
  1265. rtlCreateDictionaryFromDataset(numRows, childRows, childAllocator, lookupHelper);
  1266. }
  1267. // Go back in and patch the count, remembering it may have moved
  1268. rtlWriteInt4(builder.getSelf()+offset, numRows);
  1269. * ( const void * * ) (builder.getSelf()+offset+sizeof(size32_t)) = childRows;
  1270. offset += sizeInBytes;
  1271. }
  1272. else
  1273. {
  1274. size32_t countOffset = offset;
  1275. byte *dest = builder.ensureCapacity(offset+sizeof(size32_t), field->name)+offset;
  1276. offset += sizeof(size32_t);
  1277. size32_t initialOffset = offset;
  1278. *(size32_t *)dest = 0; // patched below when true figure known
  1279. ForEach(*iterator)
  1280. {
  1281. IDynamicFieldValueFetcher &fieldFetcher = iterator->query();
  1282. offset = match.subTrans->doTranslateOpaqueType(builder, callback, offset, &fieldFetcher);
  1283. }
  1284. dest = builder.getSelf() + countOffset; // Note - may have been moved by reallocs since last calculated
  1285. *(size32_t *)dest = offset - initialOffset;
  1286. }
  1287. break;
  1288. }
  1289. case match_recurse:
  1290. if (type->getType()==type_record)
  1291. offset = match.subTrans->doTranslate(builder, callback, offset, source);
  1292. else if (type->isLinkCounted())
  1293. {
  1294. // a 32-bit record count, and a pointer to an array of record pointers
  1295. Owned<IEngineRowAllocator> childAllocator = builder.queryAllocator()->createChildRowAllocator(type->queryChildType());
  1296. assertex(childAllocator); // May not be available when using serialized types (but unlikely to want to create linkcounted children remotely either)
  1297. size32_t sizeInBytes = sizeof(size32_t) + sizeof(void *);
  1298. builder.ensureCapacity(offset+sizeInBytes, field->name);
  1299. size32_t numRows = 0;
  1300. const byte **childRows = nullptr;
  1301. if (sourceType->isLinkCounted())
  1302. {
  1303. // a 32-bit count, then a pointer to the source rows
  1304. size32_t childCount = *(size32_t *) source;
  1305. source += sizeof(size32_t);
  1306. const byte ** sourceRows = *(const byte***) source;
  1307. for (size32_t childRow = 0; childRow < childCount; childRow++)
  1308. {
  1309. RtlDynamicRowBuilder childBuilder(*childAllocator);
  1310. size32_t childLen = match.subTrans->doTranslate(childBuilder, callback, 0, sourceRows[childRow]);
  1311. childRows = childAllocator->appendRowOwn(childRows, ++numRows, (void *) childBuilder.finalizeRowClear(childLen));
  1312. }
  1313. }
  1314. else
  1315. {
  1316. // a 32-bit size, then rows inline
  1317. size32_t childSize = *(size32_t *) source;
  1318. source += sizeof(size32_t);
  1319. const byte *initialSource = source;
  1320. while ((size_t)(source - initialSource) < childSize)
  1321. {
  1322. RtlDynamicRowBuilder childBuilder(*childAllocator);
  1323. size32_t childLen = match.subTrans->doTranslate(childBuilder, callback, 0, source);
  1324. childRows = childAllocator->appendRowOwn(childRows, ++numRows, (void *) childBuilder.finalizeRowClear(childLen));
  1325. source += sourceType->queryChildType()->size(source, nullptr); // MORE - shame to repeat a calculation that the translate above almost certainly just did
  1326. }
  1327. }
  1328. if (type->getType() == type_dictionary)
  1329. {
  1330. const RtlTypeInfo * childType = type->queryChildType();
  1331. assertex(childType && childType->getType() == type_record);
  1332. CHThorHashLookupInfo lookupHelper(static_cast<const RtlRecordTypeInfo &>(*childType));
  1333. rtlCreateDictionaryFromDataset(numRows, childRows, childAllocator, lookupHelper);
  1334. }
  1335. // Go back in and patch the count, remembering it may have moved
  1336. rtlWriteInt4(builder.getSelf()+offset, numRows);
  1337. * ( const void * * ) (builder.getSelf()+offset+sizeof(size32_t)) = childRows;
  1338. offset += sizeInBytes;
  1339. }
  1340. else
  1341. {
  1342. size32_t countOffset = offset;
  1343. byte *dest = builder.ensureCapacity(offset+sizeof(size32_t), field->name)+offset;
  1344. offset += sizeof(size32_t);
  1345. size32_t initialOffset = offset;
  1346. *(size32_t *)dest = 0; // patched below when true figure known
  1347. if (sourceType->isLinkCounted())
  1348. {
  1349. // a 32-bit count, then a pointer to the source rows
  1350. size32_t childCount = *(size32_t *) source;
  1351. source += sizeof(size32_t);
  1352. const byte ** sourceRows = *(const byte***) source;
  1353. for (size32_t childRow = 0; childRow < childCount; childRow++)
  1354. {
  1355. const byte * row = sourceRows[childRow];
  1356. //Dictionaries have blank rows - ignore them when serializing (to a dataset)
  1357. if (row)
  1358. offset = match.subTrans->doTranslate(builder, callback, offset, row);
  1359. }
  1360. }
  1361. else
  1362. {
  1363. // a 32-bit size, then rows inline
  1364. size32_t childSize = *(size32_t *) source;
  1365. source += sizeof(size32_t);
  1366. const byte *initialSource = source;
  1367. while ((size_t)(source - initialSource) < childSize)
  1368. {
  1369. offset = match.subTrans->doTranslate(builder, callback, offset, source);
  1370. source += sourceType->queryChildType()->size(source, nullptr); // MORE - shame to repeat a calculation that the translate above almost certainly just did
  1371. }
  1372. }
  1373. dest = builder.getSelf() + countOffset; // Note - may have been moved by reallocs since last calculated
  1374. *(size32_t *)dest = offset - initialOffset;
  1375. }
  1376. break;
  1377. default:
  1378. throwUnexpected();
  1379. }
  1380. }
  1381. }
  1382. }
  1383. if (estimate && offset-origOffset != estimate)
  1384. {
  1385. if (offset == origOffset)
  1386. {
  1387. //Zero size records are treated as single byte to avoid confusion with sizes returned from transforms etc.
  1388. offset++;
  1389. }
  1390. else
  1391. {
  1392. // Note - ifblocks make this assertion invalid. We do not account for potentially omitted fields
  1393. // when estimating target record size.
  1394. if (!destRecInfo.getNumIfBlocks() && !hasBlobs)
  1395. assert(offset-origOffset > estimate); // Estimate is always supposed to be conservative
  1396. #ifdef TRACE_TRANSLATION
  1397. DBGLOG("Wrote %u bytes to record (estimate was %u)\n", offset-origOffset, estimate);
  1398. #endif
  1399. }
  1400. }
  1401. return offset;
  1402. }
  1403. inline FieldMatchType match() const
  1404. {
  1405. return matchFlags;
  1406. }
  1407. const RtlRecord &destRecInfo;
  1408. const RtlRecord &sourceRecInfo;
  1409. bool binarySource = true;
  1410. type_vals callbackRawType;
  1411. unsigned fixedDelta = 0; // total size of all fixed-size source fields that are not matched
  1412. UnsignedArray allUnmatched; // List of all source fields that are unmatched (so that we can trace them)
  1413. UnsignedArray variableUnmatched; // List of all variable-size source fields that are unmatched
  1414. FieldMatchType matchFlags = match_perfect;
  1415. struct MatchInfo
  1416. {
  1417. unsigned matchIdx = 0;
  1418. FieldMatchType matchType = match_fail;
  1419. char fillChar = 0;
  1420. GeneralRecordTranslator *subTrans = nullptr;
  1421. ~MatchInfo()
  1422. {
  1423. delete subTrans;
  1424. }
  1425. } *matchInfo;
  1426. static size32_t translateScalarFromUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, const RtlTypeInfo &destType, const RtlTypeInfo &sourceType, const char *source, size_t srcSize)
  1427. {
  1428. switch(destType.getType())
  1429. {
  1430. case type_boolean:
  1431. case type_int:
  1432. case type_swapint:
  1433. case type_packedint:
  1434. case type_filepos:
  1435. case type_keyedint:
  1436. {
  1437. __int64 res = rtlStrToInt8(srcSize, source);
  1438. offset = destType.buildInt(builder, offset, field, res);
  1439. break;
  1440. }
  1441. case type_real:
  1442. {
  1443. double res = rtlStrToReal(srcSize, source);
  1444. offset = destType.buildReal(builder, offset, field, res);
  1445. break;
  1446. }
  1447. case type_data:
  1448. case type_string:
  1449. case type_decimal: // Go via string - not common enough to special-case
  1450. case type_varstring:
  1451. case type_qstring:
  1452. case type_utf8:
  1453. //MORE: Could special case casting from utf8 to utf8 similar to strings above
  1454. case type_unicode:
  1455. case type_varunicode:
  1456. {
  1457. size32_t utf8chars = rtlUtf8Length(srcSize, source);
  1458. offset = destType.buildUtf8(builder, offset, field, utf8chars, source);
  1459. break;
  1460. }
  1461. case type_set:
  1462. {
  1463. UNIMPLEMENTED; // JCS->GH - but perhaps can/should translate using iterator too?
  1464. break;
  1465. }
  1466. default:
  1467. throwUnexpected();
  1468. }
  1469. return offset;
  1470. }
  1471. static size32_t translateScalarFromString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, const RtlTypeInfo &destType, const RtlTypeInfo &sourceType, const char *source, size_t srcSize)
  1472. {
  1473. switch(destType.getType())
  1474. {
  1475. case type_boolean:
  1476. case type_int:
  1477. case type_swapint:
  1478. case type_packedint:
  1479. case type_filepos:
  1480. case type_keyedint:
  1481. {
  1482. __int64 res = rtlStrToInt8(srcSize, source);
  1483. offset = destType.buildInt(builder, offset, field, res);
  1484. break;
  1485. }
  1486. case type_real:
  1487. {
  1488. double res = rtlStrToReal(srcSize, source);
  1489. offset = destType.buildReal(builder, offset, field, res);
  1490. break;
  1491. }
  1492. case type_data:
  1493. case type_string:
  1494. case type_decimal: // Go via string - not common enough to special-case
  1495. case type_varstring:
  1496. case type_qstring:
  1497. case type_utf8:
  1498. //MORE: Could special case casting from utf8 to utf8 similar to strings above
  1499. case type_unicode:
  1500. case type_varunicode:
  1501. {
  1502. offset = destType.buildString(builder, offset, field, srcSize, source);
  1503. break;
  1504. }
  1505. case type_set:
  1506. {
  1507. UNIMPLEMENTED; // JCS->GH - but perhaps can/should translate using iterator too?
  1508. break;
  1509. }
  1510. default:
  1511. throwUnexpected();
  1512. }
  1513. return offset;
  1514. }
  1515. static bool canTranslateNonScalar(const RtlTypeInfo * type, const RtlTypeInfo * sourceType)
  1516. {
  1517. auto target = type->getType();
  1518. auto source = sourceType->getType();
  1519. if (target == source)
  1520. return true;
  1521. if ((target == type_dictionary) && (source == type_table))
  1522. return true;
  1523. if ((target == type_table) && (source == type_dictionary))
  1524. return true;
  1525. return false;
  1526. }
  1527. void createMatchInfo()
  1528. {
  1529. unsigned defaulted = 0;
  1530. for (unsigned idx = 0; idx < destRecInfo.getNumFields(); idx++)
  1531. {
  1532. const RtlFieldInfo *field = destRecInfo.queryField(idx);
  1533. const RtlTypeInfo *type = field->type;
  1534. MatchInfo &info = matchInfo[idx];
  1535. info.matchIdx = sourceRecInfo.getFieldNum(destRecInfo.queryName(idx));
  1536. if (info.matchIdx == (unsigned) -1)
  1537. {
  1538. const byte * initializer = (const byte *) field->initializer;
  1539. info.matchType = isVirtualInitializer(initializer) ? match_virtual : match_none;
  1540. size32_t defaultSize = (initializer && !isVirtualInitializer(initializer)) ? type->size(initializer, nullptr) : type->getMinSize();
  1541. fixedDelta -= defaultSize;
  1542. if ((field->flags & RFTMispayloadfield) == 0)
  1543. matchFlags |= match_keychange;
  1544. defaulted++;
  1545. //DBGLOG("Decreasing fixedDelta size by %d to %d for defaulted field %d (%s)", defaultSize, fixedDelta, idx, destRecInfo.queryName(idx));
  1546. }
  1547. else
  1548. {
  1549. bool deblob = false;
  1550. const RtlTypeInfo *sourceType = sourceRecInfo.queryType(info.matchIdx);
  1551. if (binarySource && sourceType->isBlob())
  1552. {
  1553. if (type->isBlob())
  1554. {
  1555. }
  1556. else
  1557. {
  1558. sourceType = sourceType->queryChildType();
  1559. deblob = true;
  1560. }
  1561. }
  1562. if (!type->isScalar() || !sourceType->isScalar())
  1563. {
  1564. if (!canTranslateNonScalar(type, sourceType))
  1565. info.matchType = match_fail; // No translation from one non-scalar type to another
  1566. else
  1567. {
  1568. switch (type->getType())
  1569. {
  1570. case type_set:
  1571. if (binarySource)
  1572. {
  1573. if (type->queryChildType()->fieldType==sourceType->queryChildType()->fieldType &&
  1574. type->queryChildType()->length==sourceType->queryChildType()->length)
  1575. info.matchType = match_perfect;
  1576. else
  1577. info.matchType = match_typecast;
  1578. }
  1579. else
  1580. info.matchType = match_typecast|match_dynamic;
  1581. break;
  1582. case type_row: // These are not expected I think...
  1583. throwUnexpected();
  1584. case type_ifblock:
  1585. case type_record:
  1586. case type_table:
  1587. case type_dictionary:
  1588. {
  1589. const RtlRecord *subDest = destRecInfo.queryNested(idx);
  1590. const RtlRecord *subSrc = sourceRecInfo.queryNested(info.matchIdx);
  1591. info.subTrans = new GeneralRecordTranslator(*subDest, *subSrc, binarySource);
  1592. if (!info.subTrans->needsTranslate())
  1593. {
  1594. if (!binarySource)
  1595. info.matchType = match_recurse|match_dynamic;
  1596. else
  1597. {
  1598. // Child does not require translation, but check linkcount mode matches too!
  1599. if (type->isLinkCounted())
  1600. {
  1601. if (sourceType->isLinkCounted())
  1602. info.matchType = match_link;
  1603. else
  1604. info.matchType = match_recurse;
  1605. }
  1606. else
  1607. {
  1608. if (sourceType->isLinkCounted())
  1609. info.matchType = match_recurse;
  1610. else
  1611. info.matchType = match_perfect;
  1612. }
  1613. if (info.matchType != match_recurse)
  1614. {
  1615. delete info.subTrans;
  1616. info.subTrans = nullptr;
  1617. }
  1618. }
  1619. }
  1620. else if (info.subTrans->canTranslate())
  1621. {
  1622. info.matchType = binarySource ? match_recurse : (match_recurse|match_dynamic);
  1623. unsigned childFlags = info.subTrans->matchFlags;
  1624. //Ignore differences in the keyed flag for child structures (it will be set later if this field is keyed)
  1625. matchFlags |= (FieldMatchType)(childFlags & ~match_keychange);
  1626. }
  1627. else
  1628. info.matchType = match_fail;
  1629. break;
  1630. }
  1631. case type_blob:
  1632. if (!binarySource)
  1633. info.matchType = match_fail;
  1634. else if (sourceType->isBlob())
  1635. info.matchType = match_perfect; // We don't check that the child type matches
  1636. else
  1637. info.matchType = match_fail;
  1638. break;
  1639. default:
  1640. info.matchType = match_fail;
  1641. break;
  1642. }
  1643. }
  1644. }
  1645. else if (!binarySource)
  1646. info.matchType = match_typecast|match_dynamic;
  1647. else if ((type->fieldType==sourceType->fieldType))
  1648. {
  1649. if (type->length==sourceType->length)
  1650. {
  1651. info.matchType = match_perfect;
  1652. }
  1653. else
  1654. {
  1655. assert(type->isFixedSize()); // Both variable size would have matched length above
  1656. info.matchType = match_typecast;
  1657. if (type->length < sourceType->length)
  1658. {
  1659. if (type->canTruncate())
  1660. {
  1661. info.matchType = match_truncate;
  1662. fixedDelta += sourceType->getMinSize()-type->getMinSize();
  1663. //DBGLOG("Increasing fixedDelta size by %d to %d for truncated field %d (%s)", sourceType->getMinSize()-type->getMinSize(), fixedDelta, idx, destRecInfo.queryName(idx));
  1664. }
  1665. }
  1666. else
  1667. {
  1668. if (type->canExtend(info.fillChar))
  1669. {
  1670. info.matchType = match_extend;
  1671. fixedDelta += sourceType->getMinSize()-type->getMinSize();
  1672. //DBGLOG("Decreasing fixedDelta size by %d to %d for truncated field %d (%s)", type->getMinSize()-sourceType->getMinSize(), fixedDelta, idx, destRecInfo.queryName(idx));
  1673. }
  1674. }
  1675. }
  1676. }
  1677. else
  1678. info.matchType = match_typecast;
  1679. if (deblob)
  1680. info.matchType |= match_deblob;
  1681. unsigned sourceFlags = sourceRecInfo.queryField(info.matchIdx)->flags;
  1682. if (sourceFlags & RFTMinifblock)
  1683. info.matchType |= match_inifblock; // Avoids incorrect commoning up of adjacent matches
  1684. // MORE - could note the highest interesting fieldnumber in the source and not bother filling in offsets after that
  1685. // Not sure it would help much though - usually need to know the total record size anyway in real life
  1686. if (idx != info.matchIdx)
  1687. matchFlags |= match_move;
  1688. //Whether this field is in an ifblock, or needs to be copied by linking it do not count as changes
  1689. FieldMatchType maskedType = (FieldMatchType)(info.matchType & ~(match_link|match_inifblock));
  1690. if (((maskedType != match_perfect) || (idx != info.matchIdx)) && ((field->flags & RFTMispayloadfield) == 0 || (sourceFlags & RFTMispayloadfield) == 0))
  1691. matchFlags |= match_keychange;
  1692. }
  1693. matchFlags |= info.matchType;
  1694. }
  1695. if (sourceRecInfo.getNumFields() > destRecInfo.getNumFields()-defaulted)
  1696. {
  1697. matchFlags |= match_remove;
  1698. for (unsigned idx = 0; idx < sourceRecInfo.getNumFields(); idx++)
  1699. {
  1700. const RtlFieldInfo *field = sourceRecInfo.queryField(idx);
  1701. const char *name = sourceRecInfo.queryName(idx);
  1702. if (destRecInfo.getFieldNum(name) == (unsigned) -1)
  1703. {
  1704. // unmatched field
  1705. if ((field->flags & RFTMispayloadfield) == 0)
  1706. matchFlags |= match_keychange;
  1707. if (!destRecInfo.getFixedSize())
  1708. {
  1709. const RtlTypeInfo *type = field->type;
  1710. if (type->isFixedSize())
  1711. {
  1712. //DBGLOG("Reducing estimated size by %d for (fixed size) omitted field %s", (int) type->getMinSize(), field->name);
  1713. fixedDelta += type->getMinSize();
  1714. }
  1715. else
  1716. variableUnmatched.append(idx);
  1717. }
  1718. allUnmatched.append(idx);
  1719. }
  1720. }
  1721. //DBGLOG("Source record contains %d bytes of omitted fixed size fields", fixedDelta);
  1722. }
  1723. }
  1724. size32_t estimateNewSize(const RtlRow &sourceRow) const
  1725. {
  1726. //DBGLOG("Source record size is %d", (int) sourceRow.getRecordSize());
  1727. size32_t expectedSize = sourceRow.getRecordSize() - fixedDelta;
  1728. //DBGLOG("Source record size without omitted fixed size fields is %d", expectedSize);
  1729. ForEachItemIn(i, variableUnmatched)
  1730. {
  1731. unsigned fieldNo = variableUnmatched.item(i);
  1732. expectedSize -= sourceRow.getSize(fieldNo);
  1733. //DBGLOG("Reducing estimated size by %d to %d for omitted field %d (%s)", (int) sourceRow.getSize(fieldNo), expectedSize, fieldNo, sourceRecInfo.queryName(fieldNo));
  1734. }
  1735. if (matchFlags & ~(match_perfect|match_link|match_none|match_virtual|match_extend|match_truncate))
  1736. {
  1737. for (unsigned idx = 0; idx < destRecInfo.getNumFields(); idx++)
  1738. {
  1739. const MatchInfo &match = matchInfo[idx];
  1740. const RtlTypeInfo *type = destRecInfo.queryType(idx);
  1741. unsigned matchField = match.matchIdx;
  1742. switch (match.matchType)
  1743. {
  1744. case match_perfect:
  1745. case match_link:
  1746. case match_none:
  1747. case match_virtual:
  1748. case match_extend:
  1749. case match_truncate:
  1750. // These ones were already included in fixedDelta
  1751. break;
  1752. default:
  1753. // This errs on the side of small - i.e. it assumes that all typecasts end up at minimum size
  1754. // We could do better in some cases e.g. variable string <-> variable unicode we can assume factor of 2,
  1755. // uft8 <-> string we could calculate here - but unlikely to be worth the effort.
  1756. // But it's fine for fixed size output fields, including truncate/extend
  1757. // We could also precalculate the expected delta if all omitted fields are fixed size - but not sure how likely/worthwhile that is.
  1758. expectedSize += type->getMinSize() - sourceRow.getSize(matchField);
  1759. //DBGLOG("Adjusting estimated size by (%d - %d) to %d for translated field %d (%s)", (int) sourceRow.getSize(matchField), type->getMinSize(), expectedSize, matchField, sourceRecInfo.queryName(matchField));
  1760. break;
  1761. }
  1762. }
  1763. }
  1764. return expectedSize;
  1765. }
  1766. };
  1767. extern ECLRTL_API const IDynamicTransform *createRecordTranslator(const RtlRecord &destRecInfo, const RtlRecord &srcRecInfo)
  1768. {
  1769. return new GeneralRecordTranslator(destRecInfo, srcRecInfo, true);
  1770. }
  1771. extern ECLRTL_API const IDynamicTransform *createRecordTranslatorViaCallback(const RtlRecord &destRecInfo, const RtlRecord &srcRecInfo, type_vals rawType)
  1772. {
  1773. return new GeneralRecordTranslator(destRecInfo, srcRecInfo, false, rawType);
  1774. }
  1775. extern ECLRTL_API void throwTranslationError(const RtlRecord & destRecInfo, const RtlRecord & srcRecInfo, const char * filename)
  1776. {
  1777. Owned<const IDynamicTransform> translator = createRecordTranslator(destRecInfo, srcRecInfo);
  1778. #ifdef _DEBUG
  1779. translator->describe();
  1780. #endif
  1781. if (!translator->canTranslate())
  1782. throw MakeStringException(0, "Untranslatable record layout mismatch detected for: %s", filename);
  1783. throw MakeStringException(0, "Translatable key layout mismatch reading file %s but translation disabled", filename);
  1784. }
  1785. class TranslatedRowStream : public CInterfaceOf<IRowStream>
  1786. {
  1787. public:
  1788. TranslatedRowStream(IRowStream *_inputStream, IEngineRowAllocator *_resultAllocator, const RtlRecord &outputRecord, const RtlRecord &inputRecord)
  1789. : inputStream(_inputStream), resultAllocator(_resultAllocator)
  1790. {
  1791. translator.setown(createRecordTranslator(outputRecord, inputRecord));
  1792. translator->describe();
  1793. }
  1794. virtual const void *nextRow() override
  1795. {
  1796. if (eof)
  1797. return NULL;
  1798. const void *inRow = inputStream->nextRow();
  1799. if (!inRow)
  1800. {
  1801. if (eogSeen)
  1802. eof = true;
  1803. else
  1804. eogSeen = true;
  1805. return nullptr;
  1806. }
  1807. else
  1808. eogSeen = false;
  1809. RtlDynamicRowBuilder rowBuilder(resultAllocator);
  1810. size32_t len = translator->translate(rowBuilder, fieldCallback, (const byte *) inRow);
  1811. rtlReleaseRow(inRow);
  1812. return rowBuilder.finalizeRowClear(len);
  1813. }
  1814. virtual void stop() override
  1815. {
  1816. resultAllocator.clear();
  1817. }
  1818. bool canTranslate() const
  1819. {
  1820. return translator->canTranslate();
  1821. }
  1822. bool needsTranslate() const
  1823. {
  1824. return translator->needsTranslate();
  1825. }
  1826. UnexpectedVirtualFieldCallback fieldCallback; // I'm not sure if an non unexpected callback can be implemented
  1827. protected:
  1828. Linked<IRowStream> inputStream;
  1829. Linked<IEngineRowAllocator> resultAllocator;
  1830. Owned<const IDynamicTransform> translator;
  1831. unsigned numOffsets = 0;
  1832. size_t * variableOffsets = nullptr;
  1833. bool eof = false;
  1834. bool eogSeen = false;
  1835. };
  1836. extern ECLRTL_API IRowStream * transformRecord(IEngineRowAllocator * resultAllocator,IOutputMetaData & metaInput,IRowStream * input)
  1837. {
  1838. if (resultAllocator->queryOutputMeta()==&metaInput)
  1839. return LINK(input);
  1840. Owned<TranslatedRowStream> stream = new TranslatedRowStream(input, resultAllocator,
  1841. resultAllocator->queryOutputMeta()->queryRecordAccessor(true),
  1842. metaInput.queryRecordAccessor(true));
  1843. if (!stream->needsTranslate())
  1844. return LINK(input);
  1845. else if (!stream->canTranslate())
  1846. rtlFail(0, "Cannot translate record stream");
  1847. else
  1848. return stream.getClear();
  1849. }
  1850. // A key translator allows us to transform a RowFilter that refers to src to one that refers to dest.
  1851. // Basically just a map of those fields with matching types.
  1852. class CKeyTranslator : public CInterfaceOf<IKeyTranslator>
  1853. {
  1854. public:
  1855. CKeyTranslator(const RtlRecord &actual, const RtlRecord &expected)
  1856. {
  1857. translateNeeded = false;
  1858. for (unsigned expectedIdx = 0; expectedIdx < expected.getNumFields(); expectedIdx++)
  1859. {
  1860. unsigned actualIdx = actual.getFieldNum(expected.queryName(expectedIdx));
  1861. if (actualIdx != (unsigned) -1)
  1862. {
  1863. const RtlTypeInfo *expectedType = expected.queryType(expectedIdx);
  1864. const RtlTypeInfo *actualType = actual.queryType(actualIdx);
  1865. if (!actualType->equivalent(expectedType))
  1866. actualIdx = (unsigned) -2;
  1867. }
  1868. map.append(actualIdx);
  1869. if (actualIdx != expectedIdx)
  1870. translateNeeded = true;
  1871. }
  1872. }
  1873. virtual void describe() const override
  1874. {
  1875. ForEachItemIn(idx, map)
  1876. {
  1877. unsigned mapped = map.item(idx);
  1878. switch (mapped)
  1879. {
  1880. case (unsigned) -1: DBGLOG("No match for field %d", idx); break;
  1881. case (unsigned) -2: DBGLOG("Incompatible field match for field %d", idx); break;
  1882. default: DBGLOG("keyed field %d can map to field %d", idx, mapped); break;
  1883. }
  1884. }
  1885. }
  1886. virtual bool translate(RowFilter &filters) const override
  1887. {
  1888. bool mapNeeded = false;
  1889. if (translateNeeded)
  1890. {
  1891. unsigned numFields = filters.numFilterFields();
  1892. for (unsigned idx = 0; idx < numFields; idx++)
  1893. {
  1894. unsigned fieldNum = filters.queryFilter(idx).queryFieldIndex();
  1895. unsigned mappedFieldNum = map.isItem(fieldNum) ? map.item(fieldNum) : (unsigned) -1;
  1896. if (mappedFieldNum != fieldNum)
  1897. {
  1898. mapNeeded = true;
  1899. switch (mappedFieldNum)
  1900. {
  1901. case (unsigned) -1: throw makeStringExceptionV(0, "Cannot translate keyed filter on field %u - no matching field", idx);
  1902. case (unsigned) -2: throw makeStringExceptionV(0, "Cannot translate keyed filter on field %u - incompatible matching field type", idx);
  1903. default:
  1904. filters.remapField(idx, mappedFieldNum);
  1905. break;
  1906. }
  1907. }
  1908. }
  1909. if (mapNeeded)
  1910. filters.recalcFieldsRequired();
  1911. }
  1912. return mapNeeded;
  1913. }
  1914. virtual bool translate(RowFilter &filter, IConstArrayOf<IFieldFilter> &in) const override
  1915. {
  1916. bool mapNeeded = false;
  1917. if (translateNeeded)
  1918. {
  1919. unsigned numFields = in.length();
  1920. for (unsigned idx = 0; idx < numFields; idx++)
  1921. {
  1922. unsigned fieldNum = in.item(idx).queryFieldIndex();
  1923. unsigned mappedFieldNum = map.isItem(fieldNum) ? map.item(fieldNum) : (unsigned) -1;
  1924. if (mappedFieldNum != fieldNum)
  1925. {
  1926. mapNeeded = true;
  1927. switch (mappedFieldNum)
  1928. {
  1929. case (unsigned) -1: throw makeStringExceptionV(0, "Cannot translate keyed filter on field %u - no matching field", idx);
  1930. case (unsigned) -2: throw makeStringExceptionV(0, "Cannot translate keyed filter on field %u - incompatible matching field type", idx);
  1931. default:
  1932. filter.addFilter(*in.item(idx).remap(mappedFieldNum));
  1933. break;
  1934. }
  1935. }
  1936. else
  1937. filter.addFilter(OLINK(in.item(idx)));
  1938. }
  1939. }
  1940. return mapNeeded;
  1941. }
  1942. virtual bool needsTranslate() const
  1943. {
  1944. return translateNeeded;
  1945. }
  1946. protected:
  1947. UnsignedArray map;
  1948. bool translateNeeded = false;
  1949. };
  1950. extern ECLRTL_API const IKeyTranslator *createKeyTranslator(const RtlRecord &_destRecInfo, const RtlRecord &_srcRecInfo)
  1951. {
  1952. return new CKeyTranslator(_destRecInfo, _srcRecInfo);
  1953. }
  1954. //---------------------------------------------------------------------------------------------------------------------
  1955. const char * NullVirtualFieldCallback::queryLogicalFilename(const void * row)
  1956. {
  1957. return "";
  1958. }
  1959. unsigned __int64 NullVirtualFieldCallback::getFilePosition(const void * row)
  1960. {
  1961. return 0;
  1962. }
  1963. unsigned __int64 NullVirtualFieldCallback::getLocalFilePosition(const void * row)
  1964. {
  1965. return 0;
  1966. }
  1967. const byte * NullVirtualFieldCallback::lookupBlob(unsigned __int64 id)
  1968. {
  1969. return nullptr;
  1970. }
  1971. const char * UnexpectedVirtualFieldCallback::queryLogicalFilename(const void * row)
  1972. {
  1973. throwUnexpectedX("VIRTUAL(LOGICALFILENAME)");
  1974. }
  1975. unsigned __int64 UnexpectedVirtualFieldCallback::getFilePosition(const void * row)
  1976. {
  1977. throwUnexpectedX("VIRTUAL(FILEPOSITION)");
  1978. }
  1979. unsigned __int64 UnexpectedVirtualFieldCallback::getLocalFilePosition(const void * row)
  1980. {
  1981. throwUnexpectedX("VIRTUAL(LOCALFILEPOSITION)");
  1982. }
  1983. const byte * UnexpectedVirtualFieldCallback::lookupBlob(unsigned __int64 id)
  1984. {
  1985. throwUnexpectedX("BLOB");
  1986. }
  1987. unsigned __int64 FetchVirtualFieldCallback::getFilePosition(const void * row)
  1988. {
  1989. return filepos;
  1990. }
  1991. const char * LocalVirtualFieldCallback::queryLogicalFilename(const void * row)
  1992. {
  1993. return filename;
  1994. }
  1995. unsigned __int64 LocalVirtualFieldCallback::getFilePosition(const void * row)
  1996. {
  1997. return filepos;
  1998. }
  1999. unsigned __int64 LocalVirtualFieldCallback::getLocalFilePosition(const void * row)
  2000. {
  2001. return localfilepos;
  2002. }
  2003. const byte * LocalVirtualFieldCallback::lookupBlob(unsigned __int64 id)
  2004. {
  2005. throwUnexpectedX("BLOB");
  2006. }