123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160 |
- /*##############################################################################
- HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- ############################################################################## */
- #include "platform.h"
- #include <math.h>
- #include <stdio.h>
- #include "jmisc.hpp"
- #include "jlib.hpp"
- #include "eclhelper.hpp"
- #include "eclrtl_imp.hpp"
- #include "rtldynfield.hpp"
- #include "rtlrecord.hpp"
- #include "rtlembed.hpp"
- #include "rtlnewkey.hpp"
- //#define TRACE_TRANSLATION
- #define VALIDATE_TYPEINFO_HASHES
- #define RTLTYPEINFO_FORMAT_1 81 // In case we ever want to support more than one format or change how it is stored
- //---------------------------------------------------------------------------------------------------------------------
- extern ECLRTL_API RecordTranslationMode getTranslationMode(const char *val, bool isLocal)
- {
- if (isEmptyString(val) || strToBool(val) || strieq(val, "payload"))
- return RecordTranslationMode::Payload;
- else if (strieq(val, "alwaysDisk") || strieq(val, "disk"))
- {
- if (!isLocal)
- WARNLOG("alwaysDisk translation mode should only ever be used via a HINT");
- return RecordTranslationMode::AlwaysDisk;
- }
- else if (strieq(val, "alwaysECL") || strieq(val, "ecl"))
- {
- if (isLocal)
- return RecordTranslationMode::AlwaysECL;
- WARNLOG("Unsupported alwaysECL translation mode used globally, translation disabled - use with HINT to set locally.");
- }
- return RecordTranslationMode::None;
- }
- extern ECLRTL_API const char *getTranslationModeText(RecordTranslationMode val)
- {
- switch (val)
- {
- case RecordTranslationMode::AlwaysDisk: return "alwaysDisk";
- case RecordTranslationMode::AlwaysECL: return "alwaysECL";
- case RecordTranslationMode::Payload: return "payload";
- case RecordTranslationMode::None: return "off";
- }
- throwUnexpected();
- }
- //---------------------------------------------------------------------------------------------------------------------
- const RtlTypeInfo *FieldTypeInfoStruct::createRtlTypeInfo() const
- {
- const RtlTypeInfo *ret = nullptr;
- switch (fieldType & RFTMkind)
- {
- case type_boolean:
- ret = new RtlBoolTypeInfo(fieldType, length);
- break;
- case type_keyedint:
- ret = new RtlKeyedIntTypeInfo(fieldType, length, childType);
- break;
- case type_int:
- ret = new RtlIntTypeInfo(fieldType, length);
- break;
- case type_blob:
- ret = new RtlBlobTypeInfo(fieldType, length, childType);
- break;
- case type_filepos:
- #if __BYTE_ORDER == __LITTLE_ENDIAN
- ret = new RtlSwapIntTypeInfo(fieldType, length);
- #else
- ret = new RtlIntTypeInfo(fieldType, length);
- #endif
- break;
- case type_real:
- ret = new RtlRealTypeInfo(fieldType, length);
- break;
- case type_decimal:
- ret = new RtlDecimalTypeInfo(fieldType, length);
- break;
- case type_string:
- ret = new RtlStringTypeInfo(fieldType, length);
- break;
- case type_bitfield:
- ret = new RtlBitfieldTypeInfo(fieldType, length);
- break;
- case type_varstring:
- ret = new RtlVarStringTypeInfo(fieldType, length);
- break;
- case type_data:
- ret = new RtlDataTypeInfo(fieldType, length);
- break;
- case type_table:
- assert(childType);
- ret = new RtlDatasetTypeInfo(fieldType, length, childType);
- break;
- case type_dictionary:
- assert(childType);
- ret = new RtlDictionaryTypeInfo(fieldType, length, childType);
- break;
- case type_set:
- assert(childType);
- ret = new RtlSetTypeInfo(fieldType, length, childType);
- break;
- case type_row:
- assert(childType);
- ret = new RtlRowTypeInfo(fieldType, length, childType);
- break;
- case type_swapint:
- ret = new RtlSwapIntTypeInfo(fieldType, length);
- break;
- case type_packedint:
- ret = new RtlPackedIntTypeInfo(fieldType, length);
- break;
- case type_qstring:
- ret = new RtlQStringTypeInfo(fieldType, length);
- break;
- case type_unicode:
- ret = new RtlUnicodeTypeInfo(fieldType, length, locale);
- break;
- case type_varunicode:
- ret = new RtlVarUnicodeTypeInfo(fieldType, length, locale);
- break;
- case type_utf8:
- ret = new RtlUtf8TypeInfo(fieldType, length, locale);
- break;
- case type_record:
- ret = new RtlRecordTypeInfo(fieldType, length, fieldsArray);
- break;
- case type_ifblock:
- ret = new RtlDynamicIfBlockTypeInfo(fieldType, length, fieldsArray, nullptr, filter);
- break;
- case type_alien:
- assert(childType);
- ret = new RtlAlienTypeInfo(fieldType, length, childType);
- break;
- default:
- throwUnexpected();
- }
- return ret;
- };
- typedef MapBetween<const RtlTypeInfo *, const RtlTypeInfo *, StringAttr, const char *> TypeNameMap;
- typedef MapBetween<const RtlTypeInfo *, const RtlTypeInfo *, unsigned, unsigned> TypeNumMap;
- /**
- * class CRtlFieldTypeSerializer
- *
- * Serializer class for creating json representation of a RtlTypeInfo structure.
- *
- */
- class CRtlFieldTypeSerializer
- {
- public:
- /**
- * Serialize a RtlTypeInfo structure to JSON
- *
- * @param out Buffer for resulting serialized string
- * @param type RtlTypeInfo structure to be serialized
- * @return Referenced to supplied buffer
- */
- static StringBuffer &serialize(StringBuffer &out, const RtlTypeInfo *type)
- {
- CRtlFieldTypeSerializer s(out, type);
- s.doSerialize();
- return out;
- }
- private:
- CRtlFieldTypeSerializer(StringBuffer &_out, const RtlTypeInfo *_base)
- : json(_out), base(_base)
- {
- }
- void doSerialize()
- {
- json.append("{");
- serializeType(base);
- json.append("\n}");
- }
- void serializeType(const RtlTypeInfo *type)
- {
- if (!serialized(type))
- {
- // Make sure all child types are serialized first
- const RtlTypeInfo *childType = type->queryChildType();
- if (childType)
- serializeType(childType);
- const RtlFieldInfo * const * fields = type->queryFields();
- if (fields)
- {
- for (;;)
- {
- const RtlFieldInfo * child = *fields;
- if (!child)
- break;
- serializeType(child->type);
- fields++;
- }
- }
- // Now serialize this one
- if (type != base)
- {
- VStringBuffer newName("ty%d", ++nextTypeName);
- types.setValue(type, newName.str());
- startField(newName.str());
- serializeMe(type);
- closeCurly();
- }
- else
- serializeMe(type);
- }
- }
- void serializeMe(const RtlTypeInfo *type)
- {
- if (!type->canSerialize())
- throw makeStringException(MSGAUD_user, 1000, "This type structure cannot be serialized");
- addPropHex("fieldType", type->fieldType);
- addProp("length", type->length);
- addPropNonEmpty("locale", type->queryLocale());
- const RtlTypeInfo *childType = type->queryChildType();
- if (childType)
- addPropType("child", childType);
- const IFieldFilter * filter = type->queryFilter();
- if (filter)
- {
- StringBuffer filterText;
- filter->serialize(filterText);
- addPropType("filterType", &filter->queryType());
- addProp("filter", filterText);
- }
- const RtlFieldInfo * const * fields = type->queryFields();
- if (fields)
- {
- startFields();
- for (;;)
- {
- const RtlFieldInfo * child = *fields;
- if (!child)
- break;
- newline();
- openCurly();
- addProp("name", child->name);
- addPropType("type", child->type);
- addProp("xpath", child->xpath);
- if (child->flags)
- addPropHex("flags", child->flags);
- // initializer is tricky - it's not (in general) a null-terminated string but the actual length is not easily available
- if (child->initializer)
- {
- if (isVirtualInitializer(child->initializer))
- addProp("vinit", getVirtualInitializer(child->initializer));
- else
- addProp("init", child->type->size((const byte *) child->initializer, nullptr), (const byte *) child->initializer);
- }
- closeCurly();
- fields++;
- }
- endFields();
- }
- }
- bool serialized(const RtlTypeInfo *type)
- {
- return types.find(type) != nullptr;
- }
- void startField(const char *name)
- {
- newline().appendf("\"%s\": ", name);
- openCurly();
- }
- void addProp(const char *propName, const char *propVal)
- {
- if (propVal)
- {
- newline();
- encodeJSON(json.append("\""), propName).append("\": ");
- encodeJSON(json.append("\""), propVal).append("\"");
- }
- }
- void addProp(const char *propName, size32_t propLen, const byte *propVal)
- {
- if (propVal)
- {
- newline();
- encodeJSON(json.append("\""), propName).append("\": \"");
- JBASE64_Encode(propVal, propLen, json, false);
- json.append("\"");
- }
- }
- void addPropNonEmpty(const char *propName, const char *propVal)
- {
- if (propVal && *propVal)
- addProp(propName, propVal);
- }
- void addProp(const char *propName, unsigned propVal)
- {
- newline().appendf("\"%s\": %u", propName, propVal);
- }
- void addPropHex(const char *propName, unsigned propVal)
- {
- newline().appendf("\"%s\": %u", propName, propVal); // Nice idea but json does not support hex constants :(
- }
- void addPropType(const char *propName, const RtlTypeInfo *type)
- {
- addProp(propName, queryTypeName(type));
- }
- const char *queryTypeName(const RtlTypeInfo *type)
- {
- StringAttr *typeName = types.getValue(type);
- assertex(typeName);
- return typeName->get();
- }
- void startFields()
- {
- newline().appendf("\"fields\": ");
- openCurly('[');
- }
- void endFields()
- {
- closeCurly(']');
- }
- StringBuffer &newline()
- {
- if (commaPending)
- json.append(',');
- json.appendf("\n%*s", indent, "");
- commaPending = true;
- return json;
- }
- void closeCurly(char brace = '}')
- {
- indent--;
- json.appendf("\n%*s%c", indent, "", brace);
- commaPending = true;
- }
- void openCurly(char brace = '{')
- {
- json.append(brace);
- indent++;
- commaPending = false;
- }
- TypeNameMap types;
- StringBuffer &json;
- const RtlTypeInfo *base = nullptr;
- unsigned indent = 1;
- unsigned nextTypeName = 0;
- bool commaPending = false;
- };
- class CRtlFieldTypeBinSerializer
- {
- public:
- /**
- * Serialize a RtlTypeInfo structure to binary
- *
- * @param out Buffer for resulting serialized string
- * @param type RtlTypeInfo structure to be serialized
- * @return Referenced to supplied buffer
- */
- static MemoryBuffer &serialize(MemoryBuffer &out, const RtlTypeInfo *type)
- {
- int oldEnd = out.setEndian(__LITTLE_ENDIAN);
- CRtlFieldTypeBinSerializer s(out);
- byte format = RTLTYPEINFO_FORMAT_1;
- out.append(format);
- DelayedMarker<hash64_t> hash(out);
- DelayedSizeMarker size(out);
- size32_t pos = out.length();
- s.serializeType(type);
- size.write();
- hash.write(rtlHash64Data(size.size(), out.toByteArray()+pos, 0));
- out.setEndian(oldEnd);
- return out;
- }
- private:
- CRtlFieldTypeBinSerializer(MemoryBuffer &_out)
- : out(_out)
- {
- }
- void serializeType(const RtlTypeInfo *type)
- {
- if (!serialized(type))
- {
- // Make sure all child types are serialized first
- const RtlTypeInfo *child = type->queryChildType();
- if (child)
- serializeType(child);
- const RtlFieldInfo * const * fields = type->queryFields();
- if (fields)
- {
- for (unsigned idx = 0;;idx++)
- {
- const RtlFieldInfo * child = fields[idx];
- if (!child)
- break;
- serializeType(child->type);
- }
- }
- // Now serialize this one
- types.setValue(type, nextTypeNum++);
- serializeMe(type);
- }
- }
-
- void serializeMe(const RtlTypeInfo *type)
- {
- if (!type->canSerialize())
- throw makeStringException(MSGAUD_user, 1000, "This type structure cannot be serialized");
- unsigned fieldType = type->fieldType;
- const char *locale = type->queryLocale();
- if (locale && *locale)
- fieldType |= RFTMhasLocale;
- const RtlTypeInfo *child = type->queryChildType();
- if (child)
- fieldType |= RFTMhasChildType;
- const RtlFieldInfo * const * fields = type->queryFields();
- if (fields)
- fieldType |= RFTMhasFields;
- out.append(fieldType);
- out.appendPacked(type->length);
- if (fieldType & RFTMhasLocale)
- out.append(locale);
- if (child)
- out.appendPacked(queryTypeIdx(child));
- const IFieldFilter * filter = type->queryFilter();
- if (filter)
- {
- out.appendPacked(queryTypeIdx(&filter->queryType()));
- filter->serialize(out);
- }
- if (fields)
- {
- unsigned count = countFields(fields);
- out.appendPacked(count);
- for (;;)
- {
- const RtlFieldInfo * child = *fields;
- if (!child)
- break;
- out.append(child->name);
- out.appendPacked(queryTypeIdx(child->type));
- unsigned flags = child->flags;
- if (child->xpath)
- flags |= RFTMhasXpath;
- if (child->initializer)
- {
- if (isVirtualInitializer(child->initializer))
- flags |= RFTMhasVirtualInitializer;
- else
- flags |= RFTMhasInitializer;
- }
- out.append(flags);
- if (child->xpath)
- out.append(child->xpath);
- // initializer is tricky - it's not (in general) a null-terminated string but the actual length is not easily available
- if (flags & RFTMhasInitializer)
- {
- unsigned initLength = child->type->size((const byte *) child->initializer, nullptr);
- out.appendPacked(initLength).append(initLength, (const byte *) child->initializer);
- }
- else if (flags &RFTMhasVirtualInitializer)
- out.append(getVirtualInitializer(child->initializer));
- fields++;
- }
- }
- }
- bool serialized(const RtlTypeInfo *type)
- {
- return types.find(type) != nullptr;
- }
- unsigned queryTypeIdx(const RtlTypeInfo *type)
- {
- unsigned *typeNum = types.getValue(type);
- assertex(typeNum);
- return *typeNum;
- }
- TypeNumMap types;
- MemoryBuffer &out;
- unsigned nextTypeNum = 0;
- };
- /**
- * class CRtlFieldTypeDeserializer
- *
- * Deserializer class for creating a RtlTypeInfo structure from json representation.
- *
- * Note that the resulting RtlTypeInfo structures are owned by this object and will be
- * destroyed when this object is destroyed.
- *
- */
- class CRtlFieldTypeDeserializer : public CInterfaceOf<IRtlFieldTypeDeserializer>
- {
- public:
- /**
- * CRtlFieldTypeDeserializer constructor
- *
- * @param _callback Supplies a callback to be used for blobs/filepositions.
- */
- CRtlFieldTypeDeserializer()
- {
- }
- /**
- * CRtlFieldTypeDeserializer destructor
- * <p>
- * Releases all RtlTypeInfo and related structures created by this deserializer
- */
- ~CRtlFieldTypeDeserializer()
- {
- // Need some care - all the RtlTypeInfo objects I created need to be destroyed, together with anything else I had to create
- // Strings (other than the init strings) are preserved in the AtomTable
- // First allow the types to clean up any critical cached information, then delete them in a second pass
- HashIterator allTypes(types);
- ForEach(allTypes)
- {
- const RtlTypeInfo **type = types.mapToValue(&allTypes.query());
- cleanupType(*type);
- }
- cleanupType(base);
- ForEach(allTypes)
- {
- const RtlTypeInfo **type = types.mapToValue(&allTypes.query());
- deleteType(*type);
- }
- deleteType(base);
- }
- /**
- * Obtain the deserialized type information
- * <p>
- * Note that the RtlTypeInfo objects are not link-counted, so the lifetime of these objects
- * is determined by the lifetime of the deserializer. They will be released once the deserializer
- * that created them is deleted.
- * <p>
- * Do not call more than once.
- *
- * @param _json JSON text to be deserialized, as created by CRtlFieldTypeSerializer
- * @return Deserialized type object
- */
- virtual const RtlTypeInfo *deserialize(const char *json) override
- {
- assertex(!base);
- Owned<IPropertyTree> jsonTree = createPTreeFromJSONString(json);
- base = deserializeType(jsonTree, jsonTree);
- return base;
- }
- /**
- * Obtain the deserialized type information
- * <p>
- * Note that the RtlTypeInfo objects are not link-counted, so the lifetime of these objects
- * is determined by the lifetime of the deserializer. They will be released once the deserializer
- * that created them is deleted.
- * <p>
- * Do not call more than once.
- *
- * @param _jsonTree JSON property tree to be deserialized, as created by CRtlFieldTypeSerializer
- * @return Deserialized type object
- */
- virtual const RtlTypeInfo *deserialize(IPropertyTree &jsonTree) override
- {
- assertex(!base);
- base = deserializeType(&jsonTree, &jsonTree);
- return base;
- }
- /**
- * Obtain the deserialized type information
- * <p>
- * Note that the RtlTypeInfo objects are not link-counted, so the lifetime of these objects
- * is determined by the lifetime of the deserializer. They will be released once the deserializer
- * that created them is deleted.
- * <p>
- * Do not call more than once.
- *
- * @param buf Binary serialized typeinfo to be deserialized, as created by CRtlFieldTypeSerializer
- * @return Deserialized type object
- */
- virtual const RtlTypeInfo *deserialize(MemoryBuffer &buf) override
- {
- assertex(!base);
- unsigned nextTypeNum = 0;
- int oldEndian = buf.setEndian(__LITTLE_ENDIAN);
- try
- {
- byte format;
- buf.read(format);
- if (format != RTLTYPEINFO_FORMAT_1)
- throw MakeStringException(0, "Invalid type info (%d) in CRtlFieldTypeDeserializer::deserialize", format);
- hash64_t hash;
- buf.read(hash);
- size32_t size;
- buf.read(size);
- #ifdef VALIDATE_TYPEINFO_HASHES
- hash64_t expected = rtlHash64Data(size, buf.readDirect(0), 0);
- if (expected != hash)
- throw MakeStringException(0, "Invalid type info hash in CRtlFieldTypeDeserializer::deserialize");
- #endif
- size32_t endpos = buf.getPos() + size;
- while (buf.getPos() < endpos)
- {
- if (base)
- {
- addType(base, nextTypeNum++);
- base = nullptr; // in case of exceptions...
- }
- base = deserializeType(buf);
- }
- if (buf.getPos()!=endpos)
- throw MakeStringException(0, "Invalid type info (incorrect size data) in CRtlFieldTypeDeserializer::deserialize");
- buf.setEndian(oldEndian);
- return base;
- }
- catch(...)
- {
- buf.setEndian(oldEndian);
- throw;
- }
- }
- virtual const RtlTypeInfo *addType(FieldTypeInfoStruct &info, const IInterface *typeOrIfblock) override
- {
- VStringBuffer name("%p", typeOrIfblock);
- const RtlTypeInfo ** found = types.getValue(name);
- if (found)
- return *found;
- savedTypes.append(LINK(typeOrIfblock));
- info.locale = keep(info.locale);
- const RtlTypeInfo * ret = info.createRtlTypeInfo();
- types.setValue(name, ret);
- unsigned baseType = (info.fieldType & RFTMkind);
- if (baseType == type_record)
- patchIfBlockParentRow(ret, static_cast<const RtlRecordTypeInfo *>(ret));
- return ret;
- }
- virtual const RtlTypeInfo *lookupType(const IInterface * typeOrIfBlock) const override
- {
- VStringBuffer name("%p", typeOrIfBlock);
- const RtlTypeInfo ** found = types.getValue(name);
- if (found)
- return *found;
- return nullptr;
- }
- virtual const RtlFieldInfo *addFieldInfo(const char *fieldName, const char *xpath, const RtlTypeInfo *type, unsigned flags, const char *init) override
- {
- // MORE - we could hang onto this for cleanup, rather than assuming that we keep it via a later addType() call?
- return new RtlFieldStrInfo(keep(fieldName), keep(xpath), type, flags, init);
- }
- private:
- KeptAtomTable atoms; // Used to ensure proper lifetime of strings used in type structures
- MapStringTo<const RtlTypeInfo *> types; // Ensures structures only generated once
- const RtlTypeInfo *base = nullptr; // Holds the resulting type
- IConstPointerArray savedTypes; // ensure types remain alive for subsequent lookups
- void deleteType(const RtlTypeInfo *type)
- {
- if (type)
- {
- // Releases all memory for a single RtlTypeInfo object
- const RtlFieldInfo * const * fields = type->queryFields();
- if (fields)
- {
- const RtlFieldInfo * const * cur = fields;
- for (;;)
- {
- const RtlFieldInfo * child = *cur;
- if (!child)
- break;
- // We don't need to delete other strings - they are owned by atom table.
- // But the initializer is decoded and thus owned by me
- if (!isVirtualInitializer(child->initializer))
- free((void *)child->initializer);
- delete child;
- cur++;
- }
- delete [] fields;
- }
- type->doDelete();
- }
- }
- void cleanupType(const RtlTypeInfo *type)
- {
- if (type)
- type->doCleanup();
- }
- const RtlTypeInfo *lookupType(const char *name, IPropertyTree *all)
- {
- const RtlTypeInfo ** found = types.getValue(name);
- if (found)
- return *found;
- const RtlTypeInfo *type = deserializeType(all->queryPropTree(name), all);
- types.setValue(name, type);
- return type;
- }
- const RtlTypeInfo *lookupType(unsigned idx)
- {
- // Could keep an expanding array of types instead - but the hash table is already there for json support...
- VStringBuffer key("%u", idx);
- const RtlTypeInfo ** found = types.getValue(key);
- if (found)
- return *found;
- throw makeStringException(-1, "Invalid serialized type information");
- }
- void addType(const RtlTypeInfo *type, unsigned idx)
- {
- VStringBuffer key("%u", idx);
- assert(types.getValue(key)==nullptr);
- types.setValue(key, type);
- }
- const char *keep(const char *string)
- {
- if (string)
- return str(atoms.addAtom(string));
- else
- return nullptr;
- }
- const RtlTypeInfo *deserializeType(IPropertyTree *type, IPropertyTree *all)
- {
- FieldTypeInfoStruct info;
- info.fieldType = type->getPropInt("fieldType");
- info.length = type->getPropInt("length");
- info.locale = keep(type->queryProp("locale"));
- const char *child = type->queryProp("child");
- if (child)
- info.childType = lookupType(child, all);
- unsigned baseType = (info.fieldType & RFTMkind);
- if ((baseType == type_record) || (baseType == type_ifblock))
- {
- unsigned numFields = type->getCount("fields");
- info.fieldsArray = new const RtlFieldInfo * [numFields+1];
- info.fieldsArray[numFields] = nullptr;
- Owned<IPropertyTreeIterator> fields = type->getElements("fields");
- unsigned n = 0;
- ForEach(*fields)
- {
- IPropertyTree &field = fields->query();
- const char *fieldTypeName = field.queryProp("type");
- const char *fieldName = keep(field.queryProp("name"));
- const char *fieldXpath = keep(field.queryProp("xpath"));
- unsigned flags = field.getPropInt("flags");
- const char *fieldInit = field.queryProp("init");
- const char *fieldVInit = field.queryProp("vinit");
- if (fieldInit)
- {
- StringBuffer decoded;
- JBASE64_Decode(fieldInit, decoded);
- fieldInit = decoded.detach(); // NOTE - this gets freed in cleanupType()
- }
- else if (fieldVInit)
- {
- fieldInit = (const char *)(memsize_t)atoi(fieldVInit);
- }
- info.fieldsArray[n] = new RtlFieldStrInfo(fieldName, fieldXpath, lookupType(fieldTypeName, all), flags, fieldInit);
- n++;
- }
- }
- if (baseType == type_ifblock)
- {
- //Filter field needs to be deserialized and the type resolved separately outside the deserialize call
- //because there isn't a RtlTypeInfo available to resolve the field (since we are currently deserializing it!)
- const char * filterText = type->queryProp("filter");
- StringBuffer fieldIdText;
- readFieldFromFieldFilter(fieldIdText, filterText);
- unsigned fieldId = atoi(fieldIdText);
- const RtlTypeInfo * fieldType = lookupType(type->queryProp("filterType"), all);
- info.filter = deserializeFieldFilter(fieldId, *fieldType, filterText);
- }
- const RtlTypeInfo * result = info.createRtlTypeInfo();
- if (baseType == type_record)
- patchIfBlockParentRow(result, static_cast<const RtlRecordTypeInfo *>(result));
- return result;
- }
- const RtlTypeInfo *deserializeType(MemoryBuffer &type)
- {
- FieldTypeInfoStruct info;
- type.read(info.fieldType);
- type.readPacked(info.length);
- if (info.fieldType & RFTMhasLocale)
- {
- const char *locale;
- type.read(locale);
- info.locale = keep(locale);
- }
- if (info.fieldType & RFTMhasChildType)
- {
- unsigned childIdx;
- type.readPacked(childIdx);
- info.childType = lookupType(childIdx);
- }
- unsigned baseType = (info.fieldType & RFTMkind);
- if (baseType == type_ifblock)
- {
- unsigned childIdx;
- type.readPacked(childIdx);
- const RtlTypeInfo * fieldType = lookupType(childIdx);
- unsigned fieldId;
- type.readPacked(fieldId);
- info.filter = deserializeFieldFilter(fieldId, *fieldType, type);
- }
- if (info.fieldType & RFTMhasFields)
- {
- unsigned numFields;
- type.readPacked(numFields);
- info.fieldsArray = new const RtlFieldInfo * [numFields+1];
- info.fieldsArray[numFields] = nullptr;
- for (unsigned n = 0; n < numFields; n++)
- {
- const char *fieldName;
- type.read(fieldName);
- if (fieldName[0] == '\0')
- fieldName = nullptr;
- unsigned fieldType;
- type.readPacked(fieldType);
- unsigned fieldFlags;
- type.read(fieldFlags);
- const char *xpath = nullptr;
- if (fieldFlags & RFTMhasXpath)
- type.read(xpath);
- void *init = nullptr;
- if (fieldFlags & RFTMhasInitializer)
- {
- unsigned initLength;
- type.readPacked(initLength);
- init = malloc(initLength);
- memcpy(init, type.readDirect(initLength), initLength);
- }
- else if (fieldFlags & RFTMhasVirtualInitializer)
- {
- byte virtualKind;
- type.read(virtualKind);
- init = (void *)(memsize_t)virtualKind;
- }
- fieldFlags &= ~RFTMserializerFlags;
- info.fieldsArray[n] = new RtlFieldStrInfo(keep(fieldName), keep(xpath), lookupType(fieldType), fieldFlags, (const char *) init);
- }
- }
- info.fieldType &= ~RFTMserializerFlags;
- const RtlTypeInfo * result = info.createRtlTypeInfo();
- if (baseType == type_record)
- patchIfBlockParentRow(result, static_cast<const RtlRecordTypeInfo *>(result));
- return result;
- }
- void patchIfBlockParentRow(const RtlTypeInfo * fieldType, const RtlRecordTypeInfo * parentRow)
- {
- const RtlFieldInfo * const * fields = fieldType->queryFields();
- for (;*fields;fields++)
- {
- const RtlFieldInfo * cur = *fields;
- if (!cur)
- break;
- const RtlTypeInfo * curType = cur->type;
- if ((curType->fieldType & RFTMkind) == type_ifblock)
- {
- const RtlDynamicIfBlockTypeInfo * constifblock = static_cast<const RtlDynamicIfBlockTypeInfo *>(curType);
- RtlDynamicIfBlockTypeInfo * ifblock = const_cast<RtlDynamicIfBlockTypeInfo *>(constifblock);
- ifblock->setParent(parentRow);
- patchIfBlockParentRow(curType, parentRow);
- }
- }
- }
- };
- extern ECLRTL_API IRtlFieldTypeDeserializer *createRtlFieldTypeDeserializer()
- {
- return new CRtlFieldTypeDeserializer();
- }
- extern ECLRTL_API StringBuffer &dumpTypeInfo(StringBuffer &ret, const RtlTypeInfo *t)
- {
- return CRtlFieldTypeSerializer::serialize(ret, t);
- }
- extern ECLRTL_API bool dumpTypeInfo(MemoryBuffer &ret, const RtlTypeInfo *t)
- {
- try
- {
- CRtlFieldTypeBinSerializer::serialize(ret, t);
- return true;
- }
- catch (IException *E)
- {
- EXCLOG(E);
- E->Release();
- return false;
- }
- }
- extern ECLRTL_API void serializeRecordType(size32_t & __lenResult, void * & __result, IOutputMetaData & metaVal)
- {
- MemoryBuffer ret;
- try
- {
- CRtlFieldTypeBinSerializer::serialize(ret, metaVal.queryTypeInfo());
- }
- catch (IException * e)
- {
- ret.clear();
- e->Release();
- }
- __lenResult = ret.length();
- __result = ret.detach();
- }
- extern ECLRTL_API void dumpRecordType(size32_t & __lenResult,char * & __result,IOutputMetaData &metaVal)
- {
- StringBuffer ret;
- try
- {
- CRtlFieldTypeSerializer::serialize(ret, metaVal.queryTypeInfo());
- #ifdef _DEBUG
- StringBuffer ret2;
- CRtlFieldTypeDeserializer deserializer;
- CRtlFieldTypeSerializer::serialize(ret2, deserializer.deserialize(ret));
- assert(streq(ret, ret2));
- MemoryBuffer out;
- CRtlFieldTypeBinSerializer::serialize(out, metaVal.queryTypeInfo());
- CRtlFieldTypeDeserializer bindeserializer;
- CRtlFieldTypeSerializer::serialize(ret2.clear(), bindeserializer.deserialize(out));
- assert(streq(ret, ret2));
- #endif
- }
- catch (IException * e)
- {
- e->errorMessage(ret.clear());
- e->Release();
- }
- __lenResult = ret.length();
- __result = ret.detach();
- }
- extern ECLRTL_API void getFieldVal(size32_t & __lenResult,char * & __result, int column, IOutputMetaData & metaVal, const byte *row)
- {
- __lenResult = 0;
- __result = nullptr;
- if (column >= 0)
- {
- const RtlRecord &r = metaVal.queryRecordAccessor(true);
- if ((unsigned) column < r.getNumFields())
- {
- unsigned numOffsets = r.getNumVarFields() + 1;
- size_t * variableOffsets = (size_t *)alloca(numOffsets * sizeof(size_t));
- RtlRow offsetCalculator(r, row, numOffsets, variableOffsets);
- offsetCalculator.getUtf8(__lenResult, __result, column);
- }
- }
- }
- extern ECLRTL_API int getFieldNum(const char *fieldName, IOutputMetaData & metaVal)
- {
- const RtlRecord r = metaVal.queryRecordAccessor(true);
- return r.getFieldNum(fieldName);
- }
- enum FieldMatchType {
- // On a field, exactly one of the below is set, but translator returns a bitmap indicating
- // which were required (and we can restrict translation to allow some types but not others)
- match_perfect = 0x00, // exact type match - use memcpy
- match_link = 0x01, // copy a nested dataset by linking
- match_move = 0x02, // at least one field has moved (set on translator)
- match_remove = 0x04, // at least one field has been removed (set on translator)
- match_truncate = 0x08, // dest is truncated copy of source - use memcpy
- match_extend = 0x10, // dest is padded version of source - use memcpy and memset
- match_typecast = 0x20, // type has changed - cast required
- match_none = 0x40, // No matching field in source - use null value
- match_recurse = 0x80, // Use recursive translator for child records/datasets
- match_fail = 0x100, // no translation possible
- match_keychange = 0x200, // at least one affected field not marked as payload (set on translator)
- match_virtual = 0x800, // at least one affected field is a virtual field (set on translator)
- // This flag may be set in conjunction with the others
- match_inifblock = 0x400, // matching to a field in an ifblock - may not be present
- match_deblob = 0x1000, // source needs fetching from a blob prior to translation
- match_dynamic = 0x2000, // source needs fetching from dynamic source (callback)
- match_filepos = 0x4000, // type moving in or out of filepos field - cast required
- };
- StringBuffer &describeFlags(StringBuffer &out, FieldMatchType flags)
- {
- if (flags == match_perfect)
- return out.append("perfect");
- unsigned origlen = out.length();
- if (flags & match_link) out.append("|link");
- if (flags & match_move) out.append("|move");
- if (flags & match_remove) out.append("|remove");
- if (flags & match_truncate) out.append("|truncate");
- if (flags & match_extend) out.append("|extend");
- if (flags & match_typecast) out.append("|typecast");
- if (flags & match_none) out.append("|none");
- if (flags & match_recurse) out.append("|recurse");
- if (flags & match_inifblock) out.append("|ifblock");
- if (flags & match_keychange) out.append("|keychange");
- if (flags & match_fail) out.append("|fail");
- if (flags & match_virtual) out.append("|virtual");
- if (flags & match_deblob) out.append("|blob");
- if (flags & match_dynamic) out.append("|dynamic");
- if (flags & match_filepos) out.append("|filepos");
- assertex(out.length() > origlen);
- return out.remove(origlen, 1);
- }
- inline constexpr FieldMatchType operator|(FieldMatchType a, FieldMatchType b) { return (FieldMatchType)((int)a | (int)b); }
- inline FieldMatchType &operator|=(FieldMatchType &a, FieldMatchType b) { return (FieldMatchType &) ((int &)a |= (int)b); }
- class GeneralRecordTranslator : public CInterfaceOf<IDynamicTransform>
- {
- public:
- GeneralRecordTranslator(const RtlRecord &_destRecInfo, const RtlRecord &_srcRecInfo, bool _binarySource, type_vals _callbackRawType = type_any)
- : destRecInfo(_destRecInfo), sourceRecInfo(_srcRecInfo), binarySource(_binarySource), callbackRawType(_callbackRawType)
- {
- matchInfo = new MatchInfo[destRecInfo.getNumFields()];
- createMatchInfo();
- #ifdef _DEBUG
- //describe();
- #endif
- }
- ~GeneralRecordTranslator()
- {
- delete [] matchInfo;
- }
- // IDynamicTransform impl.
- virtual void describe() const override
- {
- doDescribe(0);
- }
- virtual size32_t translate(ARowBuilder &builder, IVirtualFieldCallback & callback, const byte *sourceRec) const override
- {
- assertex(binarySource);
- return doTranslate(builder, callback, 0, sourceRec);
- }
- virtual size32_t translate(ARowBuilder &builder, IVirtualFieldCallback & callback, const RtlRow &sourceRow) const override
- {
- assertex(binarySource);
- sourceRow.lazyCalcOffsets(-1); // MORE - could save the max one we actually need...
- return doTranslateOpaqueType(builder, callback, 0, &sourceRow);
- }
- virtual size32_t translate(ARowBuilder &builder, IVirtualFieldCallback & callback, const IDynamicFieldValueFetcher & fetcher) const override
- {
- assertex(!binarySource);
- return doTranslateOpaqueType(builder, callback, 0, &fetcher);
- }
- virtual bool canTranslate() const override
- {
- return (matchFlags & match_fail) == 0;
- }
- virtual bool needsTranslate() const override
- {
- return !binarySource || (matchFlags & ~(match_link|match_inifblock)) != 0;
- }
- virtual bool needsNonVirtualTranslate() const override
- {
- return (matchFlags & ~(match_link|match_virtual|match_keychange|match_inifblock)) != 0;
- }
- virtual bool keyedTranslated() const override
- {
- return (matchFlags & match_keychange) != 0;
- }
- private:
- void doDescribe(unsigned indent) const
- {
- unsigned perfect=0;
- unsigned reported=0;
- for (unsigned idx = 0; idx < destRecInfo.getNumFields(); idx++)
- {
- const char *source = destRecInfo.queryName(idx);
- const MatchInfo &match = matchInfo[idx];
- if (match.matchType == match_none)
- DBGLOG("%*sNo match for field %s - default value will be used", indent, "", source);
- else if (match.matchType == match_virtual)
- DBGLOG("%*sUse virtual value for field %s", indent, "", source);
- else
- {
- if (match.matchType != match_perfect)
- {
- reported++;
- StringBuffer matchStr;
- DBGLOG("%*sMatch (%s) to field %d for field %s (typecode %x)", indent, "", describeFlags(matchStr, match.matchType).str(), match.matchIdx, source, destRecInfo.queryType(idx)->fieldType);
- if (match.subTrans)
- match.subTrans->doDescribe(indent+2);
- }
- else
- perfect++;
- }
- }
- if (allUnmatched.ordinality())
- {
- VStringBuffer msg("%*sDropped field", indent, "");
- if (allUnmatched.ordinality()>1)
- msg.append('s');
- for (unsigned idx = 0; idx < allUnmatched.ordinality() && idx < 5; idx++)
- {
- if (idx)
- msg.append(',');
- msg.appendf(" %s", sourceRecInfo.queryName(allUnmatched.item(idx)));
- }
- if (allUnmatched.ordinality() > 5)
- msg.appendf(" and %u other fields", allUnmatched.ordinality() - 5);
- DBGLOG("%s", msg.str());
- }
- if (!canTranslate())
- DBGLOG("%*sTranslation is NOT possible", indent, "");
- else if (needsTranslate())
- {
- StringBuffer matchStr;
- if (perfect)
- DBGLOG("%u %sfield%s matched perfectly", perfect, reported ? "other " : "", perfect==1 ? "" : "s");
- DBGLOG("%*sTranslation is possible (%s)", indent, "", describeFlags(matchStr, matchFlags).str());
- }
- else
- DBGLOG("%*sTranslation is not necessary", indent, "");
- }
- size32_t doTranslate(ARowBuilder &builder, IVirtualFieldCallback & callback, size32_t offset, const byte *sourceRec) const
- {
- unsigned numOffsets = sourceRecInfo.getNumVarFields() + 1;
- size_t * variableOffsets = (size_t *)alloca(numOffsets * sizeof(size_t));
- RtlRow sourceRow(sourceRecInfo, sourceRec, numOffsets, variableOffsets); // MORE - could save the max source offset we actually need, and only set up that many...
- return doTranslateOpaqueType(builder, callback, offset, &sourceRow);
- }
- size32_t doTranslateOpaqueType(ARowBuilder &builder, IVirtualFieldCallback & callback, size32_t offset, const void *sourceRow) const
- {
- dbgassertex(canTranslate());
- byte * destConditions = (byte *)alloca(destRecInfo.getNumIfBlocks() * sizeof(byte));
- memset(destConditions, 2, destRecInfo.getNumIfBlocks() * sizeof(byte));
- size32_t estimate = destRecInfo.getFixedSize();
- bool hasBlobs = false;
- if (!estimate)
- {
- if (binarySource)
- estimate = estimateNewSize(*(const RtlRow *)sourceRow);
- else
- estimate = destRecInfo.getMinRecordSize();
- builder.ensureCapacity(offset+estimate, "record");
- }
- size32_t origOffset = offset;
- for (unsigned idx = 0; idx < destRecInfo.getNumFields(); idx++)
- {
- const RtlFieldInfo *field = destRecInfo.queryField(idx);
- if (field->omitable() && destRecInfo.excluded(field, builder.getSelf(), destConditions))
- continue;
- const RtlTypeInfo *type = field->type;
- const MatchInfo &match = matchInfo[idx];
- if (match.matchType == match_none || match.matchType==match_fail)
- {
- offset = type->buildNull(builder, offset, field);
- }
- else if (match.matchType == match_virtual)
- {
- switch (getVirtualInitializer(field->initializer))
- {
- case FVirtualFilePosition:
- offset = type->buildInt(builder, offset, field, callback.getFilePosition(sourceRow));
- break;
- case FVirtualLocalFilePosition:
- offset = type->buildInt(builder, offset, field, callback.getLocalFilePosition(sourceRow));
- break;
- case FVirtualFilename:
- {
- const char * filename = callback.queryLogicalFilename(sourceRow);
- offset = type->buildString(builder, offset, field, strlen(filename), filename);
- break;
- }
- default:
- throwUnexpected();
- }
- }
- else
- {
- unsigned matchField = match.matchIdx;
- const RtlTypeInfo *sourceType = sourceRecInfo.queryType(matchField);
- size_t sourceOffset = 0;
- const byte *source = nullptr;
- size_t copySize = 0;
- if (binarySource)
- {
- const RtlRow &rtlRow = *(const RtlRow *)sourceRow;
- sourceOffset = rtlRow.getOffset(matchField);
- source = rtlRow.queryRow() + sourceOffset;
- copySize = rtlRow.getSize(matchField);
- }
- if (match.matchType & match_deblob)
- {
- offset_t blobId = sourceType->getInt(source);
- sourceType = sourceType->queryChildType();
- sourceOffset = 0;
- source = callback.lookupBlob(blobId);
- copySize = sourceType->size(source, source);
- hasBlobs = true;
- }
- if (copySize == 0 && (match.matchType & match_inifblock)) // Field is missing because of an ifblock - use default value
- {
- offset = type->buildNull(builder, offset, field);
- }
- else
- {
- switch (match.matchType & ~(match_inifblock|match_deblob))
- {
- case match_perfect:
- {
- // Look ahead for other perfect matches and combine the copies
- if (!(match.matchType & match_deblob))
- {
- while (idx < destRecInfo.getNumFields()-1)
- {
- const MatchInfo &nextMatch = matchInfo[idx+1];
- if (nextMatch.matchType == match_perfect && nextMatch.matchIdx == matchField+1)
- {
- idx++;
- matchField++;
- }
- else
- break;
- }
- copySize = ((const RtlRow *)sourceRow)->getOffset(matchField+1) - sourceOffset;
- }
- builder.ensureCapacity(offset+copySize, field->name);
- memcpy(builder.getSelf()+offset, source, copySize);
- offset += copySize;
- break;
- }
- case match_truncate:
- {
- assert(type->isFixedSize());
- copySize = type->getMinSize();
- builder.ensureCapacity(offset+copySize, field->name);
- memcpy(builder.getSelf()+offset, source, copySize);
- offset += copySize;
- break;
- }
- case match_extend:
- {
- assert(type->isFixedSize());
- size32_t destSize = type->getMinSize();
- builder.ensureCapacity(offset+destSize, field->name);
- memcpy(builder.getSelf()+offset, source, copySize);
- offset += copySize;
- unsigned fillSize = destSize - copySize;
- memset(builder.getSelf()+offset, match.fillChar, fillSize);
- offset += fillSize;
- break;
- }
- case match_filepos:
- case match_typecast:
- offset = translateScalar(builder, offset, field, *type, *sourceType, source);
- break;
- case match_typecast|match_dynamic:
- {
- const IDynamicFieldValueFetcher &callbackRowHandler = *(const IDynamicFieldValueFetcher *)sourceRow;
- source = callbackRowHandler.queryValue(matchField, copySize);
- if (callbackRawType == type_string)
- offset = translateScalarFromString(builder, offset, field, *type, *sourceType, (const char *)source, (size_t)copySize);
- else
- offset = translateScalarFromUtf8(builder, offset, field, *type, *sourceType, (const char *)source, (size_t)copySize);
- break;
- }
- case match_link:
- {
- // a 32-bit record count, and a (linked) pointer to an array of record pointers
- byte *dest = builder.ensureCapacity(offset+sizeof(size32_t)+sizeof(const byte **), field->name)+offset;
- *(size32_t *)dest = *(size32_t *)source;
- *(const byte ***)(dest + sizeof(size32_t)) = rtlLinkRowset(*(const byte ***)(source + sizeof(size32_t)));
- offset += sizeof(size32_t)+sizeof(const byte **);
- break;
- }
- case match_recurse|match_dynamic:
- {
- const IDynamicFieldValueFetcher &callbackRowHandler = *(const IDynamicFieldValueFetcher *)sourceRow;
- Owned<IDynamicRowIterator> iterator = callbackRowHandler.getNestedIterator(matchField);
- if (type->getType()==type_record)
- {
- IDynamicFieldValueFetcher &fieldFetcher = iterator->query();
- offset = match.subTrans->doTranslateOpaqueType(builder, callback, offset, &fieldFetcher);
- }
- else if (type->isLinkCounted())
- {
- // a 32-bit record count, and a pointer to an array of record pointers
- IEngineRowAllocator *childAllocator = builder.queryAllocator()->createChildRowAllocator(type->queryChildType());
- assertex(childAllocator); // May not be available when using serialized types (but unlikely to want to create linkcounted children remotely either)
- size32_t sizeInBytes = sizeof(size32_t) + sizeof(void *);
- builder.ensureCapacity(offset+sizeInBytes, field->name);
- size32_t numRows = 0;
- const byte **childRows = nullptr;
- ForEach(*iterator)
- {
- IDynamicFieldValueFetcher &fieldFetcher = iterator->query();
- RtlDynamicRowBuilder childBuilder(*childAllocator);
- size32_t childLen = match.subTrans->doTranslateOpaqueType(childBuilder, callback, 0, &fieldFetcher);
- childRows = childAllocator->appendRowOwn(childRows, ++numRows, (void *) childBuilder.finalizeRowClear(childLen));
- }
- if (type->getType() == type_dictionary)
- {
- const RtlTypeInfo * childType = type->queryChildType();
- assertex(childType && childType->getType() == type_record);
- CHThorHashLookupInfo lookupHelper(static_cast<const RtlRecordTypeInfo &>(*childType));
- rtlCreateDictionaryFromDataset(numRows, childRows, childAllocator, lookupHelper);
- }
- // Go back in and patch the count, remembering it may have moved
- rtlWriteInt4(builder.getSelf()+offset, numRows);
- * ( const void * * ) (builder.getSelf()+offset+sizeof(size32_t)) = childRows;
- offset += sizeInBytes;
- }
- else
- {
- size32_t countOffset = offset;
- byte *dest = builder.ensureCapacity(offset+sizeof(size32_t), field->name)+offset;
- offset += sizeof(size32_t);
- size32_t initialOffset = offset;
- *(size32_t *)dest = 0; // patched below when true figure known
- ForEach(*iterator)
- {
- IDynamicFieldValueFetcher &fieldFetcher = iterator->query();
- offset = match.subTrans->doTranslateOpaqueType(builder, callback, offset, &fieldFetcher);
- }
- dest = builder.getSelf() + countOffset; // Note - may have been moved by reallocs since last calculated
- *(size32_t *)dest = offset - initialOffset;
- }
- break;
- }
- case match_recurse:
- if (type->getType()==type_record)
- offset = match.subTrans->doTranslate(builder, callback, offset, source);
- else if (type->isLinkCounted())
- {
- // a 32-bit record count, and a pointer to an array of record pointers
- Owned<IEngineRowAllocator> childAllocator = builder.queryAllocator()->createChildRowAllocator(type->queryChildType());
- assertex(childAllocator); // May not be available when using serialized types (but unlikely to want to create linkcounted children remotely either)
- size32_t sizeInBytes = sizeof(size32_t) + sizeof(void *);
- builder.ensureCapacity(offset+sizeInBytes, field->name);
- size32_t numRows = 0;
- const byte **childRows = nullptr;
- if (sourceType->isLinkCounted())
- {
- // a 32-bit count, then a pointer to the source rows
- size32_t childCount = *(size32_t *) source;
- source += sizeof(size32_t);
- const byte ** sourceRows = *(const byte***) source;
- for (size32_t childRow = 0; childRow < childCount; childRow++)
- {
- RtlDynamicRowBuilder childBuilder(*childAllocator);
- size32_t childLen = match.subTrans->doTranslate(childBuilder, callback, 0, sourceRows[childRow]);
- childRows = childAllocator->appendRowOwn(childRows, ++numRows, (void *) childBuilder.finalizeRowClear(childLen));
- }
- }
- else
- {
- // a 32-bit size, then rows inline
- size32_t childSize = *(size32_t *) source;
- source += sizeof(size32_t);
- const byte *initialSource = source;
- while ((size_t)(source - initialSource) < childSize)
- {
- RtlDynamicRowBuilder childBuilder(*childAllocator);
- size32_t childLen = match.subTrans->doTranslate(childBuilder, callback, 0, source);
- childRows = childAllocator->appendRowOwn(childRows, ++numRows, (void *) childBuilder.finalizeRowClear(childLen));
- source += sourceType->queryChildType()->size(source, nullptr); // MORE - shame to repeat a calculation that the translate above almost certainly just did
- }
- }
- if (type->getType() == type_dictionary)
- {
- const RtlTypeInfo * childType = type->queryChildType();
- assertex(childType && childType->getType() == type_record);
- CHThorHashLookupInfo lookupHelper(static_cast<const RtlRecordTypeInfo &>(*childType));
- rtlCreateDictionaryFromDataset(numRows, childRows, childAllocator, lookupHelper);
- }
- // Go back in and patch the count, remembering it may have moved
- rtlWriteInt4(builder.getSelf()+offset, numRows);
- * ( const void * * ) (builder.getSelf()+offset+sizeof(size32_t)) = childRows;
- offset += sizeInBytes;
- }
- else
- {
- size32_t countOffset = offset;
- byte *dest = builder.ensureCapacity(offset+sizeof(size32_t), field->name)+offset;
- offset += sizeof(size32_t);
- size32_t initialOffset = offset;
- *(size32_t *)dest = 0; // patched below when true figure known
- if (sourceType->isLinkCounted())
- {
- // a 32-bit count, then a pointer to the source rows
- size32_t childCount = *(size32_t *) source;
- source += sizeof(size32_t);
- const byte ** sourceRows = *(const byte***) source;
- for (size32_t childRow = 0; childRow < childCount; childRow++)
- {
- const byte * row = sourceRows[childRow];
- //Dictionaries have blank rows - ignore them when serializing (to a dataset)
- if (row)
- offset = match.subTrans->doTranslate(builder, callback, offset, row);
- }
- }
- else
- {
- // a 32-bit size, then rows inline
- size32_t childSize = *(size32_t *) source;
- source += sizeof(size32_t);
- const byte *initialSource = source;
- while ((size_t)(source - initialSource) < childSize)
- {
- offset = match.subTrans->doTranslate(builder, callback, offset, source);
- source += sourceType->queryChildType()->size(source, nullptr); // MORE - shame to repeat a calculation that the translate above almost certainly just did
- }
- }
- dest = builder.getSelf() + countOffset; // Note - may have been moved by reallocs since last calculated
- *(size32_t *)dest = offset - initialOffset;
- }
- break;
- default:
- throwUnexpected();
- }
- }
- }
- }
- if (estimate && offset-origOffset != estimate)
- {
- if (offset == origOffset)
- {
- //Zero size records are treated as single byte to avoid confusion with sizes returned from transforms etc.
- offset++;
- }
- else
- {
- if (!hasBlobs)
- assert(offset-origOffset > estimate); // Estimate is always supposed to be conservative
- #ifdef TRACE_TRANSLATION
- DBGLOG("Wrote %u bytes to record (estimate was %u)\n", offset-origOffset, estimate);
- #endif
- }
- }
- return offset;
- }
- inline FieldMatchType match() const
- {
- return matchFlags;
- }
- const RtlRecord &destRecInfo;
- const RtlRecord &sourceRecInfo;
- bool binarySource = true;
- type_vals callbackRawType;
- int fixedDelta = 0; // total size difference from all fixed size mappings
- UnsignedArray allUnmatched; // List of all source fields that are unmatched (so that we can trace them)
- UnsignedArray variableUnmatched; // List of all variable-size source fields that are unmatched
- FieldMatchType matchFlags = match_perfect;
- struct MatchInfo
- {
- unsigned matchIdx = 0;
- FieldMatchType matchType = match_fail;
- char fillChar = 0;
- GeneralRecordTranslator *subTrans = nullptr;
- ~MatchInfo()
- {
- delete subTrans;
- }
- } *matchInfo;
- static size32_t translateScalarFromUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, const RtlTypeInfo &destType, const RtlTypeInfo &sourceType, const char *source, size_t srcSize)
- {
- switch(destType.getType())
- {
- case type_boolean:
- case type_int:
- case type_swapint:
- case type_packedint:
- case type_filepos:
- case type_keyedint:
- {
- __int64 res = rtlStrToInt8(srcSize, source);
- offset = destType.buildInt(builder, offset, field, res);
- break;
- }
- case type_real:
- {
- double res = rtlStrToReal(srcSize, source);
- offset = destType.buildReal(builder, offset, field, res);
- break;
- }
- case type_data:
- case type_string:
- case type_decimal: // Go via string - not common enough to special-case
- case type_varstring:
- case type_qstring:
- case type_utf8:
- //MORE: Could special case casting from utf8 to utf8 similar to strings above
- case type_unicode:
- case type_varunicode:
- {
- size32_t utf8chars = rtlUtf8Length(srcSize, source);
- offset = destType.buildUtf8(builder, offset, field, utf8chars, source);
- break;
- }
- case type_set:
- {
- UNIMPLEMENTED; // JCS->GH - but perhaps can/should translate using iterator too?
- break;
- }
- default:
- throwUnexpected();
- }
- return offset;
- }
- static size32_t translateScalarFromString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, const RtlTypeInfo &destType, const RtlTypeInfo &sourceType, const char *source, size_t srcSize)
- {
- switch(destType.getType())
- {
- case type_boolean:
- case type_int:
- case type_swapint:
- case type_packedint:
- case type_filepos:
- case type_keyedint:
- {
- __int64 res = rtlStrToInt8(srcSize, source);
- offset = destType.buildInt(builder, offset, field, res);
- break;
- }
- case type_real:
- {
- double res = rtlStrToReal(srcSize, source);
- offset = destType.buildReal(builder, offset, field, res);
- break;
- }
- case type_data:
- case type_string:
- case type_decimal: // Go via string - not common enough to special-case
- case type_varstring:
- case type_qstring:
- case type_utf8:
- //MORE: Could special case casting from utf8 to utf8 similar to strings above
- case type_unicode:
- case type_varunicode:
- {
- offset = destType.buildString(builder, offset, field, srcSize, source);
- break;
- }
- case type_set:
- {
- UNIMPLEMENTED; // JCS->GH - but perhaps can/should translate using iterator too?
- break;
- }
- default:
- throwUnexpected();
- }
- return offset;
- }
- static bool canTranslateNonScalar(const RtlTypeInfo * type, const RtlTypeInfo * sourceType)
- {
- auto target = type->getType();
- auto source = sourceType->getType();
- if (target == source)
- return true;
- if ((target == type_dictionary) && (source == type_table))
- return true;
- if ((target == type_table) && (source == type_dictionary))
- return true;
- return false;
- }
- void createMatchInfo()
- {
- unsigned defaulted = 0;
- bool destHasNested = destRecInfo.hasNested();
- bool sourceHasNested = sourceRecInfo.hasNested();
- for (unsigned idx = 0; idx < destRecInfo.getNumFields(); idx++)
- {
- const RtlFieldInfo *field = destRecInfo.queryField(idx);
- const RtlTypeInfo *type = field->type;
- MatchInfo &info = matchInfo[idx];
- const char *name = destRecInfo.queryName(idx);
- info.matchIdx = sourceRecInfo.getFieldNum(name);
- if (info.matchIdx == (unsigned) -1)
- {
- const byte * initializer = (const byte *) field->initializer;
- info.matchType = isVirtualInitializer(initializer) ? match_virtual : match_none;
- if ((field->flags & RFTMinifblock) == 0)
- {
- size32_t defaultSize = (initializer && !isVirtualInitializer(initializer)) ? type->size(initializer, nullptr) : type->getMinSize();
- fixedDelta -= defaultSize;
- #ifdef TRACE_TRANSLATION
- DBGLOG("Decreasing fixedDelta size by %d to %d for defaulted field %d (%s)", defaultSize, fixedDelta, idx, destRecInfo.queryName(idx));
- #endif
- }
- if ((field->flags & RFTMispayloadfield) == 0)
- matchFlags |= match_keychange;
- defaulted++;
- // If dest field is in a nested record, we need to check that there's no "non-record" field in source matching current nested record name
- if (name)
- {
- if (destHasNested)
- {
- const char *ldot = strrchr(name, '.');
- if (ldot)
- {
- StringBuffer recname(ldot-name, name);
- if (sourceRecInfo.getFieldNum(recname) != (unsigned) -1)
- info.matchType = match_fail; // No translation from non-record to record
- }
- }
- if (sourceHasNested && sourceRecInfo.queryOriginalField(name))
- {
- // Similarly if dest field IS not a nested record, but there is a field in source which is.
- // Note that we already know there is no matching field called name in the exapanded version of source,
- // so any match we find must be a record
- info.matchType = match_fail; // No translation from record to non-record
- }
- }
- }
- else
- {
- bool deblob = false;
- const RtlTypeInfo *sourceType = sourceRecInfo.queryType(info.matchIdx);
- unsigned sourceFlags = sourceRecInfo.queryField(info.matchIdx)->flags;
- unsigned destFlags = field->flags;
- if (binarySource && sourceType->isBlob())
- {
- if (type->isBlob())
- {
- }
- else
- {
- sourceType = sourceType->queryChildType();
- deblob = true;
- }
- }
- if (!type->isScalar() || !sourceType->isScalar())
- {
- if (!canTranslateNonScalar(type, sourceType))
- info.matchType = match_fail; // No translation from one non-scalar type to another
- else
- {
- switch (type->getType())
- {
- case type_set:
- if (binarySource)
- {
- if (type->queryChildType()->fieldType==sourceType->queryChildType()->fieldType &&
- type->queryChildType()->length==sourceType->queryChildType()->length)
- info.matchType = match_perfect;
- else
- info.matchType = match_typecast;
- }
- else
- info.matchType = match_typecast|match_dynamic;
- break;
- case type_row: // These are not expected I think...
- throwUnexpected();
- case type_ifblock:
- case type_record:
- case type_table:
- case type_dictionary:
- {
- const RtlRecord *subDest = destRecInfo.queryNested(idx);
- const RtlRecord *subSrc = sourceRecInfo.queryNested(info.matchIdx);
- info.subTrans = new GeneralRecordTranslator(*subDest, *subSrc, binarySource);
- if (!info.subTrans->needsTranslate())
- {
- if (!binarySource)
- info.matchType = match_recurse|match_dynamic;
- else
- {
- // Child does not require translation, but check linkcount mode matches too!
- if (type->isLinkCounted())
- {
- if (sourceType->isLinkCounted())
- info.matchType = match_link;
- else
- info.matchType = match_recurse;
- }
- else
- {
- if (sourceType->isLinkCounted())
- info.matchType = match_recurse;
- else
- info.matchType = match_perfect;
- }
- if (info.matchType != match_recurse)
- {
- delete info.subTrans;
- info.subTrans = nullptr;
- }
- }
- }
- else if (info.subTrans->canTranslate())
- {
- info.matchType = binarySource ? match_recurse : (match_recurse|match_dynamic);
- unsigned childFlags = info.subTrans->matchFlags;
- //Ignore differences in the keyed flag for child structures (it will be set later if this field is keyed)
- matchFlags |= (FieldMatchType)(childFlags & ~match_keychange);
- }
- else
- info.matchType = match_fail;
- break;
- }
- case type_blob:
- if (!binarySource)
- info.matchType = match_fail;
- else if (sourceType->isBlob())
- info.matchType = match_perfect; // We don't check that the child type matches
- else
- info.matchType = match_fail;
- break;
- default:
- info.matchType = match_fail;
- break;
- }
- }
- }
- else if (!binarySource)
- info.matchType = match_typecast|match_dynamic;
- else if ((type->fieldType==sourceType->fieldType))
- {
- if (type->length==sourceType->length)
- {
- info.matchType = match_perfect;
- }
- else
- {
- assert(type->isFixedSize()); // Both variable size would have matched length above
- info.matchType = match_typecast;
- if (type->length < sourceType->length)
- {
- if (type->canTruncate())
- {
- info.matchType = match_truncate;
- if (((sourceFlags|destFlags) & RFTMinifblock) == 0)
- fixedDelta += sourceType->getMinSize()-type->getMinSize();
- #ifdef TRACE_TRANSLATION
- DBGLOG("Increasing fixedDelta size by %d to %d for truncated field %d (%s)", sourceType->getMinSize()-type->getMinSize(), fixedDelta, idx, destRecInfo.queryName(idx));
- #endif
- }
- }
- else
- {
- if (type->canExtend(info.fillChar))
- {
- info.matchType = match_extend;
- if (((sourceFlags|destFlags) & RFTMinifblock) == 0)
- fixedDelta += sourceType->getMinSize()-type->getMinSize();
- #ifdef TRACE_TRANSLATION
- DBGLOG("Decreasing fixedDelta size by %d to %d for truncated field %d (%s)", type->getMinSize()-sourceType->getMinSize(), fixedDelta, idx, destRecInfo.queryName(idx));
- #endif
- }
- }
- }
- }
- else if ((type->getType()==type_filepos || sourceType->getType()==type_filepos) &&
- type->isUnsigned()==sourceType->isUnsigned())
- info.matchType = match_filepos;
- else
- info.matchType = match_typecast;
- if (deblob)
- info.matchType |= match_deblob;
- if (sourceFlags & RFTMinifblock || field->flags & RFTMinifblock)
- info.matchType |= match_inifblock; // Avoids incorrect commoning up of adjacent matches
- // MORE - could note the highest interesting fieldnumber in the source and not bother filling in offsets after that
- // Not sure it would help much though - usually need to know the total record size anyway in real life
- if (idx != info.matchIdx)
- matchFlags |= match_move;
- //Whether this field is in an ifblock, or needs to be copied by linking it do not count as changes
- FieldMatchType maskedType = (FieldMatchType)(info.matchType & ~(match_link|match_inifblock));
- if (((maskedType != match_perfect) || (idx != info.matchIdx)) && ((field->flags & RFTMispayloadfield) == 0 || (sourceFlags & RFTMispayloadfield) == 0))
- matchFlags |= match_keychange;
- else if ((field->flags & RFTMispayloadfield) != (sourceFlags & RFTMispayloadfield))
- matchFlags |= match_keychange;
- }
- matchFlags |= info.matchType;
- }
- if (sourceRecInfo.getNumFields() > destRecInfo.getNumFields()-defaulted)
- {
- matchFlags |= match_remove;
- for (unsigned idx = 0; idx < sourceRecInfo.getNumFields(); idx++)
- {
- const RtlFieldInfo *field = sourceRecInfo.queryField(idx);
- const char *name = sourceRecInfo.queryName(idx);
- if (destRecInfo.getFieldNum(name) == (unsigned) -1)
- {
- // unmatched field
- if ((field->flags & RFTMispayloadfield) == 0)
- matchFlags |= match_keychange;
- if (!destRecInfo.getFixedSize())
- {
- const RtlTypeInfo *type = field->type;
- if (type->isFixedSize() && (field->flags & RFTMinifblock)==0)
- {
- #ifdef TRACE_TRANSLATION
- DBGLOG("Reducing estimated size by %d for (fixed size) omitted field %s", (int) type->getMinSize(), field->name);
- #endif
- fixedDelta += type->getMinSize();
- }
- else
- variableUnmatched.append(idx);
- }
- allUnmatched.append(idx);
- }
- }
- #ifdef TRACE_TRANSLATION
- DBGLOG("Delta from fixed-size fields is %d bytes", fixedDelta);
- #endif
- }
- }
- size32_t estimateNewSize(const RtlRow &sourceRow) const
- {
- #ifdef TRACE_TRANSLATION
- DBGLOG("Source record size is %d", (int) sourceRow.getRecordSize());
- #endif
- size32_t expectedSize = sourceRow.getRecordSize();
- assertex((int) expectedSize >= fixedDelta);
- expectedSize -= fixedDelta;
- #ifdef TRACE_TRANSLATION
- DBGLOG("Source record size without fixed delta is %d", expectedSize);
- #endif
- ForEachItemIn(i, variableUnmatched)
- {
- unsigned fieldNo = variableUnmatched.item(i);
- expectedSize -= sourceRow.getSize(fieldNo);
- #ifdef TRACE_TRANSLATION
- DBGLOG("Reducing estimated size by %d to %d for omitted field %d (%s)", (int) sourceRow.getSize(fieldNo), expectedSize, fieldNo, sourceRecInfo.queryName(fieldNo));
- #endif
- }
- if (matchFlags & ~(match_perfect|match_link|match_none|match_virtual|match_extend|match_truncate))
- {
- for (unsigned idx = 0; idx < destRecInfo.getNumFields(); idx++)
- {
- const MatchInfo &match = matchInfo[idx];
- const RtlTypeInfo *type = destRecInfo.queryType(idx);
- unsigned matchField = match.matchIdx;
- if ((match.matchType & match_inifblock) == 0)
- {
- switch (match.matchType)
- {
- case match_perfect:
- case match_link:
- case match_none:
- case match_virtual:
- case match_extend:
- case match_truncate:
- // These ones were already included in fixedDelta
- break;
- default:
- // This errs on the side of small - i.e. it assumes that all typecasts end up at minimum size
- // We could do better in some cases e.g. variable string <-> variable unicode we can assume factor of 2,
- // uft8 <-> string we could calculate here - but unlikely to be worth the effort.
- // But it's fine for fixed size output fields, including truncate/extend
- // We could also precalculate the expected delta if all omitted fields are fixed size - but not sure how likely/worthwhile that is.
- auto minSize = type->getMinSize();
- auto sourceSize = sourceRow.getSize(matchField);
- expectedSize += minSize;
- assertex(expectedSize >= sourceSize);
- expectedSize -= sourceSize;
- #ifdef TRACE_TRANSLATION
- DBGLOG("Adjusting estimated size by (%d - %d) to %d for translated field %d (%s)", (int) sourceSize, minSize, expectedSize, matchField, sourceRecInfo.queryName(matchField));
- #endif
- break;
- }
- }
- }
- }
- return expectedSize;
- }
- };
- extern ECLRTL_API const IDynamicTransform *createRecordTranslator(const RtlRecord &destRecInfo, const RtlRecord &srcRecInfo)
- {
- return new GeneralRecordTranslator(destRecInfo, srcRecInfo, true);
- }
- extern ECLRTL_API const IDynamicTransform *createRecordTranslatorViaCallback(const RtlRecord &destRecInfo, const RtlRecord &srcRecInfo, type_vals rawType)
- {
- return new GeneralRecordTranslator(destRecInfo, srcRecInfo, false, rawType);
- }
- extern ECLRTL_API void throwTranslationError(const RtlRecord & destRecInfo, const RtlRecord & srcRecInfo, const char * filename)
- {
- Owned<const IDynamicTransform> translator = createRecordTranslator(destRecInfo, srcRecInfo);
- #ifdef _DEBUG
- translator->describe();
- #endif
- if (!translator->canTranslate())
- throw MakeStringException(0, "Untranslatable record layout mismatch detected for: %s", filename);
- throw MakeStringException(0, "Translatable key layout mismatch reading file %s but translation disabled", filename);
- }
- class TranslatedRowStream : public CInterfaceOf<IRowStream>
- {
- public:
- TranslatedRowStream(IRowStream *_inputStream, IEngineRowAllocator *_resultAllocator, const RtlRecord &outputRecord, const RtlRecord &inputRecord)
- : inputStream(_inputStream), resultAllocator(_resultAllocator)
- {
- translator.setown(createRecordTranslator(outputRecord, inputRecord));
- translator->describe();
- }
- virtual const void *nextRow() override
- {
- if (eof)
- return NULL;
- const void *inRow = inputStream->nextRow();
- if (!inRow)
- {
- if (eogSeen)
- eof = true;
- else
- eogSeen = true;
- return nullptr;
- }
- else
- eogSeen = false;
- RtlDynamicRowBuilder rowBuilder(resultAllocator);
- size32_t len = translator->translate(rowBuilder, fieldCallback, (const byte *) inRow);
- rtlReleaseRow(inRow);
- return rowBuilder.finalizeRowClear(len);
- }
- virtual void stop() override
- {
- resultAllocator.clear();
- }
- bool canTranslate() const
- {
- return translator->canTranslate();
- }
- bool needsTranslate() const
- {
- return translator->needsTranslate();
- }
- UnexpectedVirtualFieldCallback fieldCallback; // I'm not sure if an non unexpected callback can be implemented
- protected:
- Linked<IRowStream> inputStream;
- Linked<IEngineRowAllocator> resultAllocator;
- Owned<const IDynamicTransform> translator;
- unsigned numOffsets = 0;
- size_t * variableOffsets = nullptr;
- bool eof = false;
- bool eogSeen = false;
- };
- extern ECLRTL_API IRowStream * transformRecord(IEngineRowAllocator * resultAllocator,IOutputMetaData & metaInput,IRowStream * input)
- {
- if (resultAllocator->queryOutputMeta()==&metaInput)
- return LINK(input);
- Owned<TranslatedRowStream> stream = new TranslatedRowStream(input, resultAllocator,
- resultAllocator->queryOutputMeta()->queryRecordAccessor(true),
- metaInput.queryRecordAccessor(true));
- if (!stream->needsTranslate())
- return LINK(input);
- else if (!stream->canTranslate())
- rtlFail(0, "Cannot translate record stream");
- else
- return stream.getClear();
- }
- // A key translator allows us to transform a RowFilter that refers to src to one that refers to dest.
- // Basically just a map of those fields with matching types.
- class CKeyTranslator : public CInterfaceOf<IKeyTranslator>
- {
- public:
- CKeyTranslator(const RtlRecord &actual, const RtlRecord &expected)
- {
- translateNeeded = false;
- for (unsigned expectedIdx = 0; expectedIdx < expected.getNumFields(); expectedIdx++)
- {
- unsigned actualIdx = actual.getFieldNum(expected.queryName(expectedIdx));
- if (actualIdx != (unsigned) -1)
- {
- const RtlTypeInfo *expectedType = expected.queryType(expectedIdx);
- const RtlTypeInfo *actualType = actual.queryType(actualIdx);
- if (!actualType->equivalent(expectedType))
- actualIdx = (unsigned) -2;
- }
- map.append(actualIdx);
- if (actualIdx != expectedIdx)
- translateNeeded = true;
- }
- }
- virtual void describe() const override
- {
- ForEachItemIn(idx, map)
- {
- unsigned mapped = map.item(idx);
- switch (mapped)
- {
- case (unsigned) -1: DBGLOG("No match for field %d", idx); break;
- case (unsigned) -2: DBGLOG("Incompatible field match for field %d", idx); break;
- default: DBGLOG("keyed field %d can map to field %d", idx, mapped); break;
- }
- }
- }
- virtual bool translate(RowFilter &filters) const override
- {
- bool mapNeeded = false;
- if (translateNeeded)
- {
- unsigned numFields = filters.numFilterFields();
- for (unsigned idx = 0; idx < numFields; idx++)
- {
- unsigned fieldNum = filters.queryFilter(idx).queryFieldIndex();
- unsigned mappedFieldNum = map.isItem(fieldNum) ? map.item(fieldNum) : (unsigned) -1;
- if (mappedFieldNum != fieldNum)
- {
- mapNeeded = true;
- switch (mappedFieldNum)
- {
- case (unsigned) -1: throw makeStringExceptionV(0, "Cannot translate keyed filter on field %u - no matching field", idx);
- case (unsigned) -2: throw makeStringExceptionV(0, "Cannot translate keyed filter on field %u - incompatible matching field type", idx);
- default:
- filters.remapField(idx, mappedFieldNum);
- break;
- }
- }
- }
- if (mapNeeded)
- filters.recalcFieldsRequired();
- }
- return mapNeeded;
- }
- virtual bool translate(RowFilter &filter, IConstArrayOf<IFieldFilter> &in) const override
- {
- bool mapNeeded = false;
- if (translateNeeded)
- {
- unsigned numFields = in.length();
- for (unsigned idx = 0; idx < numFields; idx++)
- {
- unsigned fieldNum = in.item(idx).queryFieldIndex();
- unsigned mappedFieldNum = map.isItem(fieldNum) ? map.item(fieldNum) : (unsigned) -1;
- if (mappedFieldNum != fieldNum)
- {
- mapNeeded = true;
- switch (mappedFieldNum)
- {
- case (unsigned) -1: throw makeStringExceptionV(0, "Cannot translate keyed filter on field %u - no matching field", idx);
- case (unsigned) -2: throw makeStringExceptionV(0, "Cannot translate keyed filter on field %u - incompatible matching field type", idx);
- default:
- filter.addFilter(*in.item(idx).remap(mappedFieldNum));
- break;
- }
- }
- else
- filter.addFilter(OLINK(in.item(idx)));
- }
- }
- return mapNeeded;
- }
- virtual bool needsTranslate() const
- {
- return translateNeeded;
- }
- protected:
- UnsignedArray map;
- bool translateNeeded = false;
- };
- extern ECLRTL_API const IKeyTranslator *createKeyTranslator(const RtlRecord &_destRecInfo, const RtlRecord &_srcRecInfo)
- {
- return new CKeyTranslator(_destRecInfo, _srcRecInfo);
- }
- //---------------------------------------------------------------------------------------------------------------------
- const char * NullVirtualFieldCallback::queryLogicalFilename(const void * row)
- {
- return "";
- }
- unsigned __int64 NullVirtualFieldCallback::getFilePosition(const void * row)
- {
- return 0;
- }
- unsigned __int64 NullVirtualFieldCallback::getLocalFilePosition(const void * row)
- {
- return 0;
- }
- const byte * NullVirtualFieldCallback::lookupBlob(unsigned __int64 id)
- {
- return nullptr;
- }
- const char * UnexpectedVirtualFieldCallback::queryLogicalFilename(const void * row)
- {
- throwUnexpectedX("VIRTUAL(LOGICALFILENAME)");
- }
- unsigned __int64 UnexpectedVirtualFieldCallback::getFilePosition(const void * row)
- {
- throwUnexpectedX("VIRTUAL(FILEPOSITION)");
- }
- unsigned __int64 UnexpectedVirtualFieldCallback::getLocalFilePosition(const void * row)
- {
- throwUnexpectedX("VIRTUAL(LOCALFILEPOSITION)");
- }
- const byte * UnexpectedVirtualFieldCallback::lookupBlob(unsigned __int64 id)
- {
- throwUnexpectedX("BLOB");
- }
- unsigned __int64 FetchVirtualFieldCallback::getFilePosition(const void * row)
- {
- return filepos;
- }
- const char * LocalVirtualFieldCallback::queryLogicalFilename(const void * row)
- {
- return filename;
- }
- unsigned __int64 LocalVirtualFieldCallback::getFilePosition(const void * row)
- {
- return filepos;
- }
- unsigned __int64 LocalVirtualFieldCallback::getLocalFilePosition(const void * row)
- {
- return localfilepos;
- }
- const byte * LocalVirtualFieldCallback::lookupBlob(unsigned __int64 id)
- {
- throwUnexpectedX("BLOB");
- }
|