12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463 |
- /*##############################################################################
- HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- ############################################################################## */
- #include "platform.h"
- #include <math.h>
- #include <stdio.h>
- #include "jmisc.hpp"
- #include "jlib.hpp"
- #include "eclhelper.hpp"
- #include "eclrtl_imp.hpp"
- #include "rtldynfield.hpp"
- #include "rtlrecord.hpp"
- #include "rtlembed.hpp"
- //#define TRACE_TRANSLATION
- #define VALIDATE_TYPEINFO_HASHES
- #define RTLTYPEINFO_FORMAT_1 80 // In case we ever want to support more than one format
- //---------------------------------------------------------------------------------------------------------------------
- const RtlTypeInfo *FieldTypeInfoStruct::createRtlTypeInfo(IThorIndexCallback *_callback) const
- {
- const RtlTypeInfo *ret = nullptr;
- switch (fieldType & RFTMkind)
- {
- case type_boolean:
- ret = new RtlBoolTypeInfo(fieldType, length);
- break;
- case type_keyedint:
- ret = new RtlKeyedIntTypeInfo(fieldType, length, childType);
- break;
- case type_blob: // MORE - will need its own type (see code below)
- case type_int:
- ret = new RtlIntTypeInfo(fieldType, length);
- break;
- #if 0 // Later when implemented
- case type_blob:
- ret = new RtlBlobTypeInfo(fieldType, length, childType, _callback);
- break;
- #endif
- case type_filepos:
- #if __BYTE_ORDER == __LITTLE_ENDIAN
- ret = new RtlSwapIntTypeInfo(fieldType, length);
- #else
- ret = new RtlIntTypeInfo(fieldType, length);
- #endif
- break;
- case type_real:
- ret = new RtlRealTypeInfo(fieldType, length);
- break;
- case type_decimal:
- ret = new RtlDecimalTypeInfo(fieldType, length);
- break;
- case type_string:
- ret = new RtlStringTypeInfo(fieldType, length);
- break;
- case type_bitfield:
- ret = new RtlBitfieldTypeInfo(fieldType, length);
- break;
- case type_varstring:
- ret = new RtlVarStringTypeInfo(fieldType, length);
- break;
- case type_data:
- ret = new RtlDataTypeInfo(fieldType, length);
- break;
- case type_table:
- assert(childType);
- ret = new RtlDatasetTypeInfo(fieldType, length, childType);
- break;
- case type_set:
- assert(childType);
- ret = new RtlSetTypeInfo(fieldType, length, childType);
- break;
- case type_row:
- assert(childType);
- ret = new RtlRowTypeInfo(fieldType, length, childType);
- break;
- case type_swapint:
- ret = new RtlSwapIntTypeInfo(fieldType, length);
- break;
- case type_packedint:
- ret = new RtlPackedIntTypeInfo(fieldType, length);
- break;
- case type_qstring:
- ret = new RtlQStringTypeInfo(fieldType, length);
- break;
- case type_unicode:
- ret = new RtlUnicodeTypeInfo(fieldType, length, locale);
- break;
- case type_varunicode:
- ret = new RtlVarUnicodeTypeInfo(fieldType, length, locale);
- break;
- case type_utf8:
- ret = new RtlUtf8TypeInfo(fieldType, length, locale);
- break;
- case type_record:
- ret = new RtlRecordTypeInfo(fieldType, length, fieldsArray);
- break;
- case type_ifblock:
- ret = new RtlDynamicIfBlockTypeInfo(fieldType, length, fieldsArray);
- break;
- default:
- throwUnexpected();
- }
- return ret;
- };
- typedef MapBetween<const RtlTypeInfo *, const RtlTypeInfo *, StringAttr, const char *> TypeNameMap;
- typedef MapBetween<const RtlTypeInfo *, const RtlTypeInfo *, unsigned, unsigned> TypeNumMap;
- /**
- * class CRtlFieldTypeSerializer
- *
- * Serializer class for creating json representation of a RtlTypeInfo structure.
- *
- */
- class CRtlFieldTypeSerializer
- {
- public:
- /**
- * Serialize a RtlTypeInfo structure to JSON
- *
- * @param out Buffer for resulting serialized string
- * @param type RtlTypeInfo structure to be serialized
- * @return Referenced to supplied buffer
- */
- static StringBuffer &serialize(StringBuffer &out, const RtlTypeInfo *type)
- {
- CRtlFieldTypeSerializer s(out, type);
- s.doSerialize();
- return out;
- }
- private:
- CRtlFieldTypeSerializer(StringBuffer &_out, const RtlTypeInfo *_base)
- : json(_out), base(_base)
- {
- }
- void doSerialize()
- {
- json.append("{");
- serializeType(base);
- json.append("\n}");
- }
- void serializeType(const RtlTypeInfo *type)
- {
- if (!serialized(type))
- {
- // Make sure all child types are serialized first
- const RtlTypeInfo *child = type->queryChildType();
- if (child)
- serializeType(child);
- const RtlFieldInfo * const * fields = type->queryFields();
- if (fields)
- {
- for (;;)
- {
- const RtlFieldInfo * child = *fields;
- if (!child)
- break;
- serializeType(child->type);
- fields++;
- }
- }
- // Now serialize this one
- if (type != base)
- {
- VStringBuffer newName("ty%d", ++nextTypeName);
- types.setValue(type, newName.str());
- startField(newName.str());
- serializeMe(type);
- closeCurly();
- }
- else
- serializeMe(type);
- }
- }
- void serializeMe(const RtlTypeInfo *type)
- {
- if (!type->canSerialize())
- throw makeStringException(MSGAUD_user, 1000, "This type structure cannot be serialized");
- addPropHex("fieldType", type->fieldType);
- addProp("length", type->length);
- addPropNonEmpty("locale", type->queryLocale());
- const RtlTypeInfo *child = type->queryChildType();
- if (child)
- addPropType("child", child);
- const RtlFieldInfo * const * fields = type->queryFields();
- if (fields)
- {
- startFields();
- for (;;)
- {
- const RtlFieldInfo * child = *fields;
- if (!child)
- break;
- newline();
- openCurly();
- addProp("name", child->name);
- addPropType("type", child->type);
- addProp("xpath", child->xpath);
- if (child->flags)
- addPropHex("flags", child->flags);
- // initializer is tricky - it's not (in general) a null-terminated string but the actual length is not easily available
- if (child->initializer)
- {
- addProp("init", child->type->size(child->initializer, nullptr), child->initializer);
- }
- closeCurly();
- fields++;
- }
- endFields();
- }
- }
- bool serialized(const RtlTypeInfo *type)
- {
- return types.find(type) != nullptr;
- }
- void startField(const char *name)
- {
- newline().appendf("\"%s\": ", name);
- openCurly();
- }
- void addProp(const char *propName, const char *propVal)
- {
- if (propVal)
- {
- newline();
- encodeJSON(json.append("\""), propName).append("\": ");
- encodeJSON(json.append("\""), propVal).append("\"");
- }
- }
- void addProp(const char *propName, size32_t propLen, const byte *propVal)
- {
- if (propVal)
- {
- newline();
- encodeJSON(json.append("\""), propName).append("\": \"");
- JBASE64_Encode(propVal, propLen, json, false);
- json.append("\"");
- }
- }
- void addPropNonEmpty(const char *propName, const char *propVal)
- {
- if (propVal && *propVal)
- addProp(propName, propVal);
- }
- void addProp(const char *propName, unsigned propVal)
- {
- newline().appendf("\"%s\": %u", propName, propVal);
- }
- void addPropHex(const char *propName, unsigned propVal)
- {
- newline().appendf("\"%s\": %u", propName, propVal); // Nice idea but json does not support hex constants :(
- }
- void addPropType(const char *propName, const RtlTypeInfo *type)
- {
- addProp(propName, queryTypeName(type));
- }
- const char *queryTypeName(const RtlTypeInfo *type)
- {
- StringAttr *typeName = types.getValue(type);
- assertex(typeName);
- return typeName->get();
- }
- void startFields()
- {
- newline().appendf("\"fields\": ");
- openCurly('[');
- }
- void endFields()
- {
- closeCurly(']');
- }
- StringBuffer &newline()
- {
- if (commaPending)
- json.append(',');
- json.appendf("\n%*s", indent, "");
- commaPending = true;
- return json;
- }
- void closeCurly(char brace = '}')
- {
- indent--;
- json.appendf("\n%*s%c", indent, "", brace);
- commaPending = true;
- }
- void openCurly(char brace = '{')
- {
- json.append(brace);
- indent++;
- commaPending = false;
- }
- TypeNameMap types;
- StringBuffer &json;
- const RtlTypeInfo *base = nullptr;
- unsigned indent = 1;
- unsigned nextTypeName = 0;
- bool commaPending = false;
- };
- class CRtlFieldTypeBinSerializer
- {
- public:
- /**
- * Serialize a RtlTypeInfo structure to binary
- *
- * @param out Buffer for resulting serialized string
- * @param type RtlTypeInfo structure to be serialized
- * @return Referenced to supplied buffer
- */
- static MemoryBuffer &serialize(MemoryBuffer &out, const RtlTypeInfo *type)
- {
- int oldEnd = out.setEndian(__LITTLE_ENDIAN);
- CRtlFieldTypeBinSerializer s(out);
- byte format = RTLTYPEINFO_FORMAT_1;
- out.append(format);
- DelayedMarker<hash64_t> hash(out);
- DelayedSizeMarker size(out);
- size32_t pos = out.length();
- s.serializeType(type);
- size.write();
- hash.write(rtlHash64Data(size.size(), out.toByteArray()+pos, 0));
- out.setEndian(oldEnd);
- return out;
- }
- private:
- CRtlFieldTypeBinSerializer(MemoryBuffer &_out)
- : out(_out)
- {
- }
- void serializeType(const RtlTypeInfo *type)
- {
- if (!serialized(type))
- {
- // Make sure all child types are serialized first
- const RtlTypeInfo *child = type->queryChildType();
- if (child)
- serializeType(child);
- const RtlFieldInfo * const * fields = type->queryFields();
- if (fields)
- {
- for (unsigned idx = 0;;idx++)
- {
- const RtlFieldInfo * child = fields[idx];
- if (!child)
- break;
- serializeType(child->type);
- }
- }
- // Now serialize this one
- types.setValue(type, nextTypeNum++);
- serializeMe(type);
- }
- }
-
- void serializeMe(const RtlTypeInfo *type)
- {
- if (!type->canSerialize())
- throw makeStringException(MSGAUD_user, 1000, "This type structure cannot be serialized");
- unsigned fieldType = type->fieldType;
- const char *locale = type->queryLocale();
- if (locale && *locale)
- fieldType |= RFTMhasLocale;
- const RtlTypeInfo *child = type->queryChildType();
- if (child)
- fieldType |= RFTMhasChildType;
- const RtlFieldInfo * const * fields = type->queryFields();
- if (fields)
- fieldType |= RFTMhasFields;
- out.append(fieldType);
- out.appendPacked(type->length);
- if (fieldType & RFTMhasLocale)
- out.append(locale);
- if (child)
- out.appendPacked(queryTypeIdx(child));
- if (fields)
- {
- unsigned count = countFields(fields);
- out.appendPacked(count);
- for (;;)
- {
- const RtlFieldInfo * child = *fields;
- if (!child)
- break;
- out.append(child->name);
- out.appendPacked(queryTypeIdx(child->type));
- unsigned flags = child->flags;
- if (child->xpath)
- flags |= RFTMhasXpath;
- if (child->initializer)
- flags |= RFTMhasInitializer;
- out.append(flags);
- if (child->xpath)
- out.append(child->xpath);
- // initializer is tricky - it's not (in general) a null-terminated string but the actual length is not easily available
- if (child->initializer)
- {
- unsigned initLength = child->type->size(child->initializer, nullptr);
- out.appendPacked(initLength).append(initLength, child->initializer);
- }
- fields++;
- }
- }
- }
- bool serialized(const RtlTypeInfo *type)
- {
- return types.find(type) != nullptr;
- }
- unsigned queryTypeIdx(const RtlTypeInfo *type)
- {
- unsigned *typeNum = types.getValue(type);
- assertex(typeNum);
- return *typeNum;
- }
- TypeNumMap types;
- MemoryBuffer &out;
- unsigned nextTypeNum = 0;
- };
- /**
- * class CRtlFieldTypeDeserializer
- *
- * Deserializer class for creating a RtlTypeInfo structure from json representation.
- *
- * Note that the resulting RtlTypeInfo structures are owned by this object and will be
- * destroyed when this object is destroyed.
- *
- */
- class CRtlFieldTypeDeserializer : public CInterfaceOf<IRtlFieldTypeDeserializer>
- {
- public:
- /**
- * CRtlFieldTypeDeserializer constructor
- *
- * @param _callback Supplies a callback to be used for blobs/filepositions.
- */
- CRtlFieldTypeDeserializer(IThorIndexCallback *_callback)
- : callback(_callback)
- {
- }
- /**
- * CRtlFieldTypeDeserializer destructor
- * <p>
- * Releases all RtlTypeInfo and related structures created by this deserializer
- */
- ~CRtlFieldTypeDeserializer()
- {
- // Need some care - all the RtlTypeInfo objects I created need to be destroyed, together with anything else I had to create
- // Strings (other than the init strings) are preserved in the AtomTable
- HashIterator allTypes(types);
- ForEach(allTypes)
- {
- const RtlTypeInfo **type = types.mapToValue(&allTypes.query());
- cleanupType(*type);
- }
- cleanupType(base);
- }
- /**
- * Obtain the deserialized type information
- * <p>
- * Note that the RtlTypeInfo objects are not link-counted, so the lifetime of these objects
- * is determined by the lifetime of the deserializer. They will be released once the deserializer
- * that created them is deleted.
- * <p>
- * Do not call more than once.
- *
- * @param _json JSON text to be deserialized, as created by CRtlFieldTypeSerializer
- * @return Deserialized type object
- */
- virtual const RtlTypeInfo *deserialize(const char *json) override
- {
- assertex(!base);
- Owned<IPropertyTree> jsonTree = createPTreeFromJSONString(json);
- base = deserializeType(jsonTree, jsonTree);
- return base;
- }
- /**
- * Obtain the deserialized type information
- * <p>
- * Note that the RtlTypeInfo objects are not link-counted, so the lifetime of these objects
- * is determined by the lifetime of the deserializer. They will be released once the deserializer
- * that created them is deleted.
- * <p>
- * Do not call more than once.
- *
- * @param _jsonTree JSON property tree to be deserialized, as created by CRtlFieldTypeSerializer
- * @return Deserialized type object
- */
- virtual const RtlTypeInfo *deserialize(IPropertyTree &jsonTree) override
- {
- assertex(!base);
- base = deserializeType(&jsonTree, &jsonTree);
- return base;
- }
- /**
- * Obtain the deserialized type information
- * <p>
- * Note that the RtlTypeInfo objects are not link-counted, so the lifetime of these objects
- * is determined by the lifetime of the deserializer. They will be released once the deserializer
- * that created them is deleted.
- * <p>
- * Do not call more than once.
- *
- * @param buf Binary serialized typeinfo to be deserialized, as created by CRtlFieldTypeSerializer
- * @return Deserialized type object
- */
- virtual const RtlTypeInfo *deserialize(MemoryBuffer &buf) override
- {
- assertex(!base);
- unsigned nextTypeNum = 0;
- int oldEndian = buf.setEndian(__LITTLE_ENDIAN);
- try
- {
- byte format;
- buf.read(format);
- if (format != RTLTYPEINFO_FORMAT_1)
- throw MakeStringException(0, "Invalid type info (%d) in CRtlFieldTypeDeserializer::deserialize", format);
- hash64_t hash;
- buf.read(hash);
- size32_t size;
- buf.read(size);
- #ifdef VALIDATE_TYPEINFO_HASHES
- hash64_t expected = rtlHash64Data(size, buf.readDirect(0), 0);
- if (expected != hash)
- throw MakeStringException(0, "Invalid type info hash in CRtlFieldTypeDeserializer::deserialize");
- #endif
- size32_t endpos = buf.getPos() + size;
- while (buf.getPos() < endpos)
- {
- if (base)
- {
- addType(base, nextTypeNum++);
- base = nullptr; // in case of exceptions...
- }
- base = deserializeType(buf);
- }
- if (buf.getPos()!=endpos)
- throw MakeStringException(0, "Invalid type info (incorrect size data) in CRtlFieldTypeDeserializer::deserialize");
- buf.setEndian(oldEndian);
- return base;
- }
- catch(...)
- {
- buf.setEndian(oldEndian);
- throw;
- }
- }
- virtual const RtlTypeInfo *addType(FieldTypeInfoStruct &info, const ITypeInfo *type) override
- {
- VStringBuffer name("%p", type);
- const RtlTypeInfo ** found = types.getValue(name);
- if (found)
- return *found;
- info.locale = keep(info.locale);
- const RtlTypeInfo * ret = info.createRtlTypeInfo(callback);
- types.setValue(name, ret);
- return ret;
- }
- virtual const RtlTypeInfo *lookupType(const ITypeInfo *type) const override
- {
- VStringBuffer name("%p", type);
- const RtlTypeInfo ** found = types.getValue(name);
- if (found)
- return *found;
- return nullptr;
- }
- virtual const RtlFieldInfo *addFieldInfo(const char *fieldName, const char *xpath, const RtlTypeInfo *type, unsigned flags, const char *init) override
- {
- // MORE - we could hang onto this for cleanup, rather than assuming that we keep it via a later addType() call?
- return new RtlFieldStrInfo(keep(fieldName), keep(xpath), type, flags, init);
- }
- private:
- KeptAtomTable atoms; // Used to ensure proper lifetime of strings used in type structures
- MapStringTo<const RtlTypeInfo *> types; // Ensures structures only generated once
- const RtlTypeInfo *base = nullptr; // Holds the resulting type
- IThorIndexCallback *callback = nullptr;
- void cleanupType(const RtlTypeInfo *type)
- {
- if (type)
- {
- // Releases all memory for a single RtlTypeInfo object
- const RtlFieldInfo * const * fields = type->queryFields();
- if (fields)
- {
- const RtlFieldInfo * const * cur = fields;
- for (;;)
- {
- const RtlFieldInfo * child = *cur;
- if (!child)
- break;
- // We don't need to delete other strings - they are owned by atom table.
- // But the initializer is decoded and thus owned by me
- delete child->initializer;
- delete child;
- cur++;
- }
- delete [] fields;
- }
- type->doDelete();
- }
- }
- const RtlTypeInfo *lookupType(const char *name, IPropertyTree *all)
- {
- const RtlTypeInfo ** found = types.getValue(name);
- if (found)
- return *found;
- const RtlTypeInfo *type = deserializeType(all->queryPropTree(name), all);
- types.setValue(name, type);
- return type;
- }
- const RtlTypeInfo *lookupType(unsigned idx)
- {
- // Could keep an expanding array of types instead - but the hash table is already there for json support...
- VStringBuffer key("%u", idx);
- const RtlTypeInfo ** found = types.getValue(key);
- if (found)
- return *found;
- throw makeStringException(-1, "Invalid serialized type information");
- }
- void addType(const RtlTypeInfo *type, unsigned idx)
- {
- VStringBuffer key("%u", idx);
- assert(types.getValue(key)==nullptr);
- types.setValue(key, type);
- }
- const char *keep(const char *string)
- {
- if (string)
- return str(atoms.addAtom(string));
- else
- return nullptr;
- }
- const RtlTypeInfo *deserializeType(IPropertyTree *type, IPropertyTree *all)
- {
- FieldTypeInfoStruct info;
- info.fieldType = type->getPropInt("fieldType");
- info.length = type->getPropInt("length");
- info.locale = keep(type->queryProp("locale"));
- const char *child = type->queryProp("child");
- if (child)
- info.childType = lookupType(child, all);
- if ((info.fieldType & RFTMkind) == type_record)
- {
- unsigned numFields = type->getCount("fields");
- info.fieldsArray = new const RtlFieldInfo * [numFields+1];
- info.fieldsArray[numFields] = nullptr;
- Owned<IPropertyTreeIterator> fields = type->getElements("fields");
- unsigned n = 0;
- ForEach(*fields)
- {
- IPropertyTree &field = fields->query();
- const char *fieldTypeName = field.queryProp("type");
- const char *fieldName = keep(field.queryProp("name"));
- const char *fieldXpath = keep(field.queryProp("xpath"));
- unsigned flags = field.getPropInt("flags");
- const char *fieldInit = field.queryProp("init");
- if (fieldInit)
- {
- StringBuffer decoded;
- JBASE64_Decode(fieldInit, decoded);
- fieldInit = decoded.detach(); // NOTE - this gets freed in cleanupType()
- }
- info.fieldsArray[n] = new RtlFieldStrInfo(fieldName, fieldXpath, lookupType(fieldTypeName, all), flags, fieldInit);
- n++;
- }
- }
- return info.createRtlTypeInfo(callback);
- }
- const RtlTypeInfo *deserializeType(MemoryBuffer &type)
- {
- FieldTypeInfoStruct info;
- type.read(info.fieldType);
- type.readPacked(info.length);
- if (info.fieldType & RFTMhasLocale)
- {
- const char *locale;
- type.read(locale);
- info.locale = keep(locale);
- }
- if (info.fieldType & RFTMhasChildType)
- {
- unsigned childIdx;
- type.readPacked(childIdx);
- info.childType = lookupType(childIdx);
- }
- if (info.fieldType & RFTMhasFields)
- {
- unsigned numFields;
- type.readPacked(numFields);
- info.fieldsArray = new const RtlFieldInfo * [numFields+1];
- info.fieldsArray[numFields] = nullptr;
- for (int n = 0; n < numFields; n++)
- {
- const char *fieldName;
- type.read(fieldName);
- unsigned fieldType;
- type.readPacked(fieldType);
- unsigned fieldFlags;
- type.read(fieldFlags);
- const char *xpath = nullptr;
- if (fieldFlags & RFTMhasXpath)
- type.read(xpath);
- void *init = nullptr;
- if (fieldFlags & RFTMhasInitializer)
- {
- unsigned initLength;
- type.readPacked(initLength);
- init = malloc(initLength);
- memcpy(init, type.readDirect(initLength), initLength);
- }
- fieldFlags &= ~RFTMserializerFlags;
- info.fieldsArray[n] = new RtlFieldStrInfo(keep(fieldName), keep(xpath), lookupType(fieldType), fieldFlags, (const char *) init);
- }
- }
- info.fieldType &= ~RFTMserializerFlags;
- return info.createRtlTypeInfo(callback);
- }
- void patchIndexFilePos()
- {
- if (callback && (base->fieldType & RFTMkind) == type_record)
- {
- // Yukky hack time
- // Assumes that the fieldinfo is not shared...
- // But that is also assumed by the code that cleans them up.
- const RtlFieldInfo * const *fields = base->queryFields();
- for(;;)
- {
- const RtlFieldInfo *field = *fields++;
- if (!field)
- break;
- if (field->type->getType() == type_blob)
- {
- static_cast<RtlBlobTypeInfo *>(const_cast<RtlTypeInfo *>(field->type))->setCallback(callback);
- }
- }
- }
- }
- };
- extern ECLRTL_API IRtlFieldTypeDeserializer *createRtlFieldTypeDeserializer(IThorIndexCallback *callback)
- {
- return new CRtlFieldTypeDeserializer(callback);
- }
- extern ECLRTL_API StringBuffer &dumpTypeInfo(StringBuffer &ret, const RtlTypeInfo *t)
- {
- return CRtlFieldTypeSerializer::serialize(ret, t);
- }
- extern ECLRTL_API MemoryBuffer &dumpTypeInfo(MemoryBuffer &ret, const RtlTypeInfo *t)
- {
- return CRtlFieldTypeBinSerializer::serialize(ret, t);
- }
- extern ECLRTL_API void serializeRecordType(size32_t & __lenResult, void * & __result, IOutputMetaData & metaVal)
- {
- MemoryBuffer ret;
- CRtlFieldTypeBinSerializer::serialize(ret, metaVal.queryTypeInfo());
- __lenResult = ret.length();
- __result = ret.detach();
- }
- extern ECLRTL_API void dumpRecordType(size32_t & __lenResult,char * & __result,IOutputMetaData &metaVal)
- {
- StringBuffer ret;
- CRtlFieldTypeSerializer::serialize(ret, metaVal.queryTypeInfo());
- #ifdef _DEBUG
- StringBuffer ret2;
- CRtlFieldTypeDeserializer deserializer(nullptr);
- CRtlFieldTypeSerializer::serialize(ret2, deserializer.deserialize(ret));
- assert(streq(ret, ret2));
- MemoryBuffer out;
- CRtlFieldTypeBinSerializer::serialize(out, metaVal.queryTypeInfo());
- CRtlFieldTypeDeserializer bindeserializer(nullptr);
- CRtlFieldTypeSerializer::serialize(ret2.clear(), bindeserializer.deserialize(out));
- assert(streq(ret, ret2));
- #endif
- __lenResult = ret.length();
- __result = ret.detach();
- }
- extern ECLRTL_API void getFieldVal(size32_t & __lenResult,char * & __result, int column, IOutputMetaData & metaVal, const byte *row)
- {
- __lenResult = 0;
- __result = nullptr;
- if (column >= 0)
- {
- const RtlRecord &r = metaVal.queryRecordAccessor(true);
- unsigned numOffsets = r.getNumVarFields() + 1;
- size_t * variableOffsets = (size_t *)alloca(numOffsets * sizeof(size_t));
- RtlRow offsetCalculator(r, row, numOffsets, variableOffsets);
- offsetCalculator.getUtf8(__lenResult, __result, column);
- }
- }
- extern ECLRTL_API int getFieldNum(const char *fieldName, IOutputMetaData & metaVal)
- {
- const RtlRecord r = metaVal.queryRecordAccessor(true);
- return r.getFieldNum(fieldName);
- }
- enum FieldMatchType {
- // On a field, exactly one of the below is set, but translator returns a bitmap indicating
- // which were required (and we can restrict translation to allow some types but not others)
- match_perfect = 0x00, // exact type match - use memcpy
- match_link = 0x01, // copy a nested dataset by linking
- match_move = 0x02, // at least one field has moved (set on translator)
- match_remove = 0x04, // at least one field has been removed (set on translator)
- match_truncate = 0x08, // dest is truncated copy of source - use memcpy
- match_extend = 0x10, // dest is padded version of source - use memcpy and memset
- match_typecast = 0x20, // type has changed - cast required
- match_none = 0x40, // No matching field in source - use null value
- match_recurse = 0x80, // Use recursive translator for child records/datasets
- match_fail = 0x100, // no translation possible
- // This flag may be set in conjunction with the others
- match_inifblock = 0x200, // matching to a field in an ifblock - may not be present
- };
- StringBuffer &describeFlags(StringBuffer &out, FieldMatchType flags)
- {
- if (flags == match_perfect)
- return out.append("perfect");
- unsigned origlen = out.length();
- if (flags & match_link) out.append("|link");
- if (flags & match_move) out.append("|move");
- if (flags & match_remove) out.append("|remove");
- if (flags & match_truncate) out.append("|truncate");
- if (flags & match_extend) out.append("|extend");
- if (flags & match_typecast) out.append("|typecast");
- if (flags & match_none) out.append("|none");
- if (flags & match_recurse) out.append("|recurse");
- if (flags & match_inifblock) out.append("|ifblock");
- if (flags & match_fail) out.append("|fail");
- assertex(out.length() > origlen);
- return out.remove(origlen, 1);
- }
- inline constexpr FieldMatchType operator|(FieldMatchType a, FieldMatchType b) { return (FieldMatchType)((int)a | (int)b); }
- inline FieldMatchType &operator|=(FieldMatchType &a, FieldMatchType b) { return (FieldMatchType &) ((int &)a |= (int)b); }
- class GeneralRecordTranslator : public CInterfaceOf<IDynamicTransform>
- {
- public:
- GeneralRecordTranslator(const RtlRecord &_destRecInfo, const RtlRecord &_srcRecInfo)
- : destRecInfo(_destRecInfo), sourceRecInfo(_srcRecInfo)
- {
- matchInfo = new MatchInfo[destRecInfo.getNumFields()];
- createMatchInfo();
- }
- ~GeneralRecordTranslator()
- {
- delete [] matchInfo;
- }
- virtual void describe() const override
- {
- doDescribe(0);
- }
- virtual size32_t translate(ARowBuilder &builder, const byte *sourceRec) const override
- {
- dbgassertex(canTranslate());
- return doTranslate(builder, 0, sourceRec);
- }
- virtual bool canTranslate() const override
- {
- return (matchFlags & match_fail) == 0;
- }
- virtual bool needsTranslate() const override
- {
- return (matchFlags & ~match_link) != 0;
- }
- private:
- void doDescribe(unsigned indent) const
- {
- for (unsigned idx = 0; idx < destRecInfo.getNumFields(); idx++)
- {
- const char *source = destRecInfo.queryName(idx);
- const MatchInfo &match = matchInfo[idx];
- if (match.matchType == match_none)
- DBGLOG("%*sNo match for field %s - default value will be used", indent, "", source);
- else
- {
- StringBuffer matchStr;
- DBGLOG("%*sMatch (%s) to field %d for field %s (%x)", indent, "", describeFlags(matchStr, match.matchType).str(), match.matchIdx, source, destRecInfo.queryType(idx)->fieldType);
- if (match.subTrans)
- match.subTrans->doDescribe(indent+2);
- }
- }
- if (!canTranslate())
- DBGLOG("%*sTranslation is NOT possible", indent, "");
- else if (needsTranslate())
- {
- StringBuffer matchStr;
- DBGLOG("%*sTranslation is possible (%s)", indent, "", describeFlags(matchStr, matchFlags).str());
- }
- else
- DBGLOG("%*sTranslation is not necessary", indent, "");
- }
- size32_t doTranslate(ARowBuilder &builder, size32_t offset, const byte *sourceRec) const
- {
- unsigned numOffsets = sourceRecInfo.getNumVarFields() + 1;
- size_t * variableOffsets = (size_t *)alloca(numOffsets * sizeof(size_t));
- byte * destConditions = (byte *)alloca(destRecInfo.getNumIfBlocks() * sizeof(byte));
- RtlRow sourceRow(sourceRecInfo, sourceRec, numOffsets, variableOffsets);
- size32_t estimate = destRecInfo.getFixedSize();
- if (!estimate)
- {
- estimate = estimateNewSize(sourceRow);
- builder.ensureCapacity(offset+estimate, "record");
- }
- size32_t origOffset = offset;
- for (unsigned idx = 0; idx < destRecInfo.getNumFields(); idx++)
- {
- const RtlFieldInfo *field = destRecInfo.queryField(idx);
- if (field->omitable() && destRecInfo.excluded(field, builder.getSelf(), destConditions))
- continue;
- const RtlTypeInfo *type = field->type;
- const MatchInfo &match = matchInfo[idx];
- if (match.matchType == match_none || match.matchType==match_fail)
- offset = type->buildNull(builder, offset, field);
- else
- {
- unsigned matchField = match.matchIdx;
- const RtlTypeInfo *sourceType = sourceRecInfo.queryType(matchField);
- size_t sourceOffset = sourceRow.getOffset(matchField);
- const byte *source = sourceRec + sourceOffset;
- size_t copySize = sourceRow.getSize(matchField);
- if (copySize == 0 && (match.matchType & match_inifblock)) // Field is missing because of an ifblock - use default value
- {
- offset = type->buildNull(builder, offset, field);
- }
- else
- {
- switch (match.matchType & ~match_inifblock)
- {
- case match_perfect:
- {
- // Look ahead for other perfect matches and combine the copies
- while (idx < destRecInfo.getNumFields()-1)
- {
- const MatchInfo &nextMatch = matchInfo[idx+1];
- if (nextMatch.matchType == match_perfect && nextMatch.matchIdx == matchField+1)
- {
- idx++;
- matchField++;
- }
- else
- break;
- }
- size_t copySize = sourceRow.getOffset(matchField+1) - sourceOffset;
- builder.ensureCapacity(offset+copySize, field->name);
- memcpy(builder.getSelf()+offset, source, copySize);
- offset += copySize;
- break;
- }
- case match_truncate:
- {
- assert(type->isFixedSize());
- size32_t copySize = type->getMinSize();
- builder.ensureCapacity(offset+copySize, field->name);
- memcpy(builder.getSelf()+offset, source, copySize);
- offset += copySize;
- break;
- }
- case match_extend:
- {
- assert(type->isFixedSize());
- size32_t destSize = type->getMinSize();
- builder.ensureCapacity(offset+destSize, field->name);
- memcpy(builder.getSelf()+offset, source, copySize);
- offset += copySize;
- unsigned fillSize = destSize - copySize;
- memset(builder.getSelf()+offset, match.fillChar, fillSize);
- offset += fillSize;
- break;
- }
- case match_typecast:
- offset = translateScalar(builder, offset, field, type, sourceType, source);
- break;
- case match_link:
- {
- // a 32-bit record count, and a (linked) pointer to an array of record pointers
- byte *dest = builder.ensureCapacity(offset+sizeof(size32_t)+sizeof(const byte **), field->name)+offset;
- *(size32_t *)dest = *(size32_t *)source;
- *(const byte ***)(dest + sizeof(size32_t)) = rtlLinkRowset(*(const byte ***)(source + sizeof(size32_t)));
- offset += sizeof(size32_t)+sizeof(const byte **);
- break;
- }
- case match_recurse:
- if (type->getType()==type_record)
- offset = match.subTrans->doTranslate(builder, offset, source);
- else if (type->isLinkCounted())
- {
- // a 32-bit record count, and a pointer to an array of record pointers
- IEngineRowAllocator *childAllocator = builder.queryAllocator()->createChildRowAllocator(type->queryChildType());
- assertex(childAllocator); // May not be available when using serialized types (but unlikely to want to create linkcounted children remotely either)
- size32_t sizeInBytes = sizeof(size32_t) + sizeof(void *);
- builder.ensureCapacity(offset+sizeInBytes, field->name);
- size32_t numRows = 0;
- const byte **childRows = nullptr;
- if (sourceType->isLinkCounted())
- {
- // a 32-bit count, then a pointer to the source rows
- size32_t childCount = *(size32_t *) source;
- source += sizeof(size32_t);
- const byte ** sourceRows = *(const byte***) source;
- for (size32_t childRow = 0; childRow < childCount; childRow++)
- {
- RtlDynamicRowBuilder childBuilder(*childAllocator);
- size32_t childLen = match.subTrans->doTranslate(childBuilder, 0, sourceRows[childRow]);
- childRows = childAllocator->appendRowOwn(childRows, ++numRows, (void *) childBuilder.finalizeRowClear(childLen));
- }
- }
- else
- {
- // a 32-bit size, then rows inline
- size32_t childSize = *(size32_t *) source;
- source += sizeof(size32_t);
- const byte *initialSource = source;
- while ((size_t)(source - initialSource) < childSize)
- {
- RtlDynamicRowBuilder childBuilder(*childAllocator);
- size32_t childLen = match.subTrans->doTranslate(childBuilder, 0, source);
- childRows = childAllocator->appendRowOwn(childRows, ++numRows, (void *) childBuilder.finalizeRowClear(childLen));
- source += sourceType->queryChildType()->size(source, nullptr); // MORE - shame to repeat a calculation that the translate above almost certainly just did
- }
- }
- // Go back in and patch the count, remembering it may have moved
- rtlWriteInt4(builder.getSelf()+offset, numRows);
- * ( const void * * ) (builder.getSelf()+offset+sizeof(size32_t)) = childRows;
- offset += sizeInBytes;
- }
- else
- {
- size32_t countOffset = offset;
- byte *dest = builder.ensureCapacity(offset+sizeof(size32_t), field->name)+offset;
- offset += sizeof(size32_t);
- size32_t initialOffset = offset;
- *(size32_t *)dest = 0; // patched below when true figure known
- if (sourceType->isLinkCounted())
- {
- // a 32-bit count, then a pointer to the source rows
- size32_t childCount = *(size32_t *) source;
- source += sizeof(size32_t);
- const byte ** sourceRows = *(const byte***) source;
- for (size32_t childRow = 0; childRow < childCount; childRow++)
- {
- offset = match.subTrans->doTranslate(builder, offset, sourceRows[childRow]);
- }
- }
- else
- {
- // a 32-bit size, then rows inline
- size32_t childSize = *(size32_t *) source;
- source += sizeof(size32_t);
- const byte *initialSource = source;
- while ((size_t)(source - initialSource) < childSize)
- {
- offset = match.subTrans->doTranslate(builder, offset, source);
- source += sourceType->queryChildType()->size(source, nullptr); // MORE - shame to repeat a calculation that the translate above almost certainly just did
- }
- }
- dest = builder.getSelf() + countOffset; // Note - may have been moved by reallocs since last calculated
- *(size32_t *)dest = offset - initialOffset;
- }
- break;
- default:
- throwUnexpected();
- }
- }
- }
- }
- if (estimate && offset-origOffset != estimate)
- {
- // Note - ifblocks make this assertion invalid. We do not account for potentially omitted fields
- // when estimating target record size.
- if (!destRecInfo.getNumIfBlocks())
- assert(offset-origOffset > estimate); // Estimate is always supposed to be conservative
- #ifdef TRACE_TRANSLATION
- DBGLOG("Wrote %u bytes to record (estimate was %u)\n", offset-origOffset, estimate);
- #endif
- }
- return offset;
- }
- inline FieldMatchType match() const
- {
- return matchFlags;
- }
- const RtlRecord &destRecInfo;
- const RtlRecord &sourceRecInfo;
- unsigned fixedDelta = 0; // total size of all fixed-size source fields that are not matched
- UnsignedArray unmatched; // List of all variable-size source fields that are unmatched
- FieldMatchType matchFlags = match_perfect;
- struct MatchInfo
- {
- unsigned matchIdx = 0;
- FieldMatchType matchType = match_fail;
- char fillChar = 0;
- GeneralRecordTranslator *subTrans = nullptr;
- ~MatchInfo()
- {
- delete subTrans;
- }
- } *matchInfo;
- static size32_t translateScalar(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, const RtlTypeInfo *destType, const RtlTypeInfo *sourceType, const byte *source)
- {
- // This code COULD move into rtlfield.cpp?
- switch(destType->getType())
- {
- case type_boolean:
- case type_int:
- case type_swapint:
- case type_packedint:
- case type_filepos:
- offset = destType->buildInt(builder, offset, field, sourceType->getInt(source));
- break;
- case type_real:
- offset = destType->buildReal(builder, offset, field, sourceType->getReal(source));
- break;
- case type_decimal: // Go via string - not common enough to special-case
- case type_data:
- case type_string:
- case type_varstring:
- case type_qstring:
- {
- size32_t size;
- rtlDataAttr text;
- sourceType->getString(size, text.refstr(), source);
- offset = destType->buildString(builder, offset, field, size, text.getstr());
- break;
- }
- case type_unicode:
- case type_varunicode:
- case type_utf8:
- {
- size32_t utf8chars;
- rtlDataAttr utf8Text;
- sourceType->getUtf8(utf8chars, utf8Text.refstr(), source);
- offset = destType->buildUtf8(builder, offset, field, utf8chars, utf8Text.getstr());
- break;
- }
- case type_set:
- {
- bool isAll = *(bool *) source;
- source+= sizeof(bool);
- byte *dest = builder.ensureCapacity(offset+sizeof(bool)+sizeof(size32_t), field->name)+offset;
- *(size32_t *) (dest + sizeof(bool)) = 0; // Patch later when size known
- offset += sizeof(bool) + sizeof(size32_t);
- if (isAll)
- {
- *(bool*) dest = true;
- }
- else
- {
- *(bool*) dest = false;
- size32_t sizeOffset = offset - sizeof(size32_t); // Where we need to patch
- size32_t childSize = *(size32_t *)source;
- source += sizeof(size32_t);
- const byte *initialSource = source;
- size32_t initialOffset = offset;
- const RtlTypeInfo *destChildType = destType->queryChildType();
- const RtlTypeInfo *sourceChildType = sourceType->queryChildType();
- while ((size_t)(source - initialSource) < childSize)
- {
- offset = translateScalar(builder, offset, field, destChildType, sourceChildType, source);
- source += sourceChildType->size(source, nullptr); // MORE - shame to repeat a calculation that the translate above almost certainly just did
- }
- dest = builder.getSelf() + sizeOffset; // Note - man have been moved by reallocs since last calculated
- *(size32_t *)dest = offset - initialOffset;
- }
- break;
- }
- default:
- throwUnexpected();
- }
- return offset;
- }
- size32_t estimateNewSize(const RtlRow &sourceRow) const
- {
- //DBGLOG("Source record size is %d", (int) sourceRow.getRecordSize());
- size32_t expectedSize = sourceRow.getRecordSize() - fixedDelta;
- //DBGLOG("Source record size without omitted fixed size fields is %d", expectedSize);
- ForEachItemIn(i, unmatched)
- {
- unsigned fieldNo = unmatched.item(i);
- expectedSize -= sourceRow.getSize(fieldNo);
- //DBGLOG("Reducing estimated size by %d to %d for omitted field %d (%s)", (int) sourceRow.getSize(fieldNo), expectedSize, fieldNo, sourceRecInfo.queryName(fieldNo));
- }
- if (matchFlags & ~(match_perfect|match_link|match_none|match_extend|match_truncate))
- {
- for (unsigned idx = 0; idx < destRecInfo.getNumFields(); idx++)
- {
- const MatchInfo &match = matchInfo[idx];
- const RtlTypeInfo *type = destRecInfo.queryType(idx);
- unsigned matchField = match.matchIdx;
- switch (match.matchType)
- {
- case match_perfect:
- case match_link:
- case match_none:
- case match_extend:
- case match_truncate:
- // These ones were already included in fixedDelta
- break;
- default:
- // This errs on the side of small - i.e. it assumes that all typecasts end up at minimum size
- // We could do better in some cases e.g. variable string <-> variable unicode we can assume factor of 2,
- // uft8 <-> string we could calculate here - but unlikely to be worth the effort.
- // But it's fine for fixed size output fields, including truncate/extend
- // We could also precalculate the expected delta if all omitted fields are fixed size - but not sure how likely/worthwhile that is.
- expectedSize += type->getMinSize() - sourceRow.getSize(matchField);
- //DBGLOG("Adjusting estimated size by (%d - %d) to %d for translated field %d (%s)", (int) sourceRow.getSize(matchField), type->getMinSize(), expectedSize, matchField, sourceRecInfo.queryName(matchField));
- break;
- }
- }
- }
- return expectedSize;
- }
- void createMatchInfo()
- {
- for (unsigned idx = 0; idx < destRecInfo.getNumFields(); idx++)
- {
- const RtlFieldInfo *field = destRecInfo.queryField(idx);
- const RtlTypeInfo *type = field->type;
- MatchInfo &info = matchInfo[idx];
- info.matchIdx = sourceRecInfo.getFieldNum(destRecInfo.queryName(idx));
- if (info.matchIdx == -1)
- {
- info.matchType = match_none;
- size32_t defaultSize = field->initializer ? type->size(field->initializer, nullptr) : type->getMinSize();
- fixedDelta -= defaultSize;
- //DBGLOG("Decreasing fixedDelta size by %d to %d for defaulted field %d (%s)", defaultSize, fixedDelta, idx, destRecInfo.queryName(idx));
- }
- else
- {
- const RtlTypeInfo *sourceType = sourceRecInfo.queryType(info.matchIdx);
- if (!type->isScalar() || !sourceType->isScalar())
- {
- if (type->getType() != sourceType->getType())
- info.matchType = match_fail; // No translation from one non-scalar type to another
- else
- {
- switch (type->getType())
- {
- case type_set:
- if (type->queryChildType()->fieldType==sourceType->queryChildType()->fieldType &&
- type->queryChildType()->length==sourceType->queryChildType()->length)
- info.matchType = match_perfect;
- else
- info.matchType = match_typecast;
- break;
- case type_row: // These are not expected I think...
- throwUnexpected();
- case type_ifblock:
- case type_record:
- case type_table:
- {
- const RtlRecord *subDest = destRecInfo.queryNested(idx);
- const RtlRecord *subSrc = sourceRecInfo.queryNested(info.matchIdx);
- info.subTrans = new GeneralRecordTranslator(*subDest, *subSrc);
- if (!info.subTrans->needsTranslate())
- {
- // Child does not require translation, but check linkcount mode matches too!
- if (type->isLinkCounted())
- if (sourceType->isLinkCounted())
- info.matchType = match_link;
- else
- info.matchType = match_recurse;
- else
- if (sourceType->isLinkCounted())
- info.matchType = match_recurse;
- else
- info.matchType = match_perfect;
- if (info.matchType != match_recurse)
- {
- delete info.subTrans;
- info.subTrans = nullptr;
- }
- }
- else if (info.subTrans->canTranslate())
- {
- info.matchType = match_recurse;
- matchFlags |= info.subTrans->matchFlags;
- }
- else
- info.matchType = match_fail;
- break;
- }
- default:
- info.matchType = match_fail;
- break;
- }
- }
- }
- else if (type->fieldType==sourceType->fieldType)
- {
- if (type->length==sourceType->length)
- {
- info.matchType = match_perfect;
- }
- else
- {
- assert(type->isFixedSize()); // Both variable size would have matched length above
- info.matchType = match_typecast;
- if (type->length < sourceType->length)
- {
- if (type->canTruncate())
- {
- info.matchType = match_truncate;
- fixedDelta += sourceType->getMinSize()-type->getMinSize();
- //DBGLOG("Increasing fixedDelta size by %d to %d for truncated field %d (%s)", sourceType->getMinSize()-type->getMinSize(), fixedDelta, idx, destRecInfo.queryName(idx));
- }
- }
- else
- {
- if (type->canExtend(info.fillChar))
- {
- info.matchType = match_extend;
- fixedDelta += sourceType->getMinSize()-type->getMinSize();
- //DBGLOG("Decreasing fixedDelta size by %d to %d for truncated field %d (%s)", type->getMinSize()-sourceType->getMinSize(), fixedDelta, idx, destRecInfo.queryName(idx));
- }
- }
- }
- }
- else
- info.matchType = match_typecast;
- if (sourceRecInfo.queryField(info.matchIdx)->flags & RFTMinifblock)
- info.matchType |= match_inifblock; // Avoids incorrect commoning up of adjacent matches
- // MORE - could note the highest interesting fieldnumber in the source and not bother filling in offsets after that
- // Not sure it would help much though - usually need to know the total record size anyway in real life
- if (idx != info.matchIdx)
- matchFlags |= match_move;
- }
- matchFlags |= info.matchType;
- }
- if (sourceRecInfo.getNumFields() > destRecInfo.getNumFields())
- matchFlags |= match_remove;
- if (matchFlags && !destRecInfo.getFixedSize())
- {
- for (unsigned idx = 0; idx < sourceRecInfo.getNumFields(); idx++)
- {
- const RtlFieldInfo *field = sourceRecInfo.queryField(idx);
- const RtlTypeInfo *type = field->type;
- if (destRecInfo.getFieldNum(field->name) == (unsigned) -1)
- {
- // unmatched field
- if (type->isFixedSize())
- {
- //DBGLOG("Reducing estimated size by %d for (fixed size) omitted field %s", (int) type->getMinSize(), field->name);
- fixedDelta += type->getMinSize();
- }
- else
- unmatched.append(idx);
- }
- }
- //DBGLOG("Source record contains %d bytes of omitted fixed size fields", fixedDelta);
- }
- }
- };
- extern ECLRTL_API const IDynamicTransform *createRecordTranslator(const RtlRecord &_destRecInfo, const RtlRecord &_srcRecInfo)
- {
- return new GeneralRecordTranslator(_destRecInfo, _srcRecInfo);
- }
- class TranslatedRowStream : public CInterfaceOf<IRowStream>
- {
- public:
- TranslatedRowStream(IRowStream *_inputStream, IEngineRowAllocator *_resultAllocator, const RtlRecord &outputRecord, const RtlRecord &inputRecord)
- : inputStream(_inputStream), resultAllocator(_resultAllocator), translator(new GeneralRecordTranslator(outputRecord, inputRecord))
- {
- translator->describe();
- }
- virtual const void *nextRow()
- {
- if (eof)
- return NULL;
- const void *inRow = inputStream->nextRow();
- if (!inRow)
- {
- if (eogSeen)
- eof = true;
- else
- eogSeen = true;
- return nullptr;
- }
- else
- eogSeen = false;
- RtlDynamicRowBuilder rowBuilder(resultAllocator);
- size32_t len = translator->translate(rowBuilder, (const byte *) inRow);
- rtlReleaseRow(inRow);
- return rowBuilder.finalizeRowClear(len);
- }
- virtual void stop() override
- {
- resultAllocator.clear();
- }
- bool canTranslate() const
- {
- return translator->canTranslate();
- }
- bool needsTranslate() const
- {
- return translator->needsTranslate();
- }
- protected:
- Linked<IRowStream> inputStream;
- Linked<IEngineRowAllocator> resultAllocator;
- Owned<const IDynamicTransform> translator;
- unsigned numOffsets = 0;
- size_t * variableOffsets = nullptr;
- bool eof = false;
- bool eogSeen = false;
- };
- extern ECLRTL_API IRowStream * transformRecord(IEngineRowAllocator * resultAllocator,IOutputMetaData & metaInput,IRowStream * input)
- {
- if (resultAllocator->queryOutputMeta()==&metaInput)
- return LINK(input);
- Owned<TranslatedRowStream> stream = new TranslatedRowStream(input, resultAllocator,
- resultAllocator->queryOutputMeta()->queryRecordAccessor(true),
- metaInput.queryRecordAccessor(true));
- if (!stream->needsTranslate())
- return LINK(input);
- else if (!stream->canTranslate())
- rtlFail(0, "Cannot translate record stream");
- else
- return stream.getClear();
- }
|