Big-Data-HPC-Platform/ecllibrary/std/DataPatterns/BestRecordStructure.ecl

/***
 * Function macro that leverages DataPatterns to return a string defining the
 * best ECL record structure for the input data.
 *
 * @param   inFile          The dataset to process; REQUIRED
 * @param   sampling        A positive integer representing a percentage of
 *                          inFile to examine, which is useful when analyzing a
 *                          very large dataset and only an estimatation is
 *                          sufficient; valid range for this argument is
 *                          1-100; values outside of this range will be
 *                          clamped; OPTIONAL, defaults to 100 (which indicates
 *                          that the entire dataset will be analyzed)
 * @param   emitTransform   Boolean governing whether the function emits a
 *                          TRANSFORM function that could be used to rewrite
 *                          the dataset into the 'best' record definition;
 *                          OPTIONAL, defaults to FALSE.
 * @param   textOutput      Boolean governing the type of result that is
 *                          delivered by this function; if FALSE then a
 *                          recordset of STRINGs will be returned; if TRUE
 *                          then a dataset with a single STRING field, with
 *                          the contents formatted for HTML, will be
 *                          returned (this is the ideal output if the
 *                          intention is to copy the output from ECL Watch);
 *                          OPTIONAL, defaults to FALSE
 *
 * @return  A recordset defining the best ECL record structure for the data.
 *          If textOutput is FALSE (the default) then each record will contain
 *          one field declaration, and the list of declarations will be wrapped
 *          with RECORD and END strings; if the emitTransform argument was
 *          TRUE, there will also be a set of records that that comprise a
 *          stand-alone TRANSFORM function.  If textOutput is TRUE then only
 *          one record will be returned, containing an HTML-formatted string
 *          containing the new field declarations (and optionally the
 *          TRANSFORM); this is the ideal format if the intention is to copy
 *          the result from ECL Watch.
 */
EXPORT BestRecordStructure(inFile, sampling = 100, emitTransform = FALSE, textOutput = FALSE) := FUNCTIONMACRO
    LOADXML('<xml/>');
    #EXPORTXML(bestInFileFields, RECORDOF(inFile));
    #UNIQUENAME(bestFieldStack);
    #UNIQUENAME(bestStructType);
    #UNIQUENAME(bestLayoutType);
    #UNIQUENAME(bestCapturedPos);
    #UNIQUENAME(bestPrevCapturedPos);
    #UNIQUENAME(bestLayoutName);
    #UNIQUENAME(bestNeedsDelim);
    #UNIQUENAME(bestNamePrefix);
    #UNIQUENAME(recLevel);

    IMPORT Std;

    LOCAL DATAREC_NAME := 'DataRec';
    LOCAL LAYOUT_NAME := 'Layout';

    LOCAL StringRec := {STRING s};

    // Helper function for determining if old and new data types need
    // explicit type casting
    LOCAL NeedCoercion(STRING oldType, STRING newType) := FUNCTION
        GenericType(STRING theType) := MAP
            (
                theType[..6] = 'string'                 =>  'string',
                theType[..13] = 'ebcdic string'         =>  'string',
                theType[..7] = 'qstring'                =>  'string',
                theType[..9] = 'varstring'              =>  'string',
                theType[..3] = 'utf'                    =>  'string',
                theType[..7] = 'unicode'                =>  'string',
                theType[..10] = 'varunicode'            =>  'string',
                theType[..4] = 'data'                   =>  'data',
                theType[..7] = 'boolean'                =>  'boolean',
                theType[..7] = 'integer'                =>  'numeric',
                theType[..18] = 'big_endian integer'    =>  'numeric',
                theType[..4] = 'real'                   =>  'numeric',
                theType[..7] = 'decimal'                =>  'numeric',
                theType[..8] = 'udecimal'               =>  'numeric',
                theType[..8] = 'unsigned'               =>  'numeric',
                theType[..19] = 'big_endian unsigned'   =>  'numeric',
                theType
            );

        oldGenericType := GenericType(Std.Str.ToLowerCase(oldType));
        newGenericType := GenericType(Std.Str.ToLowerCase(newType));

        RETURN oldGenericType != newGenericType;
    END;

    // Build a dataset containing information about embedded records and
    // child datasets; we need to track the beginning and ending positions
    // of the fields defined within those structures
    LOCAL ChildRecInfoLayout := RECORD
        STRING      layoutType;
        STRING      layoutName;
        STRING      fieldName;
        UNSIGNED2   startPos;
        UNSIGNED2   endPos;
    END;

    LOCAL childRecInfo := DATASET
        (
            [
                #SET(bestFieldStack, '')
                #SET(bestNeedsDelim, 0)
                #FOR(bestInFileFields)
                    #FOR(Field)
                        #IF(%{@isRecord}% = 1 OR %{@isDataset}% = 1)
                            #IF(%{@isRecord}% = 1)
                                #SET(bestStructType, 'r')
                            #ELSE
                                #SET(bestStructType, 'd')
                            #END
                            #IF(%'bestFieldStack'% != '')
                                #SET(bestFieldStack, ';' + %'bestFieldStack'%)
                            #END
                            #SET(bestFieldStack, %'bestStructType'% + ':' + %'@position'% + ':' + %'@ecltype'% + %'bestFieldStack'%)
                        #ELSEIF(%{@isEnd}% = 1)
                            #SET(bestLayoutType, %'bestFieldStack'%[1])
                            #SET(bestCapturedPos, REGEXFIND('.:(\\d+)', %'bestFieldStack'%, 1))
                            #SET(bestLayoutName, REGEXFIND('.:\\d+:([^;]+)', %'bestFieldStack'%, 1))
                            #SET(bestFieldStack, REGEXFIND('^[^;]+;(.*)', %'bestFieldStack'%, 1))

                            #IF(%bestNeedsDelim% = 1) , #END

                            {
                                %'bestLayoutType'%,
                                %'bestLayoutName'%,
                                %'@name'%,
                                %bestCapturedPos%,
                                %bestPrevCapturedPos%
                            }

                            #SET(bestNeedsDelim, 1)
                        #ELSE
                            #SET(bestPrevCapturedPos, %@position%)
                        #END
                    #END
                #END
            ],
            ChildRecInfoLayout
        );

    // Extract the original data type and position of the fields within the
    // input dataset
    LOCAL FieldInfoLayout := RECORD
        STRING      eclType;
        STRING      name;
        STRING      fullName;
        BOOLEAN     isRecord;
        BOOLEAN     isDataset;
        UNSIGNED2   depth;
        UNSIGNED2   position;
    END;

    LOCAL fieldInfo0 := DATASET
        (
            [
                #SET(bestFieldStack, '')
                #SET(bestNeedsDelim, 0)
                #SET(bestNamePrefix, '')
                #SET(recLevel, 0)
                #FOR(bestInFileFields)
                    #FOR(Field)
                        #IF(%@isEnd% != 1)
                            #IF(%bestNeedsDelim% = 1) , #END

                            {
                                %'@ecltype'%,
                                %'@name'%,
                                %'bestNamePrefix'% + %'@name'%,
                                #IF(%@isRecord% = 1) TRUE #ELSE FALSE #END,
                                #IF(%@isDataset% = 1) TRUE #ELSE FALSE #END,
                                %recLevel%,
                                %@position%
                            }

                            #SET(bestNeedsDelim, 1)
                        #END

                        #IF(%{@isRecord}% = 1 OR %{@isDataset}% = 1)
                            #APPEND(bestNamePrefix, %'@name'% + '.')
                            #SET(recLevel, %recLevel% + 1)
                        #ELSEIF(%{@isEnd}% = 1)
                            #SET(bestNamePrefix, REGEXREPLACE('\\w+\\.$', %'bestNamePrefix'%, ''))
                            #SET(recLevel, %recLevel% - 1)
                        #END
                    #END
                #END
            ],
            FieldInfoLayout
        );

    // Attach the record end positions for embedded records and child datasets
    LOCAL fieldInfo10 := JOIN
        (
            fieldInfo0,
            childRecInfo,
            LEFT.name = RIGHT.fieldName AND LEFT.position = RIGHT.startPos,
            TRANSFORM
                (
                    {
                        RECORDOF(LEFT),
                        UNSIGNED2   endPosition
                    },
                    SELF.endPosition := RIGHT.endPos,
                    SELF := LEFT
                ),
            LEFT OUTER
        );

    // Get the best data types from the Profile() function
    LOCAL patternRes := DataPatterns.Profile(inFile, features := 'best_ecl_types', sampleSize := sampling);

    // Append the derived 'best' data types to the field information we
    // already collected
    LOCAL fieldInfo15 := JOIN
        (
            fieldInfo10,
            patternRes,
            LEFT.fullName = RIGHT.attribute,
            TRANSFORM
                (
                    {
                        RECORDOF(LEFT),
                        STRING      bestAttributeType
                    },
                    SELF.bestAttributeType := IF(RIGHT.best_attribute_type != '', Std.Str.ToUpperCase(RIGHT.best_attribute_type), LEFT.eclType),
                    SELF := LEFT
                ),
            LEFT OUTER
        );

    // Determine fields that must have explicit coercion if we are supplying
    // transform information
    LOCAL fieldInfo20 := PROJECT
        (
            fieldInfo15,
            TRANSFORM
                (
                    {
                        RECORDOF(LEFT),
                        STRING      bestAssignment
                    },
                    shouldRewriteType := ((LEFT.isDataset OR LEFT.isRecord) AND LEFT.bestAttributeType IN ['<unnamed>', 'table of <unnamed>']);
                    tempDSName := DATAREC_NAME + '_' + INTFORMAT(COUNTER, 4, 1);
                    SELF.eclType := IF(NOT shouldRewriteType, Std.Str.ToUpperCase(LEFT.eclType), tempDSName),
                    SELF.bestAttributeType := IF(NOT shouldRewriteType, LEFT.bestAttributeType, tempDSName),
                    SELF.bestAssignment := IF
                        (
                            NeedCoercion(SELF.eclType, SELF.bestAttributeType),
                            '    SELF.' + LEFT.name + ' := (' + Std.Str.ToUppercase(SELF.bestAttributeType) + ')r.' + LEFT.name + ';',
                            ''
                        ),
                    SELF := LEFT
                )
        );

    LOCAL LayoutItems := RECORD(StringRec)
        STRING                  fullName := '';
        STRING                  bestAssignment := '';
    END;

    LOCAL ChildRecLayout := RECORD
        STRING                  layoutName;
        UNSIGNED2               startPos;
        UNSIGNED2               endPos;
        UNSIGNED2               depth;
        DATASET(LayoutItems)    items;
    END;

    // Function for creating ECL TRANSFORM assignment statements
    LOCAL MakeRecDefinition(DATASET(RECORDOF(fieldInfo20)) ds, STRING layoutName, BOOLEAN useBest = TRUE) := FUNCTION
        displayPrefix := IF(useBest, 'New', 'Old');
        displayedLayoutName := displayPrefix + layoutName;
        RETURN DATASET([{displayedLayoutName + ' := RECORD'}], LayoutItems)
            & PROJECT
                (
                    SORT(ds, position),
                    TRANSFORM
                        (
                            LayoutItems,
                            attrType := IF(useBest, LEFT.bestAttributeType, LEFT.eclType);
                            attrPrefix := IF(LEFT.isDataset OR LEFT.isRecord, displayPrefix, '');
                            fullAttrType := attrPrefix + attrType;
                            namedDataType := IF(NOT LEFT.isDataset, fullAttrType, 'DATASET(' + fullAttrType + ')');
                            SELF.s := '    ' + namedDataType + ' ' + LEFT.name + ';',
                            SELF.bestAssignment := MAP
                                (
                                    LEFT.bestAssignment != ''   =>  LEFT.bestAssignment,
                                    LEFT.isRecord   =>  '    SELF.' + LEFT.name + ' := ROW(Make_' + fullAttrType + '(r.' + LEFT.name + '));',
                                    LEFT.isDataset  =>  '    SELF.' + LEFT.name + ' := PROJECT(r.' + LEFT.name + ', Make_' + fullAttrType + '(LEFT));',
                                    ''
                                ),
                            SELF := LEFT
                        )
                )
            & DATASET([{'END;'}], LayoutItems);
    END;

    // Iteratively process embedded records and child dataset definitions,
    // extracting each into its own record
    LOCAL ProcessChildRecs(DATASET(ChildRecLayout) layoutDS, UNSIGNED2 aDepth, BOOLEAN useBest = TRUE) := FUNCTION
        bestNamedChildRecs := DENORMALIZE
            (
                fieldInfo20(depth = (aDepth - 1) AND (isRecord OR isDataset)),
                fieldInfo20(depth = aDepth),
                RIGHT.position BETWEEN LEFT.position + 1 AND LEFT.endPosition,
                GROUP,
                TRANSFORM
                    (
                        ChildRecLayout,
                        SELF.layoutName := LEFT.bestAttributeType,
                        SELF.items := MakeRecDefinition(ROWS(RIGHT), SELF.layoutName, useBest),
                        SELF.startPos := LEFT.position,
                        SELF.endPos := LEFT.endPosition,
                        SELF.depth := aDepth,
                        SELF := LEFT
                    ),
                ALL, ORDERED(TRUE)
            ) : ONWARNING(4531, IGNORE);

        RETURN layoutDS + bestNamedChildRecs;
    END;

    // Create a list of embedded records and child dataset definitions for the
    // original input dataset
    LOCAL oldNamedChildRecs0 := LOOP
        (
            DATASET([], ChildRecLayout),
            MAX(fieldInfo20, depth),
            ProcessChildRecs(ROWS(LEFT), MAX(fieldInfo20, depth) + 1 - COUNTER, FALSE)
        );

    LOCAL oldNamedChildRecs := SORT(oldNamedChildRecs0, endPos, -startPos);

    LOCAL topLevelOldRecDef := DATASET
        (
            [
                {
                    LAYOUT_NAME,
                    0,
                    0,
                    0,
                    MakeRecDefinition(fieldInfo20(depth = 0), LAYOUT_NAME, FALSE)
                }
            ],
            ChildRecLayout
        );

    LOCAL allOldRecDefs := oldNamedChildRecs & topLevelOldRecDef;

    // Create a list of embedded records and child dataset definitions using the
    // the recommended ECL datatypes
    LOCAL bestNamedChildRecs0 := LOOP
        (
            DATASET([], ChildRecLayout),
            MAX(fieldInfo20, depth),
            ProcessChildRecs(ROWS(LEFT), MAX(fieldInfo20, depth) + 1 - COUNTER, TRUE)
        );

    LOCAL bestNamedChildRecs := SORT(bestNamedChildRecs0, endPos, -startPos);

    LOCAL topLevelBestRecDef := DATASET
        (
            [
                {
                    LAYOUT_NAME,
                    0,
                    0,
                    0,
                    MakeRecDefinition(fieldInfo20(depth = 0), LAYOUT_NAME, TRUE)
                }
            ],
            ChildRecLayout
        );

    LOCAL allBestRecDefs := bestNamedChildRecs & topLevelBestRecDef;

    // Creates an ECL TRANSFORM function based on the collected information
    // about a record definition
    LOCAL MakeTransforms(ChildRecLayout recInfo) := FUNCTION
        RETURN DATASET(['New' + recInfo.layoutName + ' Make_New' + recInfo.layoutName + '(Old' + recInfo.layoutName + ' r) := TRANSFORM'], StringRec)
            & PROJECT
                (
                    recInfo.items,
                    TRANSFORM
                        (
                            StringRec,
                            assignment := LEFT.bestAssignment;
                            SELF.s := IF(assignment != '', assignment, SKIP)
                        )
                )
            & DATASET(['    SELF := r;'], StringRec)
            & DATASET(['END;'], StringRec);
    END;

    LOCAL allTransforms := PROJECT
        (
            allBestRecDefs,
            TRANSFORM
                (
                    {
                        DATASET(StringRec)  lines
                    },
                    SELF.lines := MakeTransforms(LEFT)
                )
        );

    // Create a dataset of STRINGS that contain record definitions for the
    // input dataset, TRANSFORMs for converting between the old and new
    // definitions, and a sample PROJECT for kicking it all off
    LOCAL conditionalBR := #IF((BOOLEAN)textOutput) '<br/>' #ELSE '' #END;

    LOCAL oldRecDefsPlusTransforms := DATASET(['//----------' + conditionalBR], StringRec)
        & PROJECT(allOldRecDefs.items, StringRec)
        & DATASET(['//----------' + conditionalBR], StringRec)
        & allTransforms.lines
        & DATASET(['//----------' + conditionalBR], StringRec)
        & DATASET(['oldDS := DATASET([], OldLayout);' + conditionalBR], StringRec)
        & DATASET(['newDS := PROJECT(oldDS, Make_NewLayout(LEFT));' + conditionalBR], StringRec);

    // Combine old definitions and transforms conditionally
    LOCAL conditionalOldStuff :=
        #IF((BOOLEAN)emitTransform)
            oldRecDefsPlusTransforms
        #ELSE
            DATASET([], StringRec)
        #END;

    LOCAL allOutput := PROJECT(allBestRecDefs.items, StringRec) & conditionalOldStuff;

    // Roll everything up to one string with HTML line breaks
    LOCAL htmlString := ROLLUP
        (
            allOutput,
            TRUE,
            TRANSFORM
                (
                    RECORDOF(LEFT),
                    rightString := IF(RIGHT.s = 'END;', RIGHT.s + '<br/>', RIGHT.s);
                    SELF.s := LEFT.s + '<br/>' + rightString
                )
        );

    // Stuff the HTML result into a single record, wrapped with <pre> so it
    // looks right in the browser
    LOCAL htmlResult := DATASET(['<pre>' + htmlString[1].s + '</pre>'], {STRING result__html});

    // Choose the result (dataset with each line a string, or a text blob)
    LOCAL finalResult := #IF((BOOLEAN)textOutput) htmlResult #ELSE allOutput #END;

    RETURN finalResult;
ENDMACRO;