HPCC-22093 Add DataPatterns to STD

Add UI to LogicalFile Details page
Removed unused JS file(Graph7Widget.js)
Fix "reset" on DelayLoadWidget

Signed-off-by: Gordon Smith <gordonjsmith@gmail.com>

ecllibrary/std/CMakeLists.txt

@@ -1,5 +1,5 @@
 ################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -14,6 +14,7 @@
 #    limitations under the License.
 ################################################################################
 
+add_subdirectory(DataPatterns)
 add_subdirectory(system)
 
 set(
@@ -28,10 +29,10 @@ set(
     Metaphone.ecl
     Str.ecl
     Uni.ecl
-    )
+)
 
 foreach(module ${SRCS})
     SIGN_MODULE(${module})
     install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${module} DESTINATION share/ecllibrary/std COMPONENT Runtime)
 endforeach()
-  
+

ecllibrary/std/DataPatterns/BestRecordStructure.ecl

@@ -0,0 +1,451 @@
+/***
+ * Function macro that leverages DataPatterns to return a string defining the
+ * best ECL record structure for the input data.
+ *
+ * @param   inFile          The dataset to process; REQUIRED
+ * @param   sampling        A positive integer representing a percentage of
+ *                          inFile to examine, which is useful when analyzing a
+ *                          very large dataset and only an estimatation is
+ *                          sufficient; valid range for this argument is
+ *                          1-100; values outside of this range will be
+ *                          clamped; OPTIONAL, defaults to 100 (which indicates
+ *                          that the entire dataset will be analyzed)
+ * @param   emitTransform   Boolean governing whether the function emits a
+ *                          TRANSFORM function that could be used to rewrite
+ *                          the dataset into the 'best' record definition;
+ *                          OPTIONAL, defaults to FALSE.
+ * @param   textOutput      Boolean governing the type of result that is
+ *                          delivered by this function; if FALSE then a
+ *                          recordset of STRINGs will be returned; if TRUE
+ *                          then a dataset with a single STRING field, with
+ *                          the contents formatted for HTML, will be
+ *                          returned (this is the ideal output if the
+ *                          intention is to copy the output from ECL Watch);
+ *                          OPTIONAL, defaults to FALSE
+ *
+ * @return  A recordset defining the best ECL record structure for the data.
+ *          If textOutput is FALSE (the default) then each record will contain
+ *          one field declaration, and the list of declarations will be wrapped
+ *          with RECORD and END strings; if the emitTransform argument was
+ *          TRUE, there will also be a set of records that that comprise a
+ *          stand-alone TRANSFORM function.  If textOutput is TRUE then only
+ *          one record will be returned, containing an HTML-formatted string
+ *          containing the new field declarations (and optionally the
+ *          TRANSFORM); this is the ideal format if the intention is to copy
+ *          the result from ECL Watch.
+ */
+EXPORT BestRecordStructure(inFile, sampling = 100, emitTransform = FALSE, textOutput = FALSE) := FUNCTIONMACRO
+    LOADXML('<xml/>');
+    #EXPORTXML(bestInFileFields, RECORDOF(inFile));
+    #UNIQUENAME(bestFieldStack);
+    #UNIQUENAME(bestStructType);
+    #UNIQUENAME(bestLayoutType);
+    #UNIQUENAME(bestCapturedPos);
+    #UNIQUENAME(bestPrevCapturedPos);
+    #UNIQUENAME(bestLayoutName);
+    #UNIQUENAME(bestNeedsDelim);
+    #UNIQUENAME(bestNamePrefix);
+    #UNIQUENAME(recLevel);
+
+    IMPORT Std;
+
+    LOCAL DATAREC_NAME := 'DataRec';
+    LOCAL LAYOUT_NAME := 'Layout';
+
+    LOCAL StringRec := {STRING s};
+
+    // Helper function for determining if old and new data types need
+    // explicit type casting
+    LOCAL NeedCoercion(STRING oldType, STRING newType) := FUNCTION
+        GenericType(STRING theType) := MAP
+            (
+                theType[..6] = 'string'                 =>  'string',
+                theType[..13] = 'ebcdic string'         =>  'string',
+                theType[..7] = 'qstring'                =>  'string',
+                theType[..9] = 'varstring'              =>  'string',
+                theType[..3] = 'utf'                    =>  'string',
+                theType[..7] = 'unicode'                =>  'string',
+                theType[..10] = 'varunicode'            =>  'string',
+                theType[..4] = 'data'                   =>  'data',
+                theType[..7] = 'boolean'                =>  'boolean',
+                theType[..7] = 'integer'                =>  'numeric',
+                theType[..18] = 'big_endian integer'    =>  'numeric',
+                theType[..4] = 'real'                   =>  'numeric',
+                theType[..7] = 'decimal'                =>  'numeric',
+                theType[..8] = 'udecimal'               =>  'numeric',
+                theType[..8] = 'unsigned'               =>  'numeric',
+                theType[..19] = 'big_endian unsigned'   =>  'numeric',
+                theType
+            );
+
+        oldGenericType := GenericType(Std.Str.ToLowerCase(oldType));
+        newGenericType := GenericType(Std.Str.ToLowerCase(newType));
+
+        RETURN oldGenericType != newGenericType;
+    END;
+
+    // Build a dataset containing information about embedded records and
+    // child datasets; we need to track the beginning and ending positions
+    // of the fields defined within those structures
+    LOCAL ChildRecInfoLayout := RECORD
+        STRING      layoutType;
+        STRING      layoutName;
+        STRING      fieldName;
+        UNSIGNED2   startPos;
+        UNSIGNED2   endPos;
+    END;
+
+    LOCAL childRecInfo := DATASET
+        (
+            [
+                #SET(bestFieldStack, '')
+                #SET(bestNeedsDelim, 0)
+                #FOR(bestInFileFields)
+                    #FOR(Field)
+                        #IF(%{@isRecord}% = 1 OR %{@isDataset}% = 1)
+                            #IF(%{@isRecord}% = 1)
+                                #SET(bestStructType, 'r')
+                            #ELSE
+                                #SET(bestStructType, 'd')
+                            #END
+                            #IF(%'bestFieldStack'% != '')
+                                #SET(bestFieldStack, ';' + %'bestFieldStack'%)
+                            #END
+                            #SET(bestFieldStack, %'bestStructType'% + ':' + %'@position'% + ':' + %'@ecltype'% + %'bestFieldStack'%)
+                        #ELSEIF(%{@isEnd}% = 1)
+                            #SET(bestLayoutType, %'bestFieldStack'%[1])
+                            #SET(bestCapturedPos, REGEXFIND('.:(\\d+)', %'bestFieldStack'%, 1))
+                            #SET(bestLayoutName, REGEXFIND('.:\\d+:([^;]+)', %'bestFieldStack'%, 1))
+                            #SET(bestFieldStack, REGEXFIND('^[^;]+;(.*)', %'bestFieldStack'%, 1))
+
+                            #IF(%bestNeedsDelim% = 1) , #END
+
+                            {
+                                %'bestLayoutType'%,
+                                %'bestLayoutName'%,
+                                %'@name'%,
+                                %bestCapturedPos%,
+                                %bestPrevCapturedPos%
+                            }
+
+                            #SET(bestNeedsDelim, 1)
+                        #ELSE
+                            #SET(bestPrevCapturedPos, %@position%)
+                        #END
+                    #END
+                #END
+            ],
+            ChildRecInfoLayout
+        );
+
+    // Extract the original data type and position of the fields within the
+    // input dataset
+    LOCAL FieldInfoLayout := RECORD
+        STRING      eclType;
+        STRING      name;
+        STRING      fullName;
+        BOOLEAN     isRecord;
+        BOOLEAN     isDataset;
+        UNSIGNED2   depth;
+        UNSIGNED2   position;
+    END;
+
+    LOCAL fieldInfo0 := DATASET
+        (
+            [
+                #SET(bestFieldStack, '')
+                #SET(bestNeedsDelim, 0)
+                #SET(bestNamePrefix, '')
+                #SET(recLevel, 0)
+                #FOR(bestInFileFields)
+                    #FOR(Field)
+                        #IF(%@isEnd% != 1)
+                            #IF(%bestNeedsDelim% = 1) , #END
+
+                            {
+                                %'@ecltype'%,
+                                %'@name'%,
+                                %'bestNamePrefix'% + %'@name'%,
+                                #IF(%@isRecord% = 1) TRUE #ELSE FALSE #END,
+                                #IF(%@isDataset% = 1) TRUE #ELSE FALSE #END,
+                                %recLevel%,
+                                %@position%
+                            }
+
+                            #SET(bestNeedsDelim, 1)
+                        #END
+
+                        #IF(%{@isRecord}% = 1 OR %{@isDataset}% = 1)
+                            #APPEND(bestNamePrefix, %'@name'% + '.')
+                            #SET(recLevel, %recLevel% + 1)
+                        #ELSEIF(%{@isEnd}% = 1)
+                            #SET(bestNamePrefix, REGEXREPLACE('\\w+\\.$', %'bestNamePrefix'%, ''))
+                            #SET(recLevel, %recLevel% - 1)
+                        #END
+                    #END
+                #END
+            ],
+            FieldInfoLayout
+        );
+
+    // Attach the record end positions for embedded records and child datasets
+    LOCAL fieldInfo10 := JOIN
+        (
+            fieldInfo0,
+            childRecInfo,
+            LEFT.name = RIGHT.fieldName AND LEFT.position = RIGHT.startPos,
+            TRANSFORM
+                (
+                    {
+                        RECORDOF(LEFT),
+                        UNSIGNED2   endPosition
+                    },
+                    SELF.endPosition := RIGHT.endPos,
+                    SELF := LEFT
+                ),
+            LEFT OUTER
+        );
+
+    // Get the best data types from the Profile() function
+    LOCAL patternRes := DataPatterns.Profile(inFile, features := 'best_ecl_types', sampleSize := sampling);
+
+    // Append the derived 'best' data types to the field information we
+    // already collected
+    LOCAL fieldInfo15 := JOIN
+        (
+            fieldInfo10,
+            patternRes,
+            LEFT.fullName = RIGHT.attribute,
+            TRANSFORM
+                (
+                    {
+                        RECORDOF(LEFT),
+                        STRING      bestAttributeType
+                    },
+                    SELF.bestAttributeType := IF(RIGHT.best_attribute_type != '', Std.Str.ToUpperCase(RIGHT.best_attribute_type), LEFT.eclType),
+                    SELF := LEFT
+                ),
+            LEFT OUTER
+        );
+
+    // Determine fields that must have explicit coercion if we are supplying
+    // transform information
+    LOCAL fieldInfo20 := PROJECT
+        (
+            fieldInfo15,
+            TRANSFORM
+                (
+                    {
+                        RECORDOF(LEFT),
+                        STRING      bestAssignment
+                    },
+                    shouldRewriteType := ((LEFT.isDataset OR LEFT.isRecord) AND LEFT.bestAttributeType IN ['<unnamed>', 'table of <unnamed>']);
+                    tempDSName := DATAREC_NAME + '_' + INTFORMAT(COUNTER, 4, 1);
+                    SELF.eclType := IF(NOT shouldRewriteType, Std.Str.ToUpperCase(LEFT.eclType), tempDSName),
+                    SELF.bestAttributeType := IF(NOT shouldRewriteType, LEFT.bestAttributeType, tempDSName),
+                    SELF.bestAssignment := IF
+                        (
+                            NeedCoercion(SELF.eclType, SELF.bestAttributeType),
+                            '    SELF.' + LEFT.name + ' := (' + Std.Str.ToUppercase(SELF.bestAttributeType) + ')r.' + LEFT.name + ';',
+                            ''
+                        ),
+                    SELF := LEFT
+                )
+        );
+
+    LOCAL LayoutItems := RECORD(StringRec)
+        STRING                  fullName := '';
+        STRING                  bestAssignment := '';
+    END;
+
+    LOCAL ChildRecLayout := RECORD
+        STRING                  layoutName;
+        UNSIGNED2               startPos;
+        UNSIGNED2               endPos;
+        UNSIGNED2               depth;
+        DATASET(LayoutItems)    items;
+    END;
+
+    // Function for creating ECL TRANSFORM assignment statements
+    LOCAL MakeRecDefinition(DATASET(RECORDOF(fieldInfo20)) ds, STRING layoutName, BOOLEAN useBest = TRUE) := FUNCTION
+        displayPrefix := IF(useBest, 'New', 'Old');
+        displayedLayoutName := displayPrefix + layoutName;
+        RETURN DATASET([{displayedLayoutName + ' := RECORD'}], LayoutItems)
+            & PROJECT
+                (
+                    SORT(ds, position),
+                    TRANSFORM
+                        (
+                            LayoutItems,
+                            attrType := IF(useBest, LEFT.bestAttributeType, LEFT.eclType);
+                            attrPrefix := IF(LEFT.isDataset OR LEFT.isRecord, displayPrefix, '');
+                            fullAttrType := attrPrefix + attrType;
+                            namedDataType := IF(NOT LEFT.isDataset, fullAttrType, 'DATASET(' + fullAttrType + ')');
+                            SELF.s := '    ' + namedDataType + ' ' + LEFT.name + ';',
+                            SELF.bestAssignment := MAP
+                                (
+                                    LEFT.bestAssignment != ''   =>  LEFT.bestAssignment,
+                                    LEFT.isRecord   =>  '    SELF.' + LEFT.name + ' := ROW(Make_' + fullAttrType + '(r.' + LEFT.name + '));',
+                                    LEFT.isDataset  =>  '    SELF.' + LEFT.name + ' := PROJECT(r.' + LEFT.name + ', Make_' + fullAttrType + '(LEFT));',
+                                    ''
+                                ),
+                            SELF := LEFT
+                        )
+                )
+            & DATASET([{'END;'}], LayoutItems);
+    END;
+
+    // Iteratively process embedded records and child dataset definitions,
+    // extracting each into its own record
+    LOCAL ProcessChildRecs(DATASET(ChildRecLayout) layoutDS, UNSIGNED2 aDepth, BOOLEAN useBest = TRUE) := FUNCTION
+        bestNamedChildRecs := DENORMALIZE
+            (
+                fieldInfo20(depth = (aDepth - 1) AND (isRecord OR isDataset)),
+                fieldInfo20(depth = aDepth),
+                RIGHT.position BETWEEN LEFT.position + 1 AND LEFT.endPosition,
+                GROUP,
+                TRANSFORM
+                    (
+                        ChildRecLayout,
+                        SELF.layoutName := LEFT.bestAttributeType,
+                        SELF.items := MakeRecDefinition(ROWS(RIGHT), SELF.layoutName, useBest),
+                        SELF.startPos := LEFT.position,
+                        SELF.endPos := LEFT.endPosition,
+                        SELF.depth := aDepth,
+                        SELF := LEFT
+                    ),
+                ALL, ORDERED(TRUE)
+            ) : ONWARNING(4531, IGNORE);
+
+        RETURN layoutDS + bestNamedChildRecs;
+    END;
+
+    // Create a list of embedded records and child dataset definitions for the
+    // original input dataset
+    LOCAL oldNamedChildRecs0 := LOOP
+        (
+            DATASET([], ChildRecLayout),
+            MAX(fieldInfo20, depth),
+            ProcessChildRecs(ROWS(LEFT), MAX(fieldInfo20, depth) + 1 - COUNTER, FALSE)
+        );
+
+    LOCAL oldNamedChildRecs := SORT(oldNamedChildRecs0, endPos, -startPos);
+
+    LOCAL topLevelOldRecDef := DATASET
+        (
+            [
+                {
+                    LAYOUT_NAME,
+                    0,
+                    0,
+                    0,
+                    MakeRecDefinition(fieldInfo20(depth = 0), LAYOUT_NAME, FALSE)
+                }
+            ],
+            ChildRecLayout
+        );
+
+    LOCAL allOldRecDefs := oldNamedChildRecs & topLevelOldRecDef;
+
+    // Create a list of embedded records and child dataset definitions using the
+    // the recommended ECL datatypes
+    LOCAL bestNamedChildRecs0 := LOOP
+        (
+            DATASET([], ChildRecLayout),
+            MAX(fieldInfo20, depth),
+            ProcessChildRecs(ROWS(LEFT), MAX(fieldInfo20, depth) + 1 - COUNTER, TRUE)
+        );
+
+    LOCAL bestNamedChildRecs := SORT(bestNamedChildRecs0, endPos, -startPos);
+
+    LOCAL topLevelBestRecDef := DATASET
+        (
+            [
+                {
+                    LAYOUT_NAME,
+                    0,
+                    0,
+                    0,
+                    MakeRecDefinition(fieldInfo20(depth = 0), LAYOUT_NAME, TRUE)
+                }
+            ],
+            ChildRecLayout
+        );
+
+    LOCAL allBestRecDefs := bestNamedChildRecs & topLevelBestRecDef;
+
+    // Creates an ECL TRANSFORM function based on the collected information
+    // about a record definition
+    LOCAL MakeTransforms(ChildRecLayout recInfo) := FUNCTION
+        RETURN DATASET(['New' + recInfo.layoutName + ' Make_New' + recInfo.layoutName + '(Old' + recInfo.layoutName + ' r) := TRANSFORM'], StringRec)
+            & PROJECT
+                (
+                    recInfo.items,
+                    TRANSFORM
+                        (
+                            StringRec,
+                            assignment := LEFT.bestAssignment;
+                            SELF.s := IF(assignment != '', assignment, SKIP)
+                        )
+                )
+            & DATASET(['    SELF := r;'], StringRec)
+            & DATASET(['END;'], StringRec);
+    END;
+
+    LOCAL allTransforms := PROJECT
+        (
+            allBestRecDefs,
+            TRANSFORM
+                (
+                    {
+                        DATASET(StringRec)  lines
+                    },
+                    SELF.lines := MakeTransforms(LEFT)
+                )
+        );
+
+    // Create a dataset of STRINGS that contain record definitions for the
+    // input dataset, TRANSFORMs for converting between the old and new
+    // definitions, and a sample PROJECT for kicking it all off
+    LOCAL conditionalBR := #IF((BOOLEAN)textOutput) '<br/>' #ELSE '' #END;
+
+    LOCAL oldRecDefsPlusTransforms := DATASET(['//----------' + conditionalBR], StringRec)
+        & PROJECT(allOldRecDefs.items, StringRec)
+        & DATASET(['//----------' + conditionalBR], StringRec)
+        & allTransforms.lines
+        & DATASET(['//----------' + conditionalBR], StringRec)
+        & DATASET(['oldDS := DATASET([], OldLayout);' + conditionalBR], StringRec)
+        & DATASET(['newDS := PROJECT(oldDS, Make_NewLayout(LEFT));' + conditionalBR], StringRec);
+
+    // Combine old definitions and transforms conditionally
+    LOCAL conditionalOldStuff :=
+        #IF((BOOLEAN)emitTransform)
+            oldRecDefsPlusTransforms
+        #ELSE
+            DATASET([], StringRec)
+        #END;
+
+    LOCAL allOutput := PROJECT(allBestRecDefs.items, StringRec) & conditionalOldStuff;
+
+    // Roll everything up to one string with HTML line breaks
+    LOCAL htmlString := ROLLUP
+        (
+            allOutput,
+            TRUE,
+            TRANSFORM
+                (
+                    RECORDOF(LEFT),
+                    rightString := IF(RIGHT.s = 'END;', RIGHT.s + '<br/>', RIGHT.s);
+                    SELF.s := LEFT.s + '<br/>' + rightString
+                )
+        );
+
+    // Stuff the HTML result into a single record, wrapped with <pre> so it
+    // looks right in the browser
+    LOCAL htmlResult := DATASET(['<pre>' + htmlString[1].s + '</pre>'], {STRING result__html});
+
+    // Choose the result (dataset with each line a string, or a text blob)
+    LOCAL finalResult := #IF((BOOLEAN)textOutput) htmlResult #ELSE allOutput #END;
+
+    RETURN finalResult;
+ENDMACRO;

ecllibrary/std/DataPatterns/CMakeLists.txt

@@ -0,0 +1,26 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+set(SRCS
+    BestRecordStructure.ecl
+    Profile.ecl
+)
+
+foreach(module ${SRCS})
+    SIGN_MODULE(${module})
+    INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/${module} DESTINATION share/ecllibrary/std/DataPatterns COMPONENT Runtime)
+endforeach()
+

ecllibrary/teststd/DataPatterns/TestDataPatterns.ecl

@@ -0,0 +1,598 @@
+IMPORT Std;
+
+EXPORT TestDataPatterns := MODULE
+
+    //--------------------------------------------------------------------------
+    // Useful functions
+    //--------------------------------------------------------------------------
+
+    SHARED ValueForAttr(ds, attrNameStr, f) := FUNCTIONMACRO
+        RETURN ds(attribute = attrNameStr)[1].f;
+    ENDMACRO;
+
+    //--------------------------------------------------------------------------
+    // Baseline string test
+    //--------------------------------------------------------------------------
+
+    SHARED Basic_String := DATASET
+        (
+            [
+                'Dan', 'Steve', '', 'Mike', 'Dan', 'Sebastian', 'Dan'
+            ],
+            {STRING s}
+        );
+
+    SHARED Basic_String_Profile := Std.DataPatterns.Profile(NOFOLD(Basic_String));
+
+    EXPORT Test_Basic_String_Profile :=
+        [
+            ASSERT(Basic_String_Profile[1].attribute = 's'),
+            ASSERT(Basic_String_Profile[1].rec_count = 7),
+            ASSERT(Basic_String_Profile[1].given_attribute_type = 'string'),
+            ASSERT((DECIMAL9_6)Basic_String_Profile[1].fill_rate = (DECIMAL9_6)85.714286),
+            ASSERT(Basic_String_Profile[1].fill_count = 6),
+            ASSERT(Basic_String_Profile[1].cardinality = 4),
+            ASSERT(Basic_String_Profile[1].best_attribute_type = 'string9'),
+            ASSERT(COUNT(Basic_String_Profile[1].modes) = 1),
+            ASSERT(Basic_String_Profile[1].modes[1].value = 'Dan'),
+            ASSERT(Basic_String_Profile[1].modes[1].rec_count = 3),
+            ASSERT(Basic_String_Profile[1].min_length = 3),
+            ASSERT(Basic_String_Profile[1].max_length = 9),
+            ASSERT(Basic_String_Profile[1].ave_length = 4),
+            ASSERT(COUNT(Basic_String_Profile[1].popular_patterns) = 4),
+            ASSERT(Basic_String_Profile[1].popular_patterns[1].data_pattern = 'Aaa'),
+            ASSERT(Basic_String_Profile[1].popular_patterns[1].rec_count = 3),
+            ASSERT(Basic_String_Profile[1].popular_patterns[2].data_pattern = 'Aaaa'),
+            ASSERT(Basic_String_Profile[1].popular_patterns[2].rec_count = 1),
+            ASSERT(Basic_String_Profile[1].popular_patterns[3].data_pattern = 'Aaaaa'),
+            ASSERT(Basic_String_Profile[1].popular_patterns[3].rec_count = 1),
+            ASSERT(Basic_String_Profile[1].popular_patterns[4].data_pattern = 'Aaaaaaaaa'),
+            ASSERT(Basic_String_Profile[1].popular_patterns[4].rec_count = 1),
+            ASSERT(COUNT(Basic_String_Profile[1].rare_patterns) = 0),
+            ASSERT(Basic_String_Profile[1].is_numeric = FALSE),
+            ASSERT(Basic_String_Profile[1].numeric_min = 0),
+            ASSERT(Basic_String_Profile[1].numeric_max = 0),
+            ASSERT(Basic_String_Profile[1].numeric_mean = 0),
+            ASSERT(Basic_String_Profile[1].numeric_std_dev = 0),
+            ASSERT(Basic_String_Profile[1].numeric_lower_quartile = 0),
+            ASSERT(Basic_String_Profile[1].numeric_median = 0),
+            ASSERT(Basic_String_Profile[1].numeric_upper_quartile = 0),
+            ASSERT(COUNT(Basic_String_Profile[1].numeric_correlations) = 0),
+            ASSERT(TRUE)
+        ];
+
+    //--------------------------------------------------------------------------
+    // Baseline numeric test
+    //--------------------------------------------------------------------------
+
+    SHARED Basic_Numeric := DATASET
+        (
+            [
+                -1000, 500, -250, 2000, 1500, -2000, 2000
+            ],
+            {INTEGER n}
+        );
+
+    SHARED Basic_Numeric_Profile := Std.DataPatterns.Profile(NOFOLD(Basic_Numeric));
+
+    EXPORT Test_Basic_Numeric_Profile :=
+        [
+            ASSERT(Basic_Numeric_Profile[1].attribute = 'n'),
+            ASSERT(Basic_Numeric_Profile[1].rec_count = 7),
+            ASSERT(Basic_Numeric_Profile[1].given_attribute_type = 'integer8'),
+            ASSERT(Basic_Numeric_Profile[1].fill_rate = 100),
+            ASSERT(Basic_Numeric_Profile[1].fill_count = 7),
+            ASSERT(Basic_Numeric_Profile[1].cardinality = 6),
+            ASSERT(Basic_Numeric_Profile[1].best_attribute_type = 'integer8'),
+            ASSERT(COUNT(Basic_Numeric_Profile[1].modes) = 1),
+            ASSERT(Basic_Numeric_Profile[1].modes[1].value = '2000'),
+            ASSERT(Basic_Numeric_Profile[1].modes[1].rec_count = 2),
+            ASSERT(Basic_Numeric_Profile[1].min_length = 3),
+            ASSERT(Basic_Numeric_Profile[1].max_length = 5),
+            ASSERT(Basic_Numeric_Profile[1].ave_length = 4),
+            ASSERT(COUNT(Basic_Numeric_Profile[1].popular_patterns) = 4),
+            ASSERT(Basic_Numeric_Profile[1].popular_patterns[1].data_pattern = '9999'),
+            ASSERT(Basic_Numeric_Profile[1].popular_patterns[1].rec_count = 3),
+            ASSERT(Basic_Numeric_Profile[1].popular_patterns[2].data_pattern = '-9999'),
+            ASSERT(Basic_Numeric_Profile[1].popular_patterns[2].rec_count = 2),
+            ASSERT(Basic_Numeric_Profile[1].popular_patterns[3].data_pattern = '-999'),
+            ASSERT(Basic_Numeric_Profile[1].popular_patterns[3].rec_count = 1),
+            ASSERT(Basic_Numeric_Profile[1].popular_patterns[4].data_pattern = '999'),
+            ASSERT(Basic_Numeric_Profile[1].popular_patterns[4].rec_count = 1),
+            ASSERT(COUNT(Basic_Numeric_Profile[1].rare_patterns) = 0),
+            ASSERT(Basic_Numeric_Profile[1].is_numeric = TRUE),
+            ASSERT(Basic_Numeric_Profile[1].numeric_min = -2000),
+            ASSERT(Basic_Numeric_Profile[1].numeric_max = 2000),
+            ASSERT(Basic_Numeric_Profile[1].numeric_mean = 392.8571),
+            ASSERT(Basic_Numeric_Profile[1].numeric_std_dev = 1438.3593),
+            ASSERT(Basic_Numeric_Profile[1].numeric_lower_quartile = -1000),
+            ASSERT(Basic_Numeric_Profile[1].numeric_median = 500),
+            ASSERT(Basic_Numeric_Profile[1].numeric_upper_quartile = 2000),
+            ASSERT(COUNT(Basic_Numeric_Profile[1].numeric_correlations) = 0),
+            ASSERT(TRUE)
+        ];
+
+    //--------------------------------------------------------------------------
+    // Empty data detection
+    //--------------------------------------------------------------------------
+
+    // Layout contains every ECL data type that Profile can deal with
+    SHARED EmptyDataLayout := RECORD
+        BOOLEAN f_boolean;
+        INTEGER f_integer;
+        UNSIGNED f_unsigned;
+        UNSIGNED INTEGER f_unsigned_integer;
+        BIG_ENDIAN INTEGER f_big_endian_integer;
+        BIG_ENDIAN UNSIGNED f_big_endian_unsigned;
+        BIG_ENDIAN UNSIGNED INTEGER f_big_endian_unsigned_integer;
+        LITTLE_ENDIAN INTEGER f_little_endian_integer;
+        LITTLE_ENDIAN UNSIGNED f_little_endian_unsigned;
+        LITTLE_ENDIAN UNSIGNED INTEGER f_little_endian_unsigned_integer;
+        REAL f_real;
+        DECIMAL32 f_decimal32;
+        DECIMAL32_6 f_decimal32_6;
+        UDECIMAL32 f_udecimal32;
+        UDECIMAL32_6 f_udecimal32_6;
+        UNSIGNED DECIMAL32 f_unsigned_decimal32;
+        UNSIGNED DECIMAL32_6 f_unsigned_decimal32_6;
+        STRING f_string;
+        STRING256 f_string256;
+        ASCII STRING f_ascii_string;
+        ASCII STRING256 f_ascii_string256;
+        EBCDIC STRING f_ebcdic_string;
+        EBCDIC STRING256 f_ebcdic_string256;
+        QSTRING f_qstring;
+        QSTRING256 f_qstring256;
+        UNICODE f_unicode;
+        UNICODE_de f_unicode_de;
+        UNICODE256 f_unicode256;
+        UNICODE_de256 f_unicode_de256;
+        UTF8 f_utf8;
+        UTF8_de f_utf8_de;
+        DATA f_data;
+        DATA16 f_data16;
+        VARSTRING f_varstring;
+        VARSTRING256 f_varstring256;
+        VARUNICODE f_varunicode;
+        VARUNICODE_de f_varunicode_de;
+        VARUNICODE256 f_varunicode256;
+        VARUNICODE_de256 f_varunicode_de256;
+    END;
+
+    SHARED Empty_Data := DATASET
+        (
+            1,
+            TRANSFORM
+                (
+                    EmptyDataLayout,
+                    SELF := []
+                )
+        );
+
+    SHARED Empty_Data_Profile := Std.DataPatterns.Profile(NOFOLD(Empty_Data), features := 'cardinality,best_ecl_types,lengths,modes,patterns');
+
+    // Convenience function for testing the same thing for several attributes
+    SHARED TestEmptyAttr(STRING attributeName) :=
+        [
+            ASSERT(ValueForAttr(Empty_Data_Profile, attributeName, cardinality) = 0),
+            ASSERT(ValueForAttr(Empty_Data_Profile, attributeName, best_attribute_type) = ValueForAttr(Empty_Data_Profile, attributeName, given_attribute_type)),
+            ASSERT(ValueForAttr(Empty_Data_Profile, attributeName, min_length) = 0),
+            ASSERT(ValueForAttr(Empty_Data_Profile, attributeName, max_length) = 0),
+            ASSERT(ValueForAttr(Empty_Data_Profile, attributeName, ave_length) = 0),
+            ASSERT(COUNT(ValueForAttr(Empty_Data_Profile, attributeName, popular_patterns)) = 0),
+            ASSERT(COUNT(ValueForAttr(Empty_Data_Profile, attributeName, rare_patterns)) = 0),
+            ASSERT(TRUE)
+        ];
+
+    EXPORT Test_Empty_Data_Profile :=
+        [
+            TestEmptyAttr('f_ascii_string'),
+            TestEmptyAttr('f_ascii_string16'),
+            TestEmptyAttr('f_big_endian_integer'),
+            TestEmptyAttr('f_big_endian_unsigned'),
+            TestEmptyAttr('f_big_endian_unsigned_integer'),
+            TestEmptyAttr('f_data'),
+            TestEmptyAttr('f_decimal32'),
+            TestEmptyAttr('f_decimal32_6'),
+            TestEmptyAttr('f_ebcdic_string'),
+            TestEmptyAttr('f_ebcdic_string16'),
+            TestEmptyAttr('f_integer'),
+            TestEmptyAttr('f_little_endian_integer'),
+            TestEmptyAttr('f_little_endian_unsigned'),
+            TestEmptyAttr('f_little_endian_unsigned_integer'),
+            TestEmptyAttr('f_qstring'),
+            TestEmptyAttr('f_qstring16'),
+            TestEmptyAttr('f_real'),
+            TestEmptyAttr('f_string'),
+            TestEmptyAttr('f_string16'),
+            TestEmptyAttr('f_udecimal32'),
+            TestEmptyAttr('f_udecimal32_6'),
+            TestEmptyAttr('f_unicode'),
+            TestEmptyAttr('f_unicode16'),
+            TestEmptyAttr('f_unicode_de'),
+            TestEmptyAttr('f_unicode_de16'),
+            TestEmptyAttr('f_unsigned'),
+            TestEmptyAttr('f_unsigned_decimal32'),
+            TestEmptyAttr('f_unsigned_decimal32_6'),
+            TestEmptyAttr('f_unsigned_integer'),
+            TestEmptyAttr('f_utf8'),
+            TestEmptyAttr('f_utf8_de'),
+            TestEmptyAttr('f_varstring'),
+            TestEmptyAttr('f_varstring16'),
+            TestEmptyAttr('f_varunicode'),
+            TestEmptyAttr('f_varunicode16'),
+            TestEmptyAttr('f_varunicode_de'),
+            TestEmptyAttr('f_varunicode_de16'),
+
+            // Handle BOOLEAN special because it is not truly empty
+            ASSERT(ValueForAttr(Empty_Data_Profile, 'f_boolean', cardinality) = 1),
+            ASSERT(ValueForAttr(Empty_Data_Profile, 'f_boolean', best_attribute_type) = ValueForAttr(Empty_Data_Profile, 'f_boolean', given_attribute_type)),
+            ASSERT(ValueForAttr(Empty_Data_Profile, 'f_boolean', min_length) = 1),
+            ASSERT(ValueForAttr(Empty_Data_Profile, 'f_boolean', max_length) = 1),
+            ASSERT(ValueForAttr(Empty_Data_Profile, 'f_boolean', ave_length) = 1),
+            ASSERT(COUNT(ValueForAttr(Empty_Data_Profile, 'f_boolean', popular_patterns)) = 1),
+            ASSERT(COUNT(ValueForAttr(Empty_Data_Profile, 'f_boolean', rare_patterns)) = 0),
+
+            // Handle fixed-length DATA special because it is not truly empty
+            ASSERT(ValueForAttr(Empty_Data_Profile, 'f_data16', cardinality) = 1),
+            ASSERT(ValueForAttr(Empty_Data_Profile, 'f_data16', best_attribute_type) = ValueForAttr(Empty_Data_Profile, 'f_data16', given_attribute_type)),
+            ASSERT(ValueForAttr(Empty_Data_Profile, 'f_data16', min_length) = 16),
+            ASSERT(ValueForAttr(Empty_Data_Profile, 'f_data16', max_length) = 16),
+            ASSERT(ValueForAttr(Empty_Data_Profile, 'f_data16', ave_length) = 16),
+            ASSERT(COUNT(ValueForAttr(Empty_Data_Profile, 'f_data16', popular_patterns)) = 1),
+            ASSERT(COUNT(ValueForAttr(Empty_Data_Profile, 'f_data16', rare_patterns)) = 0),
+            ASSERT(TRUE)
+        ];
+
+    //--------------------------------------------------------------------------
+    // Unicode pattern detection
+    //--------------------------------------------------------------------------
+
+    SHARED Pattern_Unicode := DATASET
+        (
+            [
+                U'abcd\353', U'ABCDË'
+            ],
+            {UNICODE_de5 s}
+        );
+
+    SHARED Pattern_Unicode_Profile := Std.DataPatterns.Profile(NOFOLD(Pattern_Unicode), features := 'patterns');
+
+    EXPORT Test_Pattern_Unicode_Profile :=
+        [
+            ASSERT(Pattern_Unicode_Profile[1].given_attribute_type = 'unicode_de5'),
+            ASSERT(COUNT(Pattern_Unicode_Profile[1].popular_patterns) = 2),
+            ASSERT(Pattern_Unicode_Profile[1].popular_patterns[1].data_pattern = 'AAAAA'),
+            ASSERT(Pattern_Unicode_Profile[1].popular_patterns[1].rec_count = 1),
+            ASSERT(Pattern_Unicode_Profile[1].popular_patterns[2].data_pattern = 'aaaaa'),
+            ASSERT(Pattern_Unicode_Profile[1].popular_patterns[2].rec_count = 1),
+            ASSERT(TRUE)
+        ];
+
+    //--------------------------------------------------------------------------
+    // Punctuation pattern test
+    //--------------------------------------------------------------------------
+
+    SHARED Pattern_Punctuation := DATASET
+        (
+            [
+                'This! Is- Not. Helpful?'
+            ],
+            {STRING s}
+        );
+
+    SHARED Pattern_Punctuation_Profile := Std.DataPatterns.Profile(NOFOLD(Pattern_Punctuation), features := 'patterns');
+
+    EXPORT Test_Pattern_Punctuation_Profile :=
+        [
+            ASSERT(Pattern_Punctuation_Profile[1].attribute = 's'),
+            ASSERT(COUNT(Pattern_Punctuation_Profile[1].popular_patterns) = 1),
+            ASSERT(Pattern_Punctuation_Profile[1].popular_patterns[1].data_pattern = 'Aaaa! Aa- Aaa. Aaaaaaa?'),
+            ASSERT(Pattern_Punctuation_Profile[1].popular_patterns[1].rec_count = 1),
+            ASSERT(TRUE)
+        ];
+
+    //--------------------------------------------------------------------------
+    // Finding integers within strings test
+    //--------------------------------------------------------------------------
+
+    SHARED Best_Integer := DATASET
+        (
+            [
+                {'-100', '-100', '-1000', '-10000', '-100000'},
+                {'100', '100', '1000', '10000', '100000'}
+            ],
+            {STRING s1, STRING s2, STRING s3, STRING s4, STRING s5}
+        );
+
+    SHARED Best_Integer_Profile := Std.DataPatterns.Profile(NOFOLD(Best_Integer), features := 'best_ecl_types');
+
+    EXPORT Test_Best_Integer_Profile :=
+        [
+            ASSERT(ValueForAttr(Best_Integer_Profile, 's1', best_attribute_type) = 'integer2'),
+            ASSERT(ValueForAttr(Best_Integer_Profile, 's2', best_attribute_type) = 'integer2'),
+            ASSERT(ValueForAttr(Best_Integer_Profile, 's3', best_attribute_type) = 'integer3'),
+            ASSERT(ValueForAttr(Best_Integer_Profile, 's4', best_attribute_type) = 'integer3'),
+            ASSERT(ValueForAttr(Best_Integer_Profile, 's5', best_attribute_type) = 'integer4'),
+            ASSERT(TRUE)
+        ];
+
+    //--------------------------------------------------------------------------
+    // Finding unsigned integers within strings test
+    //--------------------------------------------------------------------------
+
+    SHARED Best_Unsigned := DATASET
+        (
+            [
+                {'100', '100', '1000', '10000', '100000'}
+            ],
+            {STRING s1, STRING s2, STRING s3, STRING s4, STRING s5}
+        );
+
+    SHARED Best_Unsigned_Profile := Std.DataPatterns.Profile(NOFOLD(Best_Unsigned), features := 'best_ecl_types');
+
+    EXPORT Test_Best_Unsigned_Profile :=
+        [
+            ASSERT(ValueForAttr(Best_Unsigned_Profile, 's1', best_attribute_type) = 'unsigned2'),
+            ASSERT(ValueForAttr(Best_Unsigned_Profile, 's2', best_attribute_type) = 'unsigned2'),
+            ASSERT(ValueForAttr(Best_Unsigned_Profile, 's3', best_attribute_type) = 'unsigned2'),
+            ASSERT(ValueForAttr(Best_Unsigned_Profile, 's4', best_attribute_type) = 'unsigned3'),
+            ASSERT(ValueForAttr(Best_Unsigned_Profile, 's5', best_attribute_type) = 'unsigned3'),
+            ASSERT(TRUE)
+        ];
+
+    //--------------------------------------------------------------------------
+    // Finding reals within strings test
+    //--------------------------------------------------------------------------
+
+    SHARED Best_Real := DATASET
+        (
+            [
+                {'99.99', '-99.99', '9.1234e-10', '.123', '99.0'}
+            ],
+            {STRING s1, STRING s2, STRING s3, STRING s4, STRING s5}
+        );
+
+    SHARED Best_Real_Profile := Std.DataPatterns.Profile(NOFOLD(Best_Real), features := 'best_ecl_types');
+
+    EXPORT Test_Best_Real_Profile :=
+        [
+            ASSERT(ValueForAttr(Best_Real_Profile, 's1', best_attribute_type) = 'real4'),
+            ASSERT(ValueForAttr(Best_Real_Profile, 's2', best_attribute_type) = 'real4'),
+            ASSERT(ValueForAttr(Best_Real_Profile, 's3', best_attribute_type) = 'real8'),
+            ASSERT(ValueForAttr(Best_Real_Profile, 's4', best_attribute_type) = 'real4'),
+            ASSERT(ValueForAttr(Best_Real_Profile, 's5', best_attribute_type) = 'real4'),
+            ASSERT(TRUE)
+        ];
+
+    //--------------------------------------------------------------------------
+    // Not actually numbers test
+    //--------------------------------------------------------------------------
+
+    SHARED Best_NaN := DATASET
+        (
+            [
+                {'123456789012345678901', '-12345678901234567890', '9.1234e-1000', '99.1234567890123456', '123456789012345678901.0'}
+            ],
+            {STRING s1, STRING s2, STRING s3, STRING s4, STRING s5}
+        );
+
+    SHARED Best_NaN_Profile := Std.DataPatterns.Profile(NOFOLD(Best_NaN), features := 'best_ecl_types');
+
+    EXPORT Test_Best_NaN_Profile :=
+        [
+            ASSERT(ValueForAttr(Best_NaN_Profile, 's1', best_attribute_type) = 'string21'),
+            ASSERT(ValueForAttr(Best_NaN_Profile, 's2', best_attribute_type) = 'string21'),
+            ASSERT(ValueForAttr(Best_NaN_Profile, 's3', best_attribute_type) = 'string12'),
+            ASSERT(ValueForAttr(Best_NaN_Profile, 's4', best_attribute_type) = 'string19'),
+            ASSERT(ValueForAttr(Best_NaN_Profile, 's5', best_attribute_type) = 'string23'),
+            ASSERT(TRUE)
+        ];
+
+    //--------------------------------------------------------------------------
+    // Embedded child record test
+    //--------------------------------------------------------------------------
+
+    SHARED Embedded_Child1 := DATASET
+        (
+            [
+                {'Dan', {123, 345, 567}},
+                {'Mike', {987, 765, 543}}
+            ],
+            {STRING s, {UNSIGNED4 x, UNSIGNED4 y, UNSIGNED4 z} foo}
+        );
+
+    SHARED Embedded_Child1_Profile := Std.DataPatterns.Profile(NOFOLD(Embedded_Child1));
+
+    EXPORT Test_Embedded_Child1_Profile :=
+        [
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.x')[1].attribute = 'foo.x'),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.x')[1].rec_count = 2),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.x')[1].given_attribute_type = 'unsigned4'),
+            ASSERT((DECIMAL9_6)Embedded_Child1_Profile(attribute = 'foo.x')[1].fill_rate = (DECIMAL9_6)100),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.x')[1].fill_count = 2),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.x')[1].cardinality = 2),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.x')[1].best_attribute_type = 'unsigned4'),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.x')[1].min_length = 3),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.x')[1].max_length = 3),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.x')[1].ave_length = 3),
+            ASSERT(COUNT(Embedded_Child1_Profile(attribute = 'foo.x')[1].popular_patterns) = 1),
+            ASSERT(COUNT(Embedded_Child1_Profile(attribute = 'foo.x')[1].rare_patterns) = 0),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.x')[1].is_numeric = TRUE),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.x')[1].numeric_min = 123),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.x')[1].numeric_max = 987),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.x')[1].numeric_mean = 555),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.x')[1].numeric_std_dev = 432),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.x')[1].numeric_lower_quartile = 123),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.x')[1].numeric_median = 555),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.x')[1].numeric_upper_quartile = 0),
+            ASSERT(COUNT(Embedded_Child1_Profile(attribute = 'foo.x')[1].numeric_correlations) = 2),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.y')[1].attribute = 'foo.y'),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.y')[1].rec_count = 2),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.y')[1].given_attribute_type = 'unsigned4'),
+            ASSERT((DECIMAL9_6)Embedded_Child1_Profile(attribute = 'foo.y')[1].fill_rate = (DECIMAL9_6)100),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.y')[1].fill_count = 2),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.y')[1].cardinality = 2),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.y')[1].best_attribute_type = 'unsigned4'),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.y')[1].min_length = 3),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.y')[1].max_length = 3),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.y')[1].ave_length = 3),
+            ASSERT(COUNT(Embedded_Child1_Profile(attribute = 'foo.y')[1].popular_patterns) = 1),
+            ASSERT(COUNT(Embedded_Child1_Profile(attribute = 'foo.y')[1].rare_patterns) = 0),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.y')[1].is_numeric = TRUE),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.y')[1].numeric_min = 345),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.y')[1].numeric_max = 765),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.y')[1].numeric_mean = 555),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.y')[1].numeric_std_dev = 210),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.y')[1].numeric_lower_quartile = 345),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.y')[1].numeric_median = 555),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.y')[1].numeric_upper_quartile = 0),
+            ASSERT(COUNT(Embedded_Child1_Profile(attribute = 'foo.y')[1].numeric_correlations) = 2),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.z')[1].attribute = 'foo.z'),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.z')[1].rec_count = 2),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.z')[1].given_attribute_type = 'unsigned4'),
+            ASSERT((DECIMAL9_6)Embedded_Child1_Profile(attribute = 'foo.z')[1].fill_rate = (DECIMAL9_6)100),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.z')[1].fill_count = 2),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.z')[1].cardinality = 2),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.z')[1].best_attribute_type = 'unsigned4'),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.z')[1].min_length = 3),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.z')[1].max_length = 3),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.z')[1].ave_length = 3),
+            ASSERT(COUNT(Embedded_Child1_Profile(attribute = 'foo.z')[1].popular_patterns) = 1),
+            ASSERT(COUNT(Embedded_Child1_Profile(attribute = 'foo.z')[1].rare_patterns) = 0),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.z')[1].is_numeric = TRUE),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.z')[1].numeric_min = 543),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.z')[1].numeric_max = 567),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.z')[1].numeric_mean = 555),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.z')[1].numeric_std_dev = 12),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.z')[1].numeric_lower_quartile = 543),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.z')[1].numeric_median = 555),
+            ASSERT(Embedded_Child1_Profile(attribute = 'foo.z')[1].numeric_upper_quartile = 0),
+            ASSERT(COUNT(Embedded_Child1_Profile(attribute = 'foo.z')[1].numeric_correlations) = 2),
+            ASSERT(TRUE)
+        ];
+
+    //--------------------------------------------------------------------------
+    // Test strings fields containing numerics with leading zeros (issue 42)
+    //--------------------------------------------------------------------------
+
+    SHARED Leading_Zeros := DATASET
+        (
+            [
+                {'0100', '1234', '0001', '7809', '-0600'},
+                {'0020', '0001', '0023', '0001', '600'}
+            ],
+            {STRING s1, STRING s2, STRING s3, STRING s4, STRING s5}
+        );
+
+    SHARED Leading_Zeros_Profile := Std.DataPatterns.Profile(NOFOLD(Leading_Zeros), features := 'best_ecl_types');
+
+    EXPORT Test_Leading_Zeros_Profile :=
+        [
+            ASSERT(ValueForAttr(Leading_Zeros_Profile, 's1', best_attribute_type) = 'string4'),
+            ASSERT(ValueForAttr(Leading_Zeros_Profile, 's2', best_attribute_type) = 'string4'),
+            ASSERT(ValueForAttr(Leading_Zeros_Profile, 's3', best_attribute_type) = 'string4'),
+            ASSERT(ValueForAttr(Leading_Zeros_Profile, 's4', best_attribute_type) = 'string4'),
+            ASSERT(ValueForAttr(Leading_Zeros_Profile, 's5', best_attribute_type) = 'integer3'),
+            ASSERT(TRUE)
+        ];
+
+    //--------------------------------------------------------------------------
+    // String fields with wildly varying lengths (three orders of magnitude
+    // difference) should become variable-length 'string' datatypes
+    //--------------------------------------------------------------------------
+
+    SHARED STRING MLS(UNSIGNED4 len) := EMBED(C++)
+        const char  letters[] = "abcdefghijklmnopqrstuvwxyz0123456789";
+
+        __lenResult = len;
+        __result = static_cast<char*>(rtlMalloc(__lenResult));
+
+        for (uint32_t x = 0; x < len; x++)
+            __result[x] = letters[rand() % 36];
+    ENDEMBED;
+
+    SHARED Large_Strings := DATASET
+        (
+            [
+                {'abcd', '1234', '0001', '7', '-0600'},
+                {'0020', MLS(5000), MLS(500), MLS(1050), '600'}
+            ],
+            {STRING s1, STRING s2, STRING s3, STRING s4, STRING s5}
+        );
+
+    SHARED Large_Strings_Profile := Std.DataPatterns.Profile(NOFOLD(Large_Strings), features := 'best_ecl_types');
+
+    EXPORT Test_Large_Strings_Profile :=
+        [
+            ASSERT(ValueForAttr(Large_Strings_Profile, 's1', best_attribute_type) = 'string4'),
+            ASSERT(ValueForAttr(Large_Strings_Profile, 's2', best_attribute_type) = 'string'),
+            ASSERT(ValueForAttr(Large_Strings_Profile, 's3', best_attribute_type) = 'string500'),
+            ASSERT(ValueForAttr(Large_Strings_Profile, 's4', best_attribute_type) = 'string'),
+            ASSERT(ValueForAttr(Large_Strings_Profile, 's5', best_attribute_type) = 'integer3'),
+            ASSERT(TRUE)
+        ];
+
+    //--------------------------------------------------------------------------
+    // Test strings fields containing numerics with leading zeros (issue 42)
+    //--------------------------------------------------------------------------
+
+    SHARED SetOf_Types := DATASET
+        (
+            [
+                {1, [1,2,3,4]},
+                {100, [9,8]},
+                {200, [4,4,4,4,4,4,4,4,4,4,4]},
+                {300, []},
+                {150, [5,6]}
+            ],
+            {
+                UNSIGNED2           n,
+                SET OF UNSIGNED2    my_set
+            }
+        );
+
+    SHARED SetOf_Types_Profile := Std.DataPatterns.Profile(NOFOLD(SetOf_Types));
+
+    EXPORT Test_SetOf_Types_Profile :=
+        [
+            ASSERT(SetOf_Types_Profile(attribute = 'my_set')[1].attribute = 'my_set'),
+            ASSERT(SetOf_Types_Profile(attribute = 'my_set')[1].rec_count = 5),
+            ASSERT(SetOf_Types_Profile(attribute = 'my_set')[1].given_attribute_type = 'set of unsigned2'),
+            ASSERT((DECIMAL9_6)SetOf_Types_Profile(attribute = 'my_set')[1].fill_rate = (DECIMAL9_6)80),
+            ASSERT(SetOf_Types_Profile(attribute = 'my_set')[1].fill_count = 4),
+            ASSERT(SetOf_Types_Profile(attribute = 'my_set')[1].cardinality = 4),
+            ASSERT(SetOf_Types_Profile(attribute = 'my_set')[1].best_attribute_type = 'set of unsigned2'),
+            ASSERT(SetOf_Types_Profile(attribute = 'my_set')[1].min_length = 2),
+            ASSERT(SetOf_Types_Profile(attribute = 'my_set')[1].max_length = 11),
+            ASSERT(SetOf_Types_Profile(attribute = 'my_set')[1].ave_length = 4),
+            ASSERT(COUNT(SetOf_Types_Profile(attribute = 'my_set')[1].popular_patterns) = 3),
+            ASSERT(COUNT(SetOf_Types_Profile(attribute = 'my_set')[1].rare_patterns) = 0),
+            ASSERT(SetOf_Types_Profile(attribute = 'my_set')[1].is_numeric = FALSE),
+            ASSERT(SetOf_Types_Profile(attribute = 'my_set')[1].numeric_min = 0),
+            ASSERT(SetOf_Types_Profile(attribute = 'my_set')[1].numeric_max = 0),
+            ASSERT(SetOf_Types_Profile(attribute = 'my_set')[1].numeric_mean = 0),
+            ASSERT(SetOf_Types_Profile(attribute = 'my_set')[1].numeric_std_dev = 0),
+            ASSERT(SetOf_Types_Profile(attribute = 'my_set')[1].numeric_lower_quartile = 0),
+            ASSERT(SetOf_Types_Profile(attribute = 'my_set')[1].numeric_median = 0),
+            ASSERT(SetOf_Types_Profile(attribute = 'my_set')[1].numeric_upper_quartile = 0),
+            ASSERT(COUNT(SetOf_Types_Profile(attribute = 'my_set')[1].numeric_correlations) = 0),
+            ASSERT(TRUE)
+        ];
+
+    EXPORT Main := [
+        EVALUATE(Test_Basic_String_Profile), 
+        EVALUATE(Test_Basic_Numeric_Profile),
+        EVALUATE(Test_Empty_Data_Profile),
+        EVALUATE(Test_Pattern_Unicode_Profile),
+        EVALUATE(Test_Pattern_Punctuation_Profile),
+        EVALUATE(Test_Best_Integer_Profile),
+        EVALUATE(Test_Best_Unsigned_Profile),
+        EVALUATE(Test_Best_Real_Profile),
+        EVALUATE(Test_Best_NaN_Profile),
+        EVALUATE(Test_Embedded_Child1_Profile),
+        EVALUATE(Test_Leading_Zeros_Profile),
+        EVALUATE(Test_Large_Strings_Profile),
+        EVALUATE(Test_SetOf_Types_Profile)
+    ];
+END;

esp/src/eclwatch/DelayLoadWidget.js

@@ -69,7 +69,7 @@ define([
             },
 
             //  Implementation  ---
-            reset:function() {
+            reset: function () {
                 for (var key in this.widget) {
                     this.widget[key].destroyRecursive();
                     delete this.widget[key];
@@ -79,6 +79,8 @@ define([
                 delete this.__hpcc_initalized;
                 delete this.childWidgetID;
                 this.containerNode.innerHTML = "";
+                delete this.__initPromise;
+                delete this.__ensurePromise;
             },
 
             init: function (params) {
@@ -109,7 +111,7 @@ define([
                 }
             },
             doRestoreFromHash: function (hash) {
-                if (this.widget[this.childWidgetID].restoreFromHash) {
+                if (this.widget && this.widget[this.childWidgetID] && this.widget[this.childWidgetID].restoreFromHash) {
                     this.widget[this.childWidgetID].restoreFromHash(hash);
                 }
             }

esp/src/eclwatch/Graph7Widget.js

@@ -1,174 +0,0 @@
-define([
-    "dojo/_base/declare",
-    "dojo/i18n",
-    "dojo/i18n!./nls/hpcc",
-    "dojo/dom",
-    "dojo/dom-class",
-    "dojo/dom-style",
-
-    "dijit/registry",
-
-    "hpcc/_Widget",
-
-    "@hpcc-js/eclwatch",
-
-    "dojo/text!../templates/Graph7Widget.html",
-
-    "dijit/layout/BorderContainer",
-    "dijit/layout/ContentPane",
-    "dijit/ToolbarSeparator",
-    "dijit/layout/ContentPane",
-    "dijit/form/Button"
-
-], function (declare, i18n, nlsHPCC, dom, domClass, domStyle,
-    registry,
-    _Widget,
-    hpccEclWatch,
-    template) {
-
-        return declare("Graph7Widget", [_Widget], {
-            templateString: template,
-            baseClass: "Graph7Widget",
-            i18n: nlsHPCC,
-
-            KeyState_None: 0,
-            KeyState_Shift: 1,
-            KeyState_Control: 2,
-            KeyState_Menu: 4,
-
-            borderContainer: null,
-            graphContentPane: null,
-
-            graph: null,
-
-            constructor: function () {
-                this._options = {};
-            },
-
-            option: function (key, _) {
-                if (arguments.length < 1) throw Error("Invalid Call:  option");
-                if (arguments.length === 1) return this._options[key];
-                this._options[key] = _ instanceof Array ? _.length > 0 : _;
-                return this;
-            },
-
-            _onClickRefresh: function () {
-            },
-
-            _onChangeZoom: function (args) {
-                var selection = this.zoomDropCombo.get("value");
-                switch (selection) {
-                    case this.i18n.All:
-                        this.centerOnItem(0, true);
-                        break;
-                    case this.i18n.Width:
-                        this.centerOnItem(0, true, true);
-                        break;
-                    default:
-                        var scale = parseFloat(selection);
-                        if (!isNaN(scale)) {
-                            this.setScale(scale);
-                        }
-                        break;
-                }
-            },
-
-            _onOptionsApply: function () {
-            },
-
-            _onOptionsReset: function () {
-            },
-
-            buildRendering: function (args) {
-                this.inherited(arguments);
-            },
-
-            postCreate: function (args) {
-                this.inherited(arguments);
-                this.borderContainer = registry.byId(this.id + "BorderContainer");
-                this.graphContentPane = registry.byId(this.id + "GraphContentPane");
-                this.zoomDropCombo = registry.byId(this.id + "ZoomDropCombo");
-                this.optionsDropDown = registry.byId(this.id + "OptionsDropDown");
-                this.optionsForm = registry.byId(this.id + "OptionsForm");
-            },
-
-            startup: function (args) {
-                this.inherited(arguments);
-            },
-
-            init: function (params) {
-                if (this.inherited(arguments))
-                    return;
-
-                this.graph = new hpccEclWatch.WUGraph()
-                    .target(this.id + "GraphContentPane")
-                    .baseUrl("")
-                    .render(function (w) {
-                        w
-                            .wuid(params.Wuid)
-                            .graphID(params.GraphName)
-                            .subgraphID(params.SubGraphId)
-                            .render(function (w) {
-                                if (params.ActivityId) {
-                                    w.selection(params.ActivityId);
-                                }
-                            })
-                            ;
-                    })
-                    ;
-            },
-
-            resize: function (args) {
-                this.inherited(arguments);
-                this.borderContainer.resize();
-                if (this.graph) {
-                    this.graph
-                        .resize()
-                        .render()
-                        ;
-                }
-            },
-
-            layout: function (args) {
-                this.inherited(arguments);
-            },
-
-            //  Plugin wrapper  ---
-            hasOptions: function () {
-                return false;
-            },
-
-            showToolbar: function (show) {
-                if (show) {
-                    domClass.remove(this.id + "Toolbar", "hidden");
-                } else {
-                    domClass.add(this.id + "Toolbar", "hidden");
-                }
-                this.resize();
-            },
-
-            showOptions: function (show) {
-                if (show) {
-                    domStyle.set(this.optionsDropDown.domNode, 'display', 'block');
-                } else {
-                    domStyle.set(this.optionsDropDown.domNode, 'display', 'none');
-                }
-                this.resize();
-            },
-
-            clear: function () {
-            },
-
-            find: function (findText) {
-                return [];
-            },
-
-            findAsGlobalID: function (findText) {
-                return [];
-            },
-
-            setScale: function (percent) {
-                return 100;
-            }
-        });
-    });

esp/src/eclwatch/LFDetailsWidget.js

@@ -63,6 +63,7 @@ define([
             replicateForm: null,
             summaryWidget: null,
             contentWidget: null,
+            dataPatternsWidget: null,
             sourceWidget: null,
             defWidget: null,
             xmlWidget: null,
@@ -84,6 +85,7 @@ define([
                 this.replicateForm = registry.byId(this.id + "ReplicateForm");
                 this.summaryWidget = registry.byId(this.id + "_Summary");
                 this.contentWidget = registry.byId(this.id + "_Content");
+                this.dataPatternsWidget = registry.byId(this.id + "_DataPatterns");
                 this.sourceWidget = registry.byId(this.id + "_Source");
                 this.defWidget = registry.byId(this.id + "_DEF");
                 this.xmlWidget = registry.byId(this.id + "_XML");
@@ -263,7 +265,7 @@ define([
                 });
                 this.logicalFile.refresh();
 
-                this.isProtected.on("change", function(evt){
+                this.isProtected.on("change", function (evt) {
                     context._onSave();
                 });
             },
@@ -277,6 +279,11 @@ define([
                             NodeGroup: this.logicalFile.NodeGroup,
                             LogicalName: this.logicalFile.Name
                         });
+                    } else if (currSel.id === this.dataPatternsWidget.id) {
+                        this.dataPatternsWidget.init({
+                            NodeGroup: this.logicalFile.NodeGroup,
+                            LogicalName: this.logicalFile.Name
+                        });
                     } else if (currSel.id === this.sourceWidget.id) {
                         this.sourceWidget.init({
                             ECL: this.logicalFile.Ecl
@@ -451,7 +458,8 @@ define([
                 this.setDisabled(this.id + "CopyDropDown", this.logicalFile.isDeleted());
                 this.setDisabled(this.id + "RenameDropDown", this.logicalFile.isDeleted());
                 this.setDisabled(this.id + "DesprayDropDown", this.logicalFile.isDeleted());
-                this.setDisabled(this.id + "_Content", this.logicalFile.isDeleted()  || !this.logicalFile.Ecl);
+                this.setDisabled(this.id + "_Content", this.logicalFile.isDeleted() || !this.logicalFile.Ecl);
+                this.setDisabled(this.id + "_DataPatterns", this.logicalFile.isDeleted() || this.logicalFile.ContentType === "key");
                 this.setDisabled(this.id + "_Source", this.logicalFile.isDeleted() || !this.logicalFile.Ecl);
                 this.setDisabled(this.id + "_DEF", this.logicalFile.isDeleted() || !this.logicalFile.Ecl);
                 this.setDisabled(this.id + "_XML", this.logicalFile.isDeleted() || !this.logicalFile.Ecl);

esp/src/eclwatch/ResultWidget.js

@@ -5,15 +5,12 @@ define([
     "dojo/i18n",
     "dojo/i18n!./nls/hpcc",
     "dojo/io-query",
-    "dojo/dom",
 
     "dijit/registry",
     "dijit/form/TextBox",
 
     "dgrid/Grid",
     "dgrid/Keyboard",
-    "dgrid/Selection",
-    "dgrid/selector",
     "dgrid/extensions/ColumnResizer",
     "dgrid/extensions/CompoundColumns",
     "dgrid/extensions/DijitRegistry",
@@ -25,6 +22,8 @@ define([
     "src/ESPLogicalFile",
     "hpcc/FilterDropDownWidget",
     "hpcc/TableContainer",
+    "src/DataPatterns/DGridHeaderHook",
+    "@hpcc-js/common",
 
     "dojo/text!../templates/ResultWidget.html",
 
@@ -33,10 +32,10 @@ define([
     "dijit/Toolbar",
     "dijit/form/Button",
     "dijit/ToolbarSeparator"
-], function (declare, lang, arrayUtil, i18n, nlsHPCC, ioQuery, dom,
+], function (declare, lang, arrayUtil, i18n, nlsHPCC, ioQuery,
     registry, TextBox,
-    Grid, Keyboard, Selection, selector, ColumnResizer, CompoundColumns, DijitRegistry, PaginationModule,
-    _Widget, ESPBase, ESPWorkunit, ESPLogicalFile, FilterDropDownWidget, TableContainer,
+    Grid, Keyboard, ColumnResizer, CompoundColumns, DijitRegistry, PaginationModule,
+    _Widget, ESPBase, ESPWorkunit, ESPLogicalFile, FilterDropDownWidget, TableContainer, DGridHeaderHookMod, hpccCommon,
     template) {
         return declare("ResultWidget", [_Widget], {
             templateString: template,
@@ -48,6 +47,8 @@ define([
 
             loaded: false,
 
+            dataPatternsButton: null,
+
             buildRendering: function (args) {
                 this.inherited(arguments);
             },
@@ -57,6 +58,7 @@ define([
                 this.borderContainer = registry.byId(this.id + "BorderContainer");
                 this.filter = registry.byId(this.id + "Filter");
                 this.grid = registry.byId(this.id + "Grid");
+                this.dataPatternsButton = registry.byId(this.id + "DataPatterns");
             },
 
             startup: function (args) {
@@ -109,6 +111,18 @@ define([
                 alert("todo");
             },
 
+            _onDataPatterns: function (args) {
+                var context = this;
+                if (this._logicalFile) {
+                    var wuPromise = this.dataPatternsButton.get("checked") ? this._logicalFile.fetchDataPatternsWU() : Promise.resolve(null);
+                    wuPromise.then(function (wu) {
+                        return context.gridDPHook.render(wu);
+                    }).then(function () {
+                        context.grid.resize();
+                    });
+                }
+            },
+
             //  Implementation  ---
             onErrorClick: function (line, col) {
             },
@@ -133,29 +147,33 @@ define([
                         }
                     });
                 } else if (params.LogicalName) {
-                    var logicalFile = ESPLogicalFile.Get(params.NodeGroup, params.LogicalName);
-                    logicalFile.getInfo({
+                    this._logicalFile = ESPLogicalFile.Get(params.NodeGroup, params.LogicalName);
+                    this._logicalFile.getInfo({
                         onAfterSend: function (response) {
-                            context.initResult(logicalFile.result);
+                            context.initResult(context._logicalFile.result);
                         }
                     });
                 } else if (params.result && params.result.Name) {
-                    var logicalFile = ESPLogicalFile.Get(params.result.NodeGroup, params.result.Name);
-                    logicalFile.getInfo({
+                    this._logicalFile = ESPLogicalFile.Get(params.result.NodeGroup, params.result.Name);
+                    this._logicalFile.getInfo({
                         onAfterSend: function (response) {
-                            context.initResult(logicalFile.result);
+                            context.initResult(context.logicalFile.result);
                         }
                     });
                 } else {
                     this.initResult(null);
                 }
+                if (!this._logicalFile) {
+                    dojo.destroy(this.id + "DataPatterns");
+                    dojo.destroy(this.id + "DataPatternsSep");
+                }
+                this.refreshDataPatterns();
             },
 
             initResult: function (result) {
                 if (result) {
                     var context = this;
                     result.fetchStructure(function (structure) {
-                        var filterForm = registry.byId(context.filter.id + "FilterForm");
                         var origTableContainer = registry.byId(context.filter.id + "TableContainer");
                         var tableContainer = new TableContainer({
                         });
@@ -201,6 +219,13 @@ define([
                             }
                         }, context.id + "Grid");
                         context.grid.startup();
+                        context.gridDPHook = new DGridHeaderHookMod.DGridHeaderHook(context.grid, context.id + "Grid");
+                        context.grid.on("dgrid-columnresize", function (evt) {
+                            setTimeout(function () {
+                                context.gridDPHook.resize(evt.columnId);
+                            }, 20);
+                        });
+
                     });
                 } else {
                     this.grid = new declare([Grid, DijitRegistry])({
@@ -231,6 +256,33 @@ define([
                         BypassCachedResult: bypassCachedResult
                     });
                 }
+                this.refreshDataPatterns();
+            },
+
+            _wu: null, //  Null needed for initial test  ---
+            refreshDataPatterns: function () {
+                if (this._logicalFile) {
+                    var context = this;
+                    this._logicalFile.fetchDataPatternsWU().then(function (wu) {
+                        if (context._wu !== wu) {
+                            context._wu = wu;
+                            if (context._wu) {
+                                context._wu.watchUntilComplete(function (changes) {
+                                    context.refreshActionState();
+                                });
+                            } else {
+                                context.refreshActionState();
+                            }
+                        }
+                    });
+                }
+            },
+
+            refreshActionState: function () {
+                if (this._logicalFile) {
+                    var isComplete = this._wu && this._wu.isComplete();
+                    this.setDisabled(this.id + "DataPatterns", !isComplete);
+                }
             }
         });
     });

esp/src/eclwatch/css/hpcc.css

@@ -230,6 +230,11 @@ hr.dashedLine {
     cursor: pointer;
 }
 
+.hideNoIcon .dijitNoIcon {
+    width: 0px;
+    height: 0px;
+}
+
 .flat .miniTitlebar .dijitNoIcon {
     width: 0px;
     height: 0px;
@@ -1646,3 +1651,23 @@ span.dijitReset.dijitInline.dijitIcon.fa {
     min-width: 16px;
     min-height: 16px;
 }
+
+.eclwatch_DPReport.report-col-count-2 .ddCell:nth-child(4n+3),
+.eclwatch_DPReport.report-col-count-2 .ddCell:nth-child(4n+4),
+.eclwatch_DPReport.report-col-count-3 .ddCell:nth-child(6n+4),
+.eclwatch_DPReport.report-col-count-3 .ddCell:nth-child(6n+5),
+.eclwatch_DPReport.report-col-count-3 .ddCell:nth-child(6n+6),
+.eclwatch_DPReport.report-col-count-4 .ddCell:nth-child(8n+5),
+.eclwatch_DPReport.report-col-count-4 .ddCell:nth-child(8n+6),
+.eclwatch_DPReport.report-col-count-4 .ddCell:nth-child(8n+7),
+.eclwatch_DPReport.report-col-count-4 .ddCell:nth-child(8n+8),
+.eclwatch_DPReport.report-col-count-5 .ddCell:nth-child(10n+6),
+.eclwatch_DPReport.report-col-count-5 .ddCell:nth-child(10n+7),
+.eclwatch_DPReport.report-col-count-5 .ddCell:nth-child(10n+8),
+.eclwatch_DPReport.report-col-count-5 .ddCell:nth-child(10n+9),
+.eclwatch_DPReport.report-col-count-5 .ddCell:nth-child(10n+10) {
+    background:#F3F3F3;
+    -webkit-box-shadow: 0px 0px 0px 6px rgba(243,243,243,1);
+    -moz-box-shadow: 0px 0px 0px 6px rgba(243,243,243,1);
+    box-shadow: 0px 0px 0px 6px rgba(243,243,243,1);
+}

esp/src/eclwatch/dojoConfig.js

@@ -45,6 +45,7 @@ function getConfig(env) {
             "@hpcc-js/form": baseUrl + "/node_modules/@hpcc-js/form/dist/index" + hpccMin,
             "@hpcc-js/graph": baseUrl + "/node_modules/@hpcc-js/graph/dist/index" + hpccMin,
             "@hpcc-js/layout": baseUrl + "/node_modules/@hpcc-js/layout/dist/index" + hpccMin,
+            "@hpcc-js/html": baseUrl + "/node_modules/@hpcc-js/html/dist/index" + hpccMin,
             "@hpcc-js/map": baseUrl + "/node_modules/@hpcc-js/map/dist/index" + hpccMin,
             "@hpcc-js/other": baseUrl + "/node_modules/@hpcc-js/other/dist/index" + hpccMin,
             "@hpcc-js/timeline": baseUrl + "/node_modules/@hpcc-js/timeline/dist/index" + hpccMin,

esp/src/eclwatch/nls/hpcc.js

@@ -44,6 +44,7 @@ define({root:
     AllowRead: "<center>Allow<br>Read</center>",
     AllowWrite: "<center>Allow<br>Write</center>",
     AllQueuedItemsCleared: "All Queued items have been cleared. The current running job will continue to execute.",
+    Analyze: "Analyze",
     ANY: "ANY",
     AnyAdditionalProcessesToFilter: "Any Addtional Processes To Filter",
     Append: "Append",
@@ -117,6 +118,7 @@ define({root:
     Copied: "Copied!",
     Count: "Count",
     CPULoad: "CPU Load",
+    Create: "Create",
     CreateANewFile: "Create a new superfile",
     Created: "Created",
     Creating:"Creating",
@@ -129,6 +131,7 @@ define({root:
     CSV: "CSV",
     Dali: "Dali",
     DaliIP: "DaliIP",
+    DataPatterns: "Data Patterns",
     dataset: ":=dataset*",
     Date: "Date",
     Day: "Day",
@@ -468,6 +471,7 @@ define({root:
     Operation: "Operation",
     Operations: "Operations",
     Options: "Options",
+    Optimize: "Optimize",
     OriginalFile: "Original File",
     OrphanFile: "Orphan Files",
     OrphanFile2: "Orphan File",
@@ -564,6 +568,7 @@ define({root:
     Queue: "Queue",
     Quote: "Quote",
     QuotedTerminator: "Quoted Terminator",
+    RawData: "Raw Data",
     RawTextPage: "Raw Text (Current Page)",
     Ready: "Ready",
     ReallyWantToRemove: "Really want to remove?",
@@ -598,6 +603,7 @@ define({root:
     Replicate: "Replicate",
     ReplicatedLost: "Replicated Lost",
     ReplicateOffset: "Replicate Offset",
+   Report: "Report",
     RepresentsASubset: "represent a subset of the total number of matches. Using a correct filter may reduce the number of matches.",
     RequestSchema: "Request Schema",
     RequiredForFixedSpray: "Required for fixed spray",

esp/src/eclwatch/templates/DataPatternsWidget.html

@@ -0,0 +1,58 @@
+<div class="${baseClass}">
+    <div id="${id}BorderContainer" class="${baseClass}BorderContainer" style="width: 100%; height: 100%" data-dojo-props="splitter: false" data-dojo-type="dijit.layout.BorderContainer">
+        <div id="${id}TabContainer" data-dojo-props="region: 'center', tabPosition: 'top'" style="width: 100%; height: 100%" data-dojo-type="dijit.layout.TabContainer">
+            <div id="${id}_Summary" style="width: 100%; height: 100%" data-dojo-props="title:'${i18n.Report}'" data-dojo-type="dijit.layout.BorderContainer">
+                <div id="${id}Toolbar" class="topPanel" data-dojo-props="region: 'top'" data-dojo-type="dijit.Toolbar">
+                    <div id="${id}Refresh" data-dojo-attach-event="onClick:_onRefresh" data-dojo-props="iconClass:'iconRefresh'" data-dojo-type="dijit.form.Button">${i18n.Refresh}</div>
+                    <span data-dojo-type="dijit.ToolbarSeparator"></span>
+                    <div id="${id}Analyze" data-dojo-attach-event="onClick:_onAnalyze" data-dojo-type="dijit.form.Button">${i18n.Analyze}</div>
+                    <label id="${id}TargetSelectLabel" for="Target">${i18n.Target}:</label>
+                    <div id="${id}TargetSelect" style="display: inline-block; vertical-align: middle" data-dojo-type="TargetSelectWidget"></div>
+                    <div id="${id}OptimizeDropDown" data-dojo-type="dijit.form.DropDownButton">
+                        <span>${i18n.Optimize}</span>
+                        <div data-dojo-type="dijit.TooltipDialog">
+                            <div id="${id}OptimizeForm" style="width: 650px;" onsubmit="return false;" data-dojo-props="region: 'bottom'" data-dojo-type="dijit.form.Form">
+                                <div data-dojo-type="dijit.Fieldset">
+                                    <legend>${i18n.Target}</legend>
+                                    <div data-dojo-type="hpcc.TableContainer">
+                                        <input id="${id}OptimizeTargetSelect" title="${i18n.Target}:" name="target" style="width:100%;display: inline-block; vertical-align: middle" data-dojo-type="TargetSelectWidget" />
+                                        <input id="${id}OptimizeTarget" title="${i18n.Name}:" name="name"  style="width:100%;" data-dojo-type="dijit.form.TextBox" />
+                                    </div>
+                                </div>
+                                <div data-dojo-type="dijit.Fieldset">
+                                    <legend>${i18n.Options}</legend>
+                                    <div data-dojo-props="cols:2" data-dojo-type="hpcc.TableContainer">
+                                        <input id="${id}OptimizeTargetOverwrite" title="${i18n.Overwrite}:" name="overwrite" data-dojo-type="dijit.form.CheckBox" />
+                                    </div>
+                                </div>
+                                <div class="dijitDialogPaneActionBar">
+                                    <button type="submit" data-dojo-attach-event="onClick:_onOptimizeOk" data-dojo-type="dijit.form.Button">${i18n.Optimize}</button>
+                                </div>
+                            </div>
+                        </div>
+                    </div>
+                    <div id="${id}Delete" data-dojo-attach-event="onClick:_onDelete" data-dojo-type="dijit.form.Button">${i18n.Delete}</div>
+                    <span data-dojo-type="dijit.ToolbarSeparator"></span>
+                    <div id="${id}NewPage" class="right" data-dojo-attach-event="onClick:_onNewPage" data-dojo-props="iconClass:'iconNewPage', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.OpenInNewPage}</div>
+                </div>
+                <div id="${id}CenterContainer" style="overflow: hidden" data-dojo-props="region: 'center'" data-dojo-type="dijit.layout.ContentPane">
+                    <div id="${id}WU">
+                        <div style="display:inline-block">
+                            <h2>
+                                <div id="${id}StateIdImage" class="iconWorkunit" ></div>&nbsp;<span id="${id}Wuid" class="bold"></span>
+                            </h2>
+                        </div>
+                        <div id="${id}WUStatus" style="position:absolute;top:0;right:0;width:512px;height:64px">
+                        </div>
+                    </div>                    
+                    <div id="${id}DPReport" style="overflow:scroll;">
+                    </div>
+                </div>
+            </div>
+            <div id="${id}_RawData" title="${i18n.RawData}" data-dojo-props="delayWidget: 'ResultWidget'" data-dojo-type="DelayLoadWidget">
+            </div>
+            <div id="${id}_Workunit" title="${i18n.Workunit}" data-dojo-props="delayWidget: 'WUDetailsWidget'" data-dojo-type="DelayLoadWidget">
+            </div>
+        </div>
+    </div>
+</div>

esp/src/eclwatch/templates/LFDetailsWidget.html

@@ -220,6 +220,8 @@
             </div>
             <div id="${id}_Content" title="${i18n.Contents}" data-dojo-props="delayWidget: 'ResultWidget'" data-dojo-type="DelayLoadWidget">
             </div>
+            <div id="${id}_DataPatterns" title="${i18n.DataPatterns}" data-dojo-props="delayWidget: 'DataPatternsWidget'" data-dojo-type="DelayLoadWidget">
+            </div>
             <div id="${id}_Source" title="${i18n.ECL}" data-dojo-props="delayWidget: 'ECLSourceWidget', disabled: true" data-dojo-type="DelayLoadWidget">
             </div>
             <div id="${id}_DEF" title="${i18n.DEF}" data-dojo-props="delayWidget: 'ECLSourceWidget', disabled: true" data-dojo-type="DelayLoadWidget">

esp/src/eclwatch/templates/ResultWidget.html

@@ -12,6 +12,8 @@
             <div id="${id}Filter" data-dojo-type="FilterDropDownWidget">
             </div>
             <span data-dojo-type="dijit.ToolbarSeparator"></span>
+            <div id="${id}DataPatterns"  checked=false class="hideNoIcon" data-dojo-attach-event="onClick:_onDataPatterns" data-dojo-props="showIcon:false, showNoIcon:false" data-dojo-type="dijit.form.ToggleButton">${i18n.DataPatterns}</div>
+            <span id="${id}DataPatternsSep" data-dojo-type="dijit.ToolbarSeparator"></span>
             <div class="right" data-dojo-attach-event="onChange:_onMaximize" data-dojo-props="iconClass:'iconMaximize', showLabel:false" checked=false data-dojo-type="dijit.form.ToggleButton">${i18n.MaximizeRestore}</div>
             <div id="${id}NewPage" class="right" data-dojo-attach-event="onClick:_onNewPage" data-dojo-props="iconClass:'iconNewPage', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.OpenInNewPage}</div>
         </div>

esp/src/package-lock.json

@@ -116,6 +116,16 @@
         "@hpcc-js/common": "^2.16.8"
       }
     },
+    "@hpcc-js/html": {
+      "version": "2.6.8",
+      "resolved": "https://registry.npmjs.org/@hpcc-js/html/-/html-2.6.8.tgz",
+      "integrity": "sha512-YmE9ePjPO7wkPwrSUw2p7dgsgh3M++4iXerh4NCtGQt25hLkNpPZXj6SSSFlPImfSg7ASTyMT6u1Nq94eLPYuQ==",
+      "requires": {
+        "@hpcc-js/common": "^2.16.8",
+        "@hpcc-js/preact-shim": "^2.10.1",
+        "@hpcc-js/util": "^2.7.0"
+      }
+    },
     "@hpcc-js/layout": {
       "version": "2.12.8",
       "resolved": "https://registry.npmjs.org/@hpcc-js/layout/-/layout-2.12.8.tgz",
@@ -149,6 +159,14 @@
         "@hpcc-js/layout": "^2.12.8"
       }
     },
+    "@hpcc-js/preact-shim": {
+      "version": "2.10.1",
+      "resolved": "https://registry.npmjs.org/@hpcc-js/preact-shim/-/preact-shim-2.10.1.tgz",
+      "integrity": "sha512-LbG5JLrMRXI+DzRyqufXogrtFfU6ozE7+I+NoNgwyQJ/93GPqJmPoDZVCAyejJLFlKXKTKEX6q+eA+l9R2nXDg==",
+      "requires": {
+        "preact": "8.2.1"
+      }
+    },
     "@hpcc-js/timeline": {
       "version": "2.6.8",
       "resolved": "https://registry.npmjs.org/@hpcc-js/timeline/-/timeline-2.6.8.tgz",
@@ -5953,6 +5971,11 @@
       "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==",
       "dev": true
     },
+    "preact": {
+      "version": "8.2.1",
+      "resolved": "https://registry.npmjs.org/preact/-/preact-8.2.1.tgz",
+      "integrity": "sha1-Z0JD3wyEeITQGYNARKovzTEecu0="
+    },
     "preserve": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/preserve/-/preserve-0.2.0.tgz",
@@ -7521,220 +7544,6 @@
       "integrity": "sha512-uvJvOMwAheCYrxDUsqkMPpYk7t1/R+4VQnBZ3wzkaA6QRQzXxG6+/yA6VGDtK+bgzCxh6Vk5arJ7TG0Gf8GN1w==",
       "dev": true
     },
-    "uglify-js": {
-      "version": "3.5.12",
-      "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.5.12.tgz",
-      "integrity": "sha512-KeQesOpPiZNgVwJj8Ge3P4JYbQHUdZzpx6Fahy6eKAYRSV4zhVmLXoC+JtOeYxcHCHTve8RG1ZGdTvpeOUM26Q==",
-      "dev": true,
-      "requires": {
-        "commander": "~2.20.0",
-        "source-map": "~0.6.1"
-      }
-    },
-    "uglifyjs-webpack-plugin": {
-      "version": "2.1.3",
-      "resolved": "https://registry.npmjs.org/uglifyjs-webpack-plugin/-/uglifyjs-webpack-plugin-2.1.3.tgz",
-      "integrity": "sha512-/lRkCaFbI6pT3CxsQHDhBcqB6tocOnqba0vJqJ2DzSWFLRgOIiip8q0nVFydyXk+n8UtF7ZuS6hvWopcYH5FuA==",
-      "dev": true,
-      "requires": {
-        "cacache": "^11.3.2",
-        "find-cache-dir": "^2.1.0",
-        "is-wsl": "^1.1.0",
-        "schema-utils": "^1.0.0",
-        "serialize-javascript": "^1.7.0",
-        "source-map": "^0.6.1",
-        "uglify-js": "^3.5.12",
-        "webpack-sources": "^1.3.0",
-        "worker-farm": "^1.7.0"
-      },
-      "dependencies": {
-        "bluebird": {
-          "version": "3.5.4",
-          "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.4.tgz",
-          "integrity": "sha512-FG+nFEZChJrbQ9tIccIfZJBz3J7mLrAhxakAbnrJWn8d7aKOC+LWifa0G+p4ZqKp4y13T7juYvdhq9NzKdsrjw==",
-          "dev": true
-        },
-        "cacache": {
-          "version": "11.3.2",
-          "resolved": "https://registry.npmjs.org/cacache/-/cacache-11.3.2.tgz",
-          "integrity": "sha512-E0zP4EPGDOaT2chM08Als91eYnf8Z+eH1awwwVsngUmgppfM5jjJ8l3z5vO5p5w/I3LsiXawb1sW0VY65pQABg==",
-          "dev": true,
-          "requires": {
-            "bluebird": "^3.5.3",
-            "chownr": "^1.1.1",
-            "figgy-pudding": "^3.5.1",
-            "glob": "^7.1.3",
-            "graceful-fs": "^4.1.15",
-            "lru-cache": "^5.1.1",
-            "mississippi": "^3.0.0",
-            "mkdirp": "^0.5.1",
-            "move-concurrently": "^1.0.1",
-            "promise-inflight": "^1.0.1",
-            "rimraf": "^2.6.2",
-            "ssri": "^6.0.1",
-            "unique-filename": "^1.1.1",
-            "y18n": "^4.0.0"
-          }
-        },
-        "find-cache-dir": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-2.1.0.tgz",
-          "integrity": "sha512-Tq6PixE0w/VMFfCgbONnkiQIVol/JJL7nRMi20fqzA4NRs9AfeqMGeRdPi3wIhYkxjeBaWh2rxwapn5Tu3IqOQ==",
-          "dev": true,
-          "requires": {
-            "commondir": "^1.0.1",
-            "make-dir": "^2.0.0",
-            "pkg-dir": "^3.0.0"
-          }
-        },
-        "find-up": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz",
-          "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==",
-          "dev": true,
-          "requires": {
-            "locate-path": "^3.0.0"
-          }
-        },
-        "glob": {
-          "version": "7.1.4",
-          "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz",
-          "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==",
-          "dev": true,
-          "requires": {
-            "fs.realpath": "^1.0.0",
-            "inflight": "^1.0.4",
-            "inherits": "2",
-            "minimatch": "^3.0.4",
-            "once": "^1.3.0",
-            "path-is-absolute": "^1.0.0"
-          }
-        },
-        "graceful-fs": {
-          "version": "4.1.15",
-          "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz",
-          "integrity": "sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA==",
-          "dev": true
-        },
-        "locate-path": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz",
-          "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==",
-          "dev": true,
-          "requires": {
-            "p-locate": "^3.0.0",
-            "path-exists": "^3.0.0"
-          }
-        },
-        "lru-cache": {
-          "version": "5.1.1",
-          "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
-          "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
-          "dev": true,
-          "requires": {
-            "yallist": "^3.0.2"
-          }
-        },
-        "make-dir": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz",
-          "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==",
-          "dev": true,
-          "requires": {
-            "pify": "^4.0.1",
-            "semver": "^5.6.0"
-          }
-        },
-        "mississippi": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/mississippi/-/mississippi-3.0.0.tgz",
-          "integrity": "sha512-x471SsVjUtBRtcvd4BzKE9kFC+/2TeWgKCgw0bZcw1b9l2X3QX5vCWgF+KaZaYm87Ss//rHnWryupDrgLvmSkA==",
-          "dev": true,
-          "requires": {
-            "concat-stream": "^1.5.0",
-            "duplexify": "^3.4.2",
-            "end-of-stream": "^1.1.0",
-            "flush-write-stream": "^1.0.0",
-            "from2": "^2.1.0",
-            "parallel-transform": "^1.1.0",
-            "pump": "^3.0.0",
-            "pumpify": "^1.3.3",
-            "stream-each": "^1.1.0",
-            "through2": "^2.0.0"
-          }
-        },
-        "p-limit": {
-          "version": "2.2.0",
-          "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.0.tgz",
-          "integrity": "sha512-pZbTJpoUsCzV48Mc9Nh51VbwO0X9cuPFE8gYwx9BTCt9SF8/b7Zljd2fVgOxhIF/HDTKgpVzs+GPhyKfjLLFRQ==",
-          "dev": true,
-          "requires": {
-            "p-try": "^2.0.0"
-          }
-        },
-        "p-locate": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz",
-          "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==",
-          "dev": true,
-          "requires": {
-            "p-limit": "^2.0.0"
-          }
-        },
-        "p-try": {
-          "version": "2.2.0",
-          "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
-          "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==",
-          "dev": true
-        },
-        "pify": {
-          "version": "4.0.1",
-          "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz",
-          "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==",
-          "dev": true
-        },
-        "pkg-dir": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-3.0.0.tgz",
-          "integrity": "sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw==",
-          "dev": true,
-          "requires": {
-            "find-up": "^3.0.0"
-          }
-        },
-        "pump": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
-          "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
-          "dev": true,
-          "requires": {
-            "end-of-stream": "^1.1.0",
-            "once": "^1.3.1"
-          }
-        },
-        "serialize-javascript": {
-          "version": "1.7.0",
-          "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-1.7.0.tgz",
-          "integrity": "sha512-ke8UG8ulpFOxO8f8gRYabHQe/ZntKlcig2Mp+8+URDP1D8vJZ0KUt7LYo07q25Z/+JVSgpr/cui9PIp5H6/+nA==",
-          "dev": true
-        },
-        "ssri": {
-          "version": "6.0.1",
-          "resolved": "https://registry.npmjs.org/ssri/-/ssri-6.0.1.tgz",
-          "integrity": "sha512-3Wge10hNcT1Kur4PDFwEieXSCMCJs/7WvSACcrMYrNp+b8kDL1/0wJch5Ni2WrtwEa2IO8OsVfeKIciKCDx/QA==",
-          "dev": true,
-          "requires": {
-            "figgy-pudding": "^3.5.1"
-          }
-        },
-        "yallist": {
-          "version": "3.0.3",
-          "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz",
-          "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==",
-          "dev": true
-        }
-      }
-    },
     "union-value": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.0.tgz",
@@ -8254,12 +8063,14 @@
             "balanced-match": {
               "version": "1.0.0",
               "bundled": true,
-              "dev": true
+              "dev": true,
+              "optional": true
             },
             "brace-expansion": {
               "version": "1.1.11",
               "bundled": true,
               "dev": true,
+              "optional": true,
               "requires": {
                 "balanced-match": "^1.0.0",
                 "concat-map": "0.0.1"
@@ -8274,12 +8085,14 @@
             "code-point-at": {
               "version": "1.1.0",
               "bundled": true,
-              "dev": true
+              "dev": true,
+              "optional": true
             },
             "concat-map": {
               "version": "0.0.1",
               "bundled": true,
-              "dev": true
+              "dev": true,
+              "optional": true
             },
             "console-control-strings": {
               "version": "1.1.0",
@@ -8401,7 +8214,8 @@
             "inherits": {
               "version": "2.0.3",
               "bundled": true,
-              "dev": true
+              "dev": true,
+              "optional": true
             },
             "ini": {
               "version": "1.3.5",
@@ -8413,6 +8227,7 @@
               "version": "1.0.0",
               "bundled": true,
               "dev": true,
+              "optional": true,
               "requires": {
                 "number-is-nan": "^1.0.0"
               }
@@ -8427,6 +8242,7 @@
               "version": "3.0.4",
               "bundled": true,
               "dev": true,
+              "optional": true,
               "requires": {
                 "brace-expansion": "^1.1.7"
               }
@@ -8538,7 +8354,8 @@
             "number-is-nan": {
               "version": "1.0.1",
               "bundled": true,
-              "dev": true
+              "dev": true,
+              "optional": true
             },
             "object-assign": {
               "version": "4.1.1",
@@ -8671,6 +8488,7 @@
               "version": "1.0.2",
               "bundled": true,
               "dev": true,
+              "optional": true,
               "requires": {
                 "code-point-at": "^1.0.0",
                 "is-fullwidth-code-point": "^1.0.0",

esp/src/package.json

@@ -33,6 +33,7 @@
     "@hpcc-js/chart": "2.17.2",
     "@hpcc-js/comms": "2.7.0",
     "@hpcc-js/eclwatch": "2.5.8",
+    "@hpcc-js/html": "2.6.8",
     "@hpcc-js/map": "2.7.8",
     "@hpcc-js/other": "2.12.8",
     "@hpcc-js/tree": "2.7.8",
@@ -76,4 +77,4 @@
     "type": "git",
     "url": "https://github.com/hpcc-systems/HPCC-Platform"
   }
-}
+}

esp/src/src/DataPatterns/AttributeDesc.ts

@@ -0,0 +1,93 @@
+import { Grid } from "@hpcc-js/layout";
+import { Html } from "@hpcc-js/other";
+import { StyledTable } from "@hpcc-js/html";
+import { config } from "./config";
+
+class AttributeTitle extends Html {
+    constructor(row) {
+        super();
+
+        const p = 8;
+        const b = 1;
+
+        this
+            .html(`<span style="
+                color:${config.primaryColor};
+                padding:${p}px;
+                display:inline-block;
+                font-size:${config.secondaryFontSize}px;
+                margin-top:4px;
+                border:${b}px solid ${config.secondaryColor};
+                border-radius:4px;
+                background-color: ${config.offwhiteColor};
+                width: calc(100% - ${(p * 2) + (b * 2)}px);
+            ">
+                <i style="
+                    font-size:${config.secondaryFontSize}px;
+                    color:${config.blueColor};
+                " class="fa ${row.given_attribute_type.slice(0, 6) === "string" ? "fa-font" : "fa-hashtag"}"></i>
+                <b>${row.attribute}</b>
+                <span style="float:right;">${row.given_attribute_type}</span>
+            </span>
+            <span style="padding:12px 2px;display:inline-block;font-weight: bold;font-size: 13px;">
+                Optimal:
+            </span>
+            <span style="
+                color:${config.primaryColor};
+                padding:4px 8px;
+                display:inline-block;
+                font-size:${config.secondaryFontSize}px;
+                margin-top:4px;
+                border:1px solid ${config.secondaryColor};
+                border-radius:4px;
+                background-color: ${config.offwhiteColor};
+                float:right;
+            ">
+                <i style="
+                    font-size:${config.secondaryFontSize}px;
+                    color:${config.blueColor};
+                " class="fa ${row.best_attribute_type.slice(0, 6) === "string" ? "fa-font" : "fa-hashtag"}"></i>
+                ${row.best_attribute_type}
+            </span>
+            `)
+            .overflowX("hidden")
+            .overflowY("hidden")
+            ;
+    }
+}
+
+class AttributeSummary extends StyledTable {
+
+    constructor(row) {
+        super();
+        let fillRate = row.fill_rate === 100 || row.fill_rate === 0 ? row.fill_rate : row.fill_rate.toFixed(1);
+        this
+            .data([
+                ["Cardinality", row.cardinality, "(~" + (row.cardinality / row.fill_count * 100).toFixed(0) + "%)"],
+                ["Filled", row.fill_count, fillRate <= config.fillRateRedThreshold ? `(<b style="color:${config.redColor}">` + fillRate + "%</b>)" : "(" + fillRate + "%)"],
+            ])
+            .tbodyColumnStyles([
+                { "font-weight": "bold", "font-size": config.secondaryFontSize + "px", "width": "1%" },
+                { "font-weight": "normal", "font-size": config.secondaryFontSize + "px", "text-align": "right", "width": "auto" },
+                { "font-weight": "normal", "font-size": config.secondaryFontSize + "px", "text-align": "left", "width": "1%" },
+            ])
+            ;
+    }
+
+}
+
+export class AttributeDesc extends Grid {
+
+    constructor(row) {
+        super();
+        this
+            .gutter(0)
+            .surfaceShadow(false)
+            .surfacePadding("0")
+            .surfaceBorderWidth(0)
+            .setContent(0, 0, new AttributeTitle(row))
+            .setContent(1, 0, new AttributeSummary(row))
+            ;
+    }
+
+}

esp/src/src/DataPatterns/Cardinality.ts

@@ -0,0 +1,34 @@
+import { BreakdownTable } from "@hpcc-js/html";
+import { config } from "./config";
+
+export class Cardinality extends BreakdownTable {
+
+    constructor(rows, showTitle: boolean = true) {
+        super();
+        if (showTitle) {
+            this.columns(["Cardinality", ""]);
+        }
+        this
+            .theadColumnStyles([{
+                "text-align": "left",
+                "font-size": config.secondaryFontSize + "px",
+                "white-space": "nowrap"
+            }])
+            .tbodyColumnStyles([{
+                "font-weight": "normal",
+                "max-width": "60px",
+                "overflow": "hidden",
+                "text-overflow": "ellipsis",
+                "white-space": "nowrap",
+            }, {
+                "font-size": config.secondaryFontSize + "px",
+                "font-weight": "normal",
+                "text-align": "right"
+            }])
+            .data(rows.map(row => [
+                row.value.trim(),
+                row.rec_count
+            ]))
+            ;
+    }
+}

esp/src/src/DataPatterns/DGridHeaderHook.ts

@@ -0,0 +1,91 @@
+import { HTMLWidget, select as d3Select, selectAll as d3SelectAll } from "@hpcc-js/common";
+import { Workunit } from "@hpcc-js/comms";
+import { StatChart } from "./StatChart";
+import { Cardinality } from './Cardinality';
+import { PopularPatterns } from './PopularPatterns';
+
+export class DGridHeaderHook {
+
+    protected _grid;
+    protected _gridID;
+    protected _statChart;
+
+    constructor(grid, gridID: string) {
+        this._grid = grid;
+        this._gridID = gridID;
+    }
+
+    createPlaceholder(fieldID) {
+        const headerElement = d3Select<HTMLElement, HTMLWidget>(`#${this._gridID} tr:not(.dgrid-spacer-row) .field-${fieldID} > .dgrid-resize-header-container`);
+        const rect = headerElement.node().getBoundingClientRect();
+        const placeholder = headerElement.append("div") // insert("div", ":first-child")
+            .attr("class", "placeholder")
+            ;
+        return { rect, placeholder };
+    }
+
+    render(wu: Workunit): Promise<void> {
+        d3SelectAll<HTMLElement, {}>(`#${this._gridID} > .dgrid-header tr:not(.dgrid-spacer-row) > th .placeholder`).remove();
+        if (wu) {
+            return wu.watchUntilComplete().then(wu => {
+                return wu.fetchResults();
+            }).then(results => {
+                return results.length ? results[0].fetchRows() : []
+            }).then(fields => {
+                fields.forEach(field => {
+                    if (field.is_numeric) {
+                        const { rect, placeholder } = this.createPlaceholder(field.attribute);
+                        this._statChart = new StatChart()
+                            .target(placeholder.node())
+                            .resize({ width: rect.width, height: 180 })
+                            .mean(field.numeric_mean)
+                            .standardDeviation(field.numeric_std_dev)
+                            .quartiles([
+                                field.numeric_min,
+                                field.numeric_lower_quartile,
+                                field.numeric_median,
+                                field.numeric_upper_quartile,
+                                field.numeric_max
+                            ])
+                            .lazyRender()
+                            ;
+                    } else if (field.cardinality_breakdown && field.cardinality_breakdown.Row && field.cardinality_breakdown.Row.length) {
+                        const { rect, placeholder } = this.createPlaceholder(field.attribute);
+                        this._statChart = new Cardinality(field.cardinality_breakdown.Row, false)
+                            .target(placeholder.node())
+                            .resize({ width: rect.width, height: 180 })
+                            .lazyRender()
+                            ;
+                    } else if (field.popular_patterns && field.popular_patterns.Row && field.popular_patterns.Row.length) {
+                        const { rect, placeholder } = this.createPlaceholder(field.attribute);
+                        this._statChart = new PopularPatterns(field.popular_patterns.Row.map(row => ({
+                            data_pattern: `[${row.data_pattern.trim()}]`,
+                            rec_count: row.rec_count
+                        })), false)
+                            .target(placeholder.node())
+                            .resize({ width: rect.width, height: 180 })
+                            .lazyRender()
+                            ;
+                    }
+                })
+            });
+        }
+        return Promise.resolve();
+    }
+
+    resize(columnId: string) {
+        const headerElement = d3Select<HTMLElement, HTMLWidget>(`#${this._gridID} tr:not(.dgrid-spacer-row) th.dgrid-column-${columnId} > .dgrid-resize-header-container`);
+        const element = headerElement.select(`.common_HTMLWidget`);
+        if (!element.empty()) {
+            const widget = element.datum();
+            if (widget) {
+                const rect = headerElement.node().getBoundingClientRect();
+                widget
+                    .resize({ width: rect.width, height: 180 })
+                    .lazyRender()
+                    ;
+            }
+        }
+    }
+}
+// DGridHeaderWidget.prototype._class += " eclwatch_DGridHeaderWidget";

esp/src/src/DataPatterns/DPWorkunit.ts

@@ -0,0 +1,148 @@
+import { LogicalFile, Result, Workunit } from "@hpcc-js/comms";
+import { globalKeyValStore, IKeyValStore } from "../KeyValStore";
+
+function analyzeECL(name: string, format: string) {
+    return `
+IMPORT STD.DataPatterns;
+
+filePath := '~${name}';
+ds := DATASET(filePath, RECORDOF(filePath, LOOKUP), ${format});
+profileResults := DataPatterns.Profile(ds);
+OUTPUT(profileResults, ALL, NAMED('profileResults'));
+    `;
+}
+
+function optimizeECL(origName, origFormat, newFields, transformFields, newName, overwrite) {
+    return `
+oldName := '~${origName}';
+oldLayout := RECORDOF(oldName, LOOKUP);
+oldDataset := DATASET(oldName, oldLayout, ${origFormat});
+
+NewLayout := RECORD
+${newFields}
+END;
+
+NewLayout MakeNewLayout(oldLayout L) := TRANSFORM
+${transformFields}
+    SELF := L;
+END;
+
+newDataset := PROJECT(oldDataset, MakeNewLayout(LEFT));
+OUTPUT(newDataset, , '~${newName}'${overwrite ? ", OVERWRITE" : ""});
+`;
+}
+
+export class DPWorkunit {
+    private readonly _lf: LogicalFile;
+    private readonly _store: IKeyValStore;
+    private readonly _storeID: string;
+    private readonly _storeWuidID: string;
+
+    private _wu: Workunit;
+    private _resultPromise;
+
+    constructor(nodeGroup: string, name: string) {
+        this._lf = LogicalFile.attach({ baseUrl: "" }, nodeGroup, name);
+        this._store = globalKeyValStore();
+        this._storeID = `dp-${nodeGroup}-${name}`;
+        this._storeWuidID = `${this._storeID}-wuid`;
+    }
+
+    clearCache() {
+        delete this._wu;
+        delete this._resultPromise;
+    }
+
+    resolveWU(): Promise<Workunit | undefined> {
+        return this._store.get(this._storeWuidID).then(wuid => {
+            if (this._wu && this._wu.Wuid === wuid) {
+                return this._wu
+            }
+            this.clearCache();
+            return wuid && Workunit.attach({ baseUrl: "" }, wuid);
+        }).then(wu => {
+            return wu && wu.refresh();
+        }).then(wu => {
+            if (wu && !wu.isDeleted()) {
+                this._wu = wu;
+                return wu;
+            }
+            return undefined;
+        });
+    }
+
+    refreshWU(): Promise<Workunit | undefined> {
+        return this.resolveWU().then(wu => {
+            if (wu) {
+                return wu.refresh();
+            }
+            return wu;
+        }).then(wu => {
+            if (wu && wu.Archived) {
+                return wu.restore().then(() => wu);
+            }
+            return wu;
+        }).then(wu => {
+            if (wu && !wu.isFailed()) {
+                return wu;
+            }
+            return undefined;
+        });
+    }
+
+    delete(): Promise<void> {
+        return this.resolveWU().then(wu => {
+            if (wu) {
+                wu.delete();
+            }
+            this.clearCache();
+            return this._store.delete(this._storeWuidID);
+        })
+    }
+
+    create(target: string): Promise<Workunit> {
+        return this.resolveWU().then(wu => {
+            if (wu) {
+                return wu;
+            }
+            return this._lf.fetchInfo().then(() => {
+                return Workunit.submit({ baseUrl: "" }, target, analyzeECL(this._lf.Name, this._lf.ContentType));
+            }).then(wu => {
+                this._wu = wu;
+                return this._store.set(this._storeWuidID, wu.Wuid).then(() => wu);
+            });
+        });
+    }
+
+    fetchResults() {
+        if (!this._resultPromise) {
+            if (!this._wu) {
+                this._resultPromise = Promise.resolve([]);
+            } else {
+                this._resultPromise = this._wu.fetchResults().then(results => {
+                    return results && results[0];
+                }).then((result?: Result) => {
+                    if (result) {
+                        return result.fetchRows();
+                    }
+                    return [];
+                });
+            }
+        }
+        return this._resultPromise;
+    }
+
+    optimize(target: string, name: string, overwrite: boolean) {
+        return Promise.all([this._lf.fetchInfo(), this.fetchResults()]).then(([lfInfo, rows]) => {
+            let fields = "";
+            let transformFields = "";
+            rows.forEach(row => {
+                if (fields.length) fields += "\n";
+                if (transformFields.length) transformFields += "\n";
+                fields += `    ${row.best_attribute_type} ${row.attribute};`
+                transformFields += `    SELF.${row.attribute} := (${row.best_attribute_type})L.${row.attribute};`
+            }, "");
+            return Workunit.submit({ baseUrl: "" }, target, optimizeECL(this._lf.Name, this._lf.ContentType, fields, transformFields, name, overwrite));
+        });
+    }
+}

esp/src/src/DataPatterns/NAWidget.ts

@@ -0,0 +1,18 @@
+import { Html } from "@hpcc-js/other";
+import { config } from "./config";
+
+export class NAWidget extends Html {
+    constructor(message, submessage) {
+        super();
+        this
+            .html(`
+                <b style="line-height:23px;font-size:${config.secondaryFontSize}px;color: rgb(51, 51, 51);">${message}</b>
+                <br/>
+                <i style="font-size:${config.secondaryFontSize}px;color: rgb(51, 51, 51);">${submessage}</i>
+            `)
+            .overflowX("hidden")
+            .overflowY("hidden")
+            ;
+
+    }
+}

esp/src/src/DataPatterns/PopularPatterns.ts

@@ -0,0 +1,33 @@
+import { BreakdownTable } from "@hpcc-js/html";
+import { config } from "./config";
+
+export class PopularPatterns extends BreakdownTable {
+    constructor(rows, showTitle: boolean = true) {
+        super();
+        if (showTitle) {
+            this.columns(["Popular Patterns", ""]);
+        }
+        this
+            .theadColumnStyles([{
+                "font-size": config.secondaryFontSize + "px",
+                "text-align": "left",
+                "white-space": "nowrap"
+            }])
+            .tbodyColumnStyles([{
+                "font-weight": "normal",
+                "max-width": "60px",
+                "overflow": "hidden",
+                "text-overflow": "ellipsis",
+                "white-space": "nowrap",
+            }, {
+                "font-size": config.secondaryFontSize + "px",
+                "font-weight": "normal",
+                "text-align": "right"
+            }])
+            .data(rows.map(row => [
+                row.data_pattern.trim(),
+                row.rec_count
+            ]))
+            ;
+    }
+}

esp/src/src/DataPatterns/Report.ts

@@ -0,0 +1,175 @@
+import { Widget } from "@hpcc-js/common";
+import { Grid } from "@hpcc-js/layout";
+import { StatChart, NumericStatsWidget, StringStatsWidget } from "./StatChart";
+import { config } from "./config";
+import { DPWorkunit } from "./DPWorkunit";
+import { PopularPatterns } from "./PopularPatterns";
+import { AttributeDesc } from "./AttributeDesc";
+import { NAWidget } from "./NAWidget";
+import { Cardinality } from './Cardinality';
+
+export class Report extends Grid {
+
+    private _data: any[];
+    private _fixedHeight?: number;
+    private _showBreakdownColumn = true;
+    private _showPopularPatternsColumn = true;
+    private _showQuartileColumn = true;
+
+    constructor() {
+        super();
+        this
+            .gutter(12)
+            .surfaceShadow(false)
+            .surfacePadding("0")
+            .surfaceBorderWidth(0)
+            ;
+    }
+
+    private _wu: DPWorkunit;
+    wu(): DPWorkunit;
+    wu(_: DPWorkunit): this;
+    wu(_?: DPWorkunit): DPWorkunit | this {
+        if (_ === void 0) return this._wu;
+        this._wu = _;
+        return this;
+    }
+
+    enter(domNode, element) {
+        this._fixedHeight = this._data.length * config.rowHeight;
+        domNode.style.height = this._fixedHeight + "px";
+        this.height(this._fixedHeight);
+        super.enter(domNode, element);
+        const statsDataWidth = this.calcStatsWidgetDataColumnWidth();
+        this._showQuartileColumn = this._data.filter(row => row.is_numeric).length > 0;
+        this._showBreakdownColumn = this._data.filter(row => row.cardinality_breakdown.Row.length > 0).length > 0;
+        this._showPopularPatternsColumn = this._data.filter(row => row.popular_patterns.Row.length > 0).length > 0;
+        let colCount = 3;
+        this._data.forEach((row, i) => {
+            const cc = this.enterDataRow(row, i, { statsDataWidth });
+            if (cc > colCount) {
+                colCount = cc;
+            }
+        });
+        element.classed("report-col-count-" + colCount, true);
+    }
+
+    enterDataRow(row, i, ext) {
+        const y = i * config.rowHeight;
+
+        let c = 2;
+        let cPos = 0;
+        let cStep = 12;
+        this.setContent(y, cPos, new AttributeDesc(row), undefined, config.rowHeight, cStep * config.colRatios.attributeDesc);
+        cPos += cStep * config.colRatios.attributeDesc;
+        this.setContent(y, cPos, getStatsWidget(row, ext.statsDataWidth), undefined, config.rowHeight, cStep * config.colRatios.statsData);
+        cPos += cStep * config.colRatios.statsData;
+        if (this._showQuartileColumn) {
+            this.setContent(y, cPos, getQuartileWidget(row), undefined, config.rowHeight, cStep * config.colRatios.quartile);
+            cPos += cStep * config.colRatios.quartile;
+            ++c;
+        }
+        if (this._showBreakdownColumn) {
+            this.setContent(y, cPos, getBreakdownWidget(row), undefined, config.rowHeight, cStep * config.colRatios.breakdown);
+            cPos += cStep * config.colRatios.breakdown;
+            ++c;
+        }
+        if (this._showPopularPatternsColumn) {
+            this.setContent(y, cPos, getPopularPatternsWidget(row), undefined, config.rowHeight, cStep * config.colRatios.popularPatterns);
+            cPos += cStep * config.colRatios.popularPatterns;
+            ++c;
+        }
+        return c;
+
+        function getBreakdownWidget(row) {
+            if (row.cardinality_breakdown.Row.length > 0) {
+                return new Cardinality(row.cardinality_breakdown.Row);
+            } else {
+                return new NAWidget("Cardinality", "N/A");
+            }
+        }
+        function getStatsWidget(row, dataWidth) {
+            if (row.is_numeric) {
+                return new NumericStatsWidget(row, dataWidth);
+            } else if (row.popular_patterns.Row.length > 0) {
+                return new StringStatsWidget(row);
+            }
+            return new AttributeDesc(row);
+        }
+        function getPopularPatternsWidget(row) {
+            if (row.popular_patterns.Row.length > 0) {
+                return new PopularPatterns(row.popular_patterns.Row);
+            } else {
+                return new NAWidget("Popular Patterns", "N/A");
+            }
+        }
+        function getQuartileWidget(row) {
+            if (row.is_numeric) {
+                return new StatChart()
+                    //.columns(["Min", "25%", "50%", "75%", "Max"])
+                    .mean(row.numeric_mean)
+                    .standardDeviation(row.numeric_std_dev)
+                    .quartiles([
+                        row.numeric_min,
+                        row.numeric_lower_quartile,
+                        row.numeric_median,
+                        row.numeric_upper_quartile,
+                        row.numeric_max
+                    ])
+                    ;
+            } else {
+                return new NAWidget("Quartile", "N/A");
+            }
+        }
+    }
+
+    update(domNode, element) {
+        if (this.width() < 800) {
+            this.width(800);
+        }
+        this.height(this._fixedHeight);
+        super.update(domNode, element);
+    }
+
+    resize(size?) {
+        const retVal = super.resize(size);
+        if (this._placeholderElement) {
+            this._placeholderElement
+                .style("height", (this._fixedHeight ? this._fixedHeight : this._size.height) + "px")
+                ;
+        }
+        return retVal;
+    }
+
+    render(callback?: (w: Widget) => void): this {
+        this._wu.fetchResults().then(rows => {
+            this._data = rows;
+            super.render(w => {
+                if (callback) {
+                    callback(this);
+                }
+            });
+        });
+        return this;
+    }
+
+    calcStatsWidgetDataColumnWidth() {
+        let ret = 0;
+        this._data.forEach(row => {
+            const _w = Math.max(
+                this.textSize(row.numeric_mean).width,
+                this.textSize(row.numeric_std_dev).width,
+                this.textSize(row.numeric_min).width,
+                this.textSize(row.numeric_lower_quartile).width,
+                this.textSize(row.numeric_median).width,
+                this.textSize(row.numeric_upper_quartile).width,
+                this.textSize(row.numeric_max).width
+            );
+            if (_w > ret) {
+                ret = _w;
+            }
+        });
+        return ret;
+    }
+}
+Report.prototype._class += " eclwatch_DPReport";

esp/src/src/DataPatterns/StatChart.ts

@@ -0,0 +1,257 @@
+import { QuartileCandlestick, Scatter } from "@hpcc-js/chart";
+import { format as d3Format, HTMLWidget, Palette } from "@hpcc-js/common";
+import { StyledTable } from "@hpcc-js/html";
+
+const rainbow = Palette.rainbow("Blues");
+const palette = Palette.ordinal("Quartile", [rainbow(100, 0, 100), rainbow(50, 0, 100), rainbow(50, 0, 100), rainbow(75, 0, 100)]);
+palette("Std. Dev.");
+palette("MinMax");
+palette("25%");
+palette("50%");
+
+type QuertilesType = [number, number, number, number, number];
+type Mode = "min_max" | "25_75" | "normal";
+type Tick = { label: string, value: number };
+type Ticks = Tick[];
+type AxisTick = { label: string, value: string };
+type AxisTicks = AxisTick[];
+
+const CANDLE_HEIGHT = 20;   // Pixels
+const DOMAIN_PADDING = 10;  // Percentage
+
+class BellCurve extends Scatter {
+}
+interface BellCurve {
+    xAxisTicks(): Array<{ value: string, label: string }>;
+    xAxisTicks(_: Array<{ value: string, label: string }>): this;
+}
+BellCurve.prototype.publishProxy("xAxisTicks", "domainAxis", "ticks");
+
+function myFormatter(format: string): (num: number) => string {
+    const formatter = d3Format(format);
+    return function (num: number) {
+        const strVal = (Math.round(num * 100) / 100).toString();
+        if (strVal.length <= 4) return strVal;
+        return formatter(num);
+    }
+}
+
+export class StatChart extends HTMLWidget {
+
+    private _selectMode: any;
+    private _tickFormatter: (_: number) => string;
+
+    private _bellCurve: BellCurve = new BellCurve()
+        .columns(["", "Std. Dev."])
+        .paletteID("Quartile")
+        .interpolate_default("basis")
+        .pointSize(0)
+        .xAxisType("linear")
+        .xAxisOverlapMode("none")
+        .xAxisTickFormat(".2s")
+        .yAxisHidden(true)
+        .yAxisDomainLow(0)
+        .yAxisDomainHigh(110)
+        .yAxisGuideLines(false) as BellCurve
+        ;
+
+    private _candle = new QuartileCandlestick()
+        .columns(["Min", "25%", "50%", "75%", "Max"])
+        .edgePadding(0)
+        .candleWidth(CANDLE_HEIGHT - 4)
+        .roundedCorners(1)
+        .lineWidth(1)
+        .upperTextRotation(-90)
+        .lowerTextRotation(-90)
+        .labelFontSize(0)
+        .valueFontSize(0)
+        .lineColor(rainbow(90, 0, 100))
+        .innerRectColor(rainbow(10, 0, 100))
+        ;
+
+    private stdDev(degrees: number): number {
+        return this.mean() + degrees * this.standardDeviation();
+    }
+
+    private formatStdDev(degrees: number): string {
+        return this._tickFormatter(this.stdDev(degrees));
+    }
+
+    private quartile(q: 0 | 1 | 2 | 3 | 4): number {
+        return this.quartiles()[q];
+    }
+
+    private formatQ(q: 0 | 1 | 2 | 3 | 4): string {
+        return this._tickFormatter(this.quartile(q));
+    }
+
+    private domain(mode: Mode): [number, number] {
+        switch (mode) {
+            case "25_75":
+                return [this.quartile(1), this.quartile(3)]
+            case "normal":
+                return [this.stdDev(-4), this.stdDev(4)];
+            case "min_max":
+            default:
+                return [this.quartile(0), this.quartile(4)];
+        }
+    }
+
+    enter(domNode, element) {
+        super.enter(domNode, element);
+
+        this._bellCurve.target(element.append("div").node());
+
+        this._candle.target(element.append("div").node());
+
+        this._selectMode = element.append("div")
+            .style("position", "absolute")
+            .style("top", "0px")
+            .style("right", "0px").append("select")
+            .on("change", () => this.render())
+            ;
+        this._selectMode.append("option").attr("value", "min_max").text("Min / Max");
+        this._selectMode.append("option").attr("value", "25_75").text("25% / 75%");
+        this._selectMode.append("option").attr("value", "normal").text("Normal");
+    }
+
+    private bellTicks(mode: Mode): AxisTicks {
+        let ticks: Ticks;
+        switch (mode) {
+            case "25_75":
+                ticks = [
+                    { label: this.formatQ(1), value: this.quartile(1) },
+                    { label: this.formatQ(2), value: this.quartile(2) },
+                    { label: this.formatQ(3), value: this.quartile(3) }
+                ];
+                break;
+            case "normal":
+                ticks = [
+                    { label: this.formatStdDev(-4), value: this.stdDev(-4) },
+                    { label: "-3σ", value: this.stdDev(-3) },
+                    { label: "-2σ", value: this.stdDev(-2) },
+                    { label: "-1σ", value: this.stdDev(-1) },
+                    { label: this.formatStdDev(0), value: this.stdDev(0) },
+                    { label: "+1σ", value: this.stdDev(1) },
+                    { label: "+2σ", value: this.stdDev(2) },
+                    { label: "+3σ", value: this.stdDev(3) },
+                    { label: this.formatStdDev(4), value: this.stdDev(4) },
+                ];
+                break;
+            case "min_max":
+            default:
+                ticks = [
+                    { label: this.formatQ(0), value: this.quartile(0) },
+                    { label: this.formatQ(1), value: this.quartile(1) },
+                    { label: this.formatQ(2), value: this.quartile(2) },
+                    { label: this.formatQ(3), value: this.quartile(3) },
+                    { label: this.formatQ(4), value: this.quartile(4) }
+                ];
+        }
+
+        const [domainLow, domainHigh] = this.domain(this._selectMode.node().value);
+        return ticks
+            .filter(sd => sd.value >= domainLow && sd.value <= domainHigh)
+            .map(sd => ({ label: sd.label, value: sd.value.toString() }))
+            ;
+    }
+
+    updateBellCurve() {
+        const mode = this._selectMode.node().value;
+        const [domainLow, domainHigh] = this.domain(mode);
+        const padding = (domainHigh - domainLow) * (DOMAIN_PADDING / 100);
+
+        this._bellCurve
+            .xAxisDomainLow(domainLow - padding)
+            .xAxisDomainHigh(domainHigh + padding)
+            .xAxisTicks(this.bellTicks(mode))
+            .data([
+                [this.stdDev(-4), 0],
+                [this.stdDev(-3), 0.3],
+                [this.stdDev(-2), 5],
+                [this.stdDev(-1), 68],
+                [this.stdDev(0), 100],
+                [this.stdDev(1), 68],
+                [this.stdDev(2), 5],
+                [this.stdDev(3), 0.3],
+                [this.stdDev(4), 0]
+            ])
+            .resize({ width: this.width(), height: this.height() - CANDLE_HEIGHT })
+            .render()
+            ;
+    }
+
+    updateCandle() {
+        const candleX = this._bellCurve.dataPos(this.quartile(0));
+        const candleW = this._bellCurve.dataPos(this.quartile(4)) - candleX;
+        this._candle
+            .resize({ width: this.width(), height: CANDLE_HEIGHT })
+            .pos({ x: candleX + candleW / 2, y: CANDLE_HEIGHT / 2 })
+            .width(candleW)
+            .data(this.quartiles())
+            .render()
+            ;
+    }
+
+    update(domNode, element) {
+        super.update(domNode, element);
+        this._tickFormatter = myFormatter(this.tickFormat());
+        this.updateBellCurve();
+        this.updateCandle();
+    }
+}
+export interface StatChart {
+    tickFormat(): string;
+    tickFormat(_: string): this;
+    mean(): number;
+    mean(_: number): this;
+    standardDeviation(): number;
+    standardDeviation(_: number): this;
+    quartiles(): QuertilesType;
+    quartiles(_: QuertilesType): this;
+}
+StatChart.prototype.publish("tickFormat", ".2e", "string", "X-Axis Tick Format");
+StatChart.prototype.publish("mean", 0, "number", "Mean");
+StatChart.prototype.publish("standardDeviation", 0, "number", "Standard Deviation");
+StatChart.prototype.publish("quartiles", [0, 0, 0, 0, 0], "array", "Quartiles");
+
+export class NumericStatsWidget extends StyledTable {
+    constructor(row, dataWidth) {
+        super();
+
+        this
+            .tbodyColumnStyles([
+                { "font-weight": "bold", "text-align": "right", "width": "100px" },
+                { "font-weight": "normal", "width": dataWidth + "px" },
+                { "font-weight": "normal", "width": "auto" },
+            ])
+            .data([
+                ["Mean", row.numeric_mean, ""],
+                ["Std. Deviation", row.numeric_std_dev, ""],
+                ["", "", ""],
+                ["Quartiles", row.numeric_min, "Min"],
+                ["", row.numeric_lower_quartile, "25%"],
+                ["", row.numeric_median, "50%"],
+                ["", row.numeric_upper_quartile, "75%"],
+                ["", row.numeric_max, "Max"]
+            ])
+            ;
+    }
+}
+
+export class StringStatsWidget extends StyledTable {
+    constructor(row) {
+        super();
+        return new StyledTable()
+            .tbodyColumnStyles([
+                { "font-weight": "bold", "text-align": "right", "width": "100px" },
+                { "font-weight": "normal", "width": "auto" },
+            ])
+            .data([
+                ["Min Length", row.min_length],
+                ["Avg Length", row.ave_length],
+                ["Max Length", row.max_length]
+            ])
+            ;
+    }
+}

esp/src/src/DataPatterns/config.ts

@@ -0,0 +1,18 @@
+export const config = {
+    rowHeight: 190,
+    colRatios: {
+        attributeDesc: 1,
+        statsData: 1,
+        breakdown: 2 / 3,
+        quartile: 1,
+        popularPatterns: 1
+    },
+    primaryFontSize: 20,
+    secondaryFontSize: 13,
+    primaryColor: "#494949",
+    secondaryColor: "#DDD",
+    offwhiteColor: "#FBFBFB",
+    blueColor: "#1A99D5",
+    redColor: "#ED1C24",
+    fillRateRedThreshold: 33
+};

esp/src/src/DataPatternsWidget.ts

@@ -0,0 +1,275 @@
+import "dojo/i18n";
+// @ts-ignore
+import * as nlsHPCC from "dojo/i18n!hpcc/nls/hpcc";
+import * as dom from "dojo/dom";
+import * as domClass from "dojo/dom-class";
+import * as domForm from "dojo/dom-form";
+
+import * as registry from "dijit/registry";
+
+import { select as d3Select } from "@hpcc-js/common";
+import { Workunit } from "@hpcc-js/comms";
+import { DPWorkunit } from "./DataPatterns/DPWorkunit";
+import { Report } from "./DataPatterns/Report";
+import { getStateIconClass } from "./ESPWorkunit";
+
+// @ts-ignore
+import * as _TabContainerWidget from "hpcc/_TabContainerWidget";
+// @ts-ignore
+import * as DelayLoadWidget from "hpcc/DelayLoadWidget";
+
+// @ts-ignore
+import * as template from "dojo/text!hpcc/templates/DataPatternsWidget.html";
+
+import "dijit/layout/BorderContainer";
+import "dijit/layout/TabContainer";
+import "dijit/layout/ContentPane";
+import "dijit/Toolbar";
+import "dijit/ToolbarSeparator";
+import "hpcc/TargetSelectWidget";
+import "dijit/TooltipDialog";
+import "dijit/form/Form";
+import "dijit/form/Button";
+import "dijit/form/DropDownButton";
+import "dijit/Fieldset";
+import "dijit/form/CheckBox";
+import "hpcc/TableContainer";
+
+import { declareDecorator } from './DeclareDecorator';
+import { WUStatus } from './WUStatus';
+
+type _TabContainerWidget = {
+    id: string;
+    widget: any;
+    params: { [key: string]: any };
+    inherited(args: any);
+    setDisabled(id: string, disabled: boolean, icon?: string, disabledIcon?: string);
+    getSelectedChild(): any;
+    createChildTabID(id: string): string;
+    addChild(tabItem: any, pos: number);
+};
+
+export interface DataPatternsWidget extends _TabContainerWidget {
+}
+
+@declareDecorator("DataPatternsWidget", _TabContainerWidget)
+export class DataPatternsWidget {
+    templateString = template;
+    static baseClass = "DataPatternsWidget";
+    i18n = nlsHPCC;
+
+    summaryWidget;
+    rawDataWidget;
+    workunitWidget;
+    centerContainer;
+    targetSelectWidget;
+    optimizeForm;
+    optimizeTargetSelect;
+    optimizeTarget;
+
+    wuStatus: WUStatus;
+    dpReport: Report;
+
+    _wu: Workunit;
+    _dpWu: DPWorkunit;
+
+    constructor() {
+    }
+
+    //  Data ---
+    //  --- ---
+
+    buildRendering(args) {
+        this.inherited(arguments);
+    }
+
+    postCreate(args) {
+        this.inherited(arguments);
+
+        this.summaryWidget = registry.byId(this.id + "_Summary");
+        this.rawDataWidget = registry.byId(this.id + "_RawData");
+        this.workunitWidget = registry.byId(this.id + "_Workunit");
+        this.centerContainer = registry.byId(this.id + "CenterContainer");
+        this.targetSelectWidget = registry.byId(this.id + "TargetSelect");
+
+        this.optimizeForm = registry.byId(this.id + "OptimizeForm");
+        this.optimizeTargetSelect = registry.byId(this.id + "OptimizeTargetSelect");
+        this.optimizeTarget = registry.byId(this.id + "OptimizeTarget");
+
+        const context = this;
+        const origResize = this.centerContainer.resize;
+        this.centerContainer.resize = function (s) {
+            origResize.apply(this, arguments);
+            d3Select(`#${context.id}DPReport`).style("height", `${s.h - 16}px`);  // 8 * 2 margins
+            if (context.dpReport && context.dpReport.renderCount()) {
+                context.dpReport
+                    .resize()
+                    .lazyRender()
+                    ;
+            }
+        }
+
+        this.wuStatus = new WUStatus()
+            .baseUrl("")
+            ;
+        this.dpReport = new Report();
+
+    }
+
+    startup(args) {
+        this.inherited(arguments);
+    }
+
+    resize(s) {
+        this.inherited(arguments);
+    }
+
+    layout(args) {
+        this.inherited(arguments);
+    }
+
+    destroy(args) {
+        this.inherited(arguments);
+    }
+
+    //  Implementation  ---
+    _onRefresh() {
+        this.refreshData(true);
+    }
+
+    _onAnalyze() {
+        const target = this.targetSelectWidget.get("value");
+        this._dpWu.create(target).then(() => {
+            this.refreshData();
+        });
+    }
+
+    _onOptimizeOk() {
+        if (this.optimizeForm.validate()) {
+            const request = domForm.toObject(this.optimizeForm.domNode);
+            this._dpWu.optimize(request.target, request.name, request.overwrite === "on").then(wu => {
+                this.ensureWUPane(wu.Wuid);
+            });
+        }
+        registry.byId(this.id + "OptimizeDropDown").closeDropDown();
+    }
+
+    _onDelete() {
+        this._dpWu.delete().then(() => {
+            this.rawDataWidget.reset();
+            this.workunitWidget.reset();
+            this.refreshData();
+        });
+    }
+
+    init(params) {
+        if (this.inherited(arguments))
+            return;
+
+        this.targetSelectWidget.init({});
+        this.optimizeTargetSelect.init({});
+        this.optimizeTarget.set("value", params.LogicalName + "::optimized");
+
+        this._dpWu = new DPWorkunit(params.NodeGroup, params.LogicalName);
+
+        this.wuStatus.target(this.id + "WUStatus");
+        this.dpReport.target(this.id + "DPReport");
+
+        this.refreshData();
+    }
+
+    initTab() {
+        var currSel = this.getSelectedChild();
+        if (currSel && !currSel.initalized) {
+            if (currSel.id === this.summaryWidget.id) {
+            } else if (this.rawDataWidget && currSel.id === this.rawDataWidget.id) {
+                if (this._wu) {
+                    this.rawDataWidget.init({
+                        Wuid: this._wu.Wuid,
+                        Name: "profileResults"
+                    });
+                }
+            } else if (this.workunitWidget && currSel.id === this.workunitWidget.id) {
+                if (this._wu) {
+                    this.workunitWidget.init({
+                        Wuid: this._wu.Wuid
+                    });
+                }
+            } else {
+                currSel.init(currSel.params);
+            }
+        }
+    }
+
+    ensureWUPane(wuid: string) {
+        const id = this.createChildTabID(wuid);
+        var retVal = registry.byId(id);
+        if (!retVal) {
+            retVal = new DelayLoadWidget({
+                id: id,
+                title: wuid,
+                closable: true,
+                delayWidget: "WUDetailsWidget",
+                params: { Wuid: wuid }
+            });
+            this.addChild(retVal, 3);
+        }
+        return retVal;
+    }
+
+    refreshData(full: boolean = false) {
+        if (full) {
+            this._dpWu.clearCache();
+        }
+        this._dpWu.resolveWU().then(wu => {
+            if (this._wu !== wu) {
+                this._wu = wu;
+                dom.byId(this.id + "Wuid").textContent = this._wu ? this._wu.Wuid : "";
+                this.wuStatus
+                    .wuid(this._wu ? this._wu.Wuid : "")
+                    .lazyRender()
+                    ;
+
+                if (this._wu) {
+                    this._wu.watchUntilComplete(changes => {
+                        if (this._wu && this._wu.isComplete()) {
+                            this.dpReport
+                                .wu(this._dpWu)
+                                .render(w => {
+                                    w
+                                        .resize()
+                                        .render()
+                                        ;
+                                });
+                        }
+                        this.refreshActionState();
+                    });
+                }
+            }
+            this.refreshActionState();
+        });
+    }
+
+    refreshActionState() {
+        const isComplete = this._wu && this._wu.isComplete();
+        this.setDisabled(this.id + "Analyze", !!this._wu);
+        d3Select(`#${this.id}TargetSelectLabel`).style("color", !!this._wu ? "rgb(158,158,158)" : null);
+        this.setDisabled(this.id + "TargetSelect", !!this._wu);
+        this.setDisabled(this.id + "Delete", !this._wu);
+        this.setDisabled(this.id + "OptimizeDropDown", !isComplete);
+        this.setDisabled(this.id + "_RawData", !isComplete);
+        this.setDisabled(this.id + "_Workunit", !this._wu);
+
+        if (this._wu) {
+            this.targetSelectWidget.set("value", this._wu.Cluster);
+        }
+
+        const stateIconClass = this._wu ? getStateIconClass(this._wu.StateID, this._wu.isComplete(), this._wu.Archived) : "";
+        this.workunitWidget.set("iconClass", stateIconClass);
+        domClass.remove(this.id + "StateIdImage");
+        domClass.add(this.id + "StateIdImage", stateIconClass);
+
+        d3Select(`#${this.id}WU`).style("display", isComplete ? "none" : null);
+        d3Select(`#${this.id}DPReport`).style("display", isComplete ? null : "none");
+    }
+}

esp/src/src/ESPLogicalFile.ts

@@ -11,6 +11,7 @@ import * as ESPRequest from "./ESPRequest";
 import * as ESPUtil from "./ESPUtil";
 import * as ESPResult from "./ESPResult";
 import * as Utility from "./Utility";
+import { DPWorkunit } from "./DataPatterns/DPWorkunit";
 
 var _logicalFiles = {};
 
@@ -192,6 +193,8 @@ var TreeStore = declare(null, {
 });
 
 var LogicalFile = declare([ESPUtil.Singleton], {    // jshint ignore:line
+    _dpWU: DPWorkunit,
+
     _FileDetailSetter: function (FileDetail) {
         this.FileDetail = FileDetail;
         this.result = ESPResult.Get(FileDetail);
@@ -236,6 +239,7 @@ var LogicalFile = declare([ESPUtil.Singleton], {    // jshint ignore:line
             declare.safeMixin(this, args);
         }
         this.logicalFile = this;
+        this._dpWU = new DPWorkunit(this.Cluster, this.Name);
     },
     save: function (request, args) {
         //WsDfu/DFUInfo?FileName=progguide%3A%3Aexampledata%3A%3Akeys%3A%3Apeople.lastname.firstname&UpdateDescription=true&FileDesc=%C2%A0123&Save+Description=Save+Description
@@ -407,6 +411,9 @@ var LogicalFile = declare([ESPUtil.Singleton], {    // jshint ignore:line
     },
     isDeleted: function () {
         return this.StateID === 999;
+    },
+    fetchDataPatternsWU() {
+        return this._dpWU.resolveWU();
     }
 });
 

esp/src/src/ESPWorkunit.ts

@@ -23,6 +23,96 @@ declare const dojo;
 
 var _workunits = {};
 
+export function getStateIconClass(stateID: number, complete: boolean, archived: boolean): string {
+    if (archived) {
+        return "iconArchived";
+    }
+    switch (stateID) {
+        case 1:
+            if (complete) {
+                return "iconCompleted";
+            }
+            return "iconSubmitted";
+        case 3:
+            return "iconCompleted";
+        case 2:
+        case 11:
+        case 15:
+            return "iconRunning";
+        case 4:
+        case 7:
+            return "iconFailed";
+        case 5:
+        case 8:
+        case 10:
+        case 12:
+        case 13:
+        case 14:
+        case 16:
+            return "iconArchived";
+        case 6:
+            return "iconAborting";
+        case 9:
+            return "iconSubmitted";
+        case 999:
+            return "iconDeleted";
+    }
+    return "iconWorkunit";
+}
+
+export function getStateImageName(stateID: number, complete: boolean, archived: boolean): string {
+    if (archived) {
+        return "workunit_archived.png";
+    }
+    switch (stateID) {
+        case 1:
+            if (complete) {
+                return "workunit_completed.png";
+            }
+            return "workunit_submitted.png";
+        case 2:
+            return "workunit_running.png";
+        case 3:
+            return "workunit_completed.png";
+        case 4:
+            return "workunit_failed.png";
+        case 5:
+            return "workunit_warning.png";
+        case 6:
+            return "workunit_aborting.png";
+        case 7:
+            return "workunit_failed.png";
+        case 8:
+            return "workunit_warning.png";
+        case 9:
+            return "workunit_submitted.png";
+        case 10:
+            return "workunit_warning.png";
+        case 11:
+            return "workunit_running.png";
+        case 12:
+            return "workunit_warning.png";
+        case 13:
+            return "workunit_warning.png";
+        case 14:
+            return "workunit_warning.png";
+        case 15:
+            return "workunit_running.png";
+        case 16:
+            return "workunit_warning.png";
+        case 999:
+            return "workunit_deleted.png";
+    }
+    return "workunit.png";
+}
+export function getStateImage(stateID: number, complete: boolean, archived: boolean): string {
+    return Utility.getImageURL(getStateImageName(stateID, complete, archived));
+}
+
+export function getStateImageHTML(stateID: number, complete: boolean, archived: boolean): string {
+    return Utility.getImageHTML(getStateImageName(stateID, complete, archived));
+}
+
 var Store = declare([ESPRequest.Store], {
     service: "WsWorkunits",
     action: "WUQuery",
@@ -648,91 +738,16 @@ var Workunit = declare([ESPUtil.Singleton], {  // jshint ignore:line
         return this.State;
     },
     getStateIconClass: function () {
-        if (this.Archived) {
-            return "iconArchived";
-        }
-        switch (this.StateID) {
-            case 1:
-                if (this.isComplete()) {
-                    return "iconCompleted";
-                }
-                return "iconSubmitted";
-            case 3:
-                return "iconCompleted";
-            case 2:
-            case 11:
-            case 15:
-                return "iconRunning";
-            case 4:
-            case 7:
-                return "iconFailed";
-            case 5:
-            case 8:
-            case 10:
-            case 12:
-            case 13:
-            case 14:
-            case 16:
-                return "iconArchived";
-            case 6:
-                return "iconAborting";
-            case 9:
-                return "iconSubmitted";
-            case 999:
-                return "iconDeleted";
-        }
-        return "iconWorkunit";
+        return getStateIconClass(this.StateID, this.isComplete(), this.Archived);
     },
     getStateImageName: function () {
-        if (this.Archived) {
-            return "workunit_archived.png";
-        }
-        switch (this.StateID) {
-            case 1:
-                if (this.isComplete()) {
-                    return "workunit_completed.png";
-                }
-                return "workunit_submitted.png";
-            case 2:
-                return "workunit_running.png";
-            case 3:
-                return "workunit_completed.png";
-            case 4:
-                return "workunit_failed.png";
-            case 5:
-                return "workunit_warning.png";
-            case 6:
-                return "workunit_aborting.png";
-            case 7:
-                return "workunit_failed.png";
-            case 8:
-                return "workunit_warning.png";
-            case 9:
-                return "workunit_submitted.png";
-            case 10:
-                return "workunit_warning.png";
-            case 11:
-                return "workunit_running.png";
-            case 12:
-                return "workunit_warning.png";
-            case 13:
-                return "workunit_warning.png";
-            case 14:
-                return "workunit_warning.png";
-            case 15:
-                return "workunit_running.png";
-            case 16:
-                return "workunit_warning.png";
-            case 999:
-                return "workunit_deleted.png";
-        }
-        return "workunit.png";
+        return getStateImageName(this.StateID, this.isComplete(), this.Archived);
     },
     getStateImage: function () {
-        return Utility.getImageURL(this.getStateImageName());
+        return getStateImage(this.StateID, this.isComplete(), this.Archived);
     },
     getStateImageHTML: function () {
-        return Utility.getImageHTML(this.getStateImageName());
+        return getStateImageHTML(this.StateID, this.isComplete(), this.Archived);
     },
     getProtectedImageName: function () {
         if (this.Protected) {

esp/src/src/Utility.ts

@@ -419,6 +419,9 @@ export function resolve(hpccWidget, callback) {
         case "viz/DojoD3NDChart":
             require(["hpcc/viz/DojoD3NDChart"], doLoad);
             break;
+        case "DataPatternsWidget":
+            require(["src/DataPatternsWidget"], doLoad);
+            break;
         case "DynamicESDLDefinitionDetailsWidget":
             require(["hpcc/DynamicESDLDefinitionDetailsWidget"], doLoad);
             break;
@@ -461,6 +464,9 @@ export function resolve(hpccWidget, callback) {
         case "FullResultWidget":
             require(["hpcc/FullResultWidget"], doLoad);
             break;
+        case "GangliaWidget":
+            require(["ganglia/GangliaWidget"], doLoad);
+            break;
         case "GetDFUWorkunitsWidget":
             require(["hpcc/GetDFUWorkunitsWidget"], doLoad);
             break;
@@ -708,9 +714,6 @@ export function resolve(hpccWidget, callback) {
         case "XrefQueryWidget":
             require(["hpcc/XrefQueryWidget"], doLoad);
             break;
-        case "GangliaWidget":
-            require(["ganglia/GangliaWidget"], doLoad);
-            break;
         default:
             console.log("case \"" + hpccWidget + "\":\n" +
                 "    require([\"hpcc/" + hpccWidget + "\"], doLoad);\n" +

esp/src/src/WUStatus.ts

@@ -8,7 +8,7 @@ import * as nlsHPCC from "dojo/i18n!hpcc/nls/hpcc";
 
 export class WUStatus extends Graph {
 
-    protected _hpccWU: Workunit;
+    protected _hpccWU: Workunit | undefined;
     protected _hpccWatchHandle: IObserverHandle;
 
     protected _create: Vertex;
@@ -36,7 +36,8 @@ export class WUStatus extends Graph {
         });
         if (this._prevHash !== hash) {
             this._prevHash = hash;
-            this._hpccWU = Workunit.attach({ baseUrl: this.baseUrl() }, this.wuid());
+            const wuid = this.wuid();
+            this._hpccWU = wuid ? Workunit.attach({ baseUrl: this.baseUrl() }, wuid) : undefined;
             this.stopMonitor();
             this.startMonitor();
         }
@@ -56,8 +57,9 @@ export class WUStatus extends Graph {
     }
 
     startMonitor() {
-        if (this.isMonitoring())
+        if (!this._hpccWU || this.isMonitoring())
             return;
+
         this._hpccWatchHandle = this._hpccWU.watch(changes => {
             this.lazyRender();
         }, true);
@@ -92,7 +94,7 @@ export class WUStatus extends Graph {
     }
 
     updateVertexStatus(level: 0 | 1 | 2 | 3 | 4, active: boolean = false) {
-        const completeColor = this._hpccWU.isFailed() ? "darkred" : "darkgreen";
+        const completeColor = this._hpccWU && this._hpccWU.isFailed() ? "darkred" : "darkgreen";
         this._create.text(nlsHPCC.Created);
         this._compile.text(nlsHPCC.Compiled);
         this._execute.text(nlsHPCC.Executed);
@@ -163,7 +165,7 @@ export class WUStatus extends Graph {
 
     update(domNode, element) {
         this.attachWorkunit();
-        switch (this._hpccWU.StateID) {
+        switch (this._hpccWU ? this._hpccWU.StateID : WUStateID.Unknown) {
             case WUStateID.Blocked:
             case WUStateID.Wait:
             case WUStateID.Scheduled: